def test_base(self):
        self.assertEqual(JalaliDateTime(1367, 2, 14, 14, 0, 0, 0),
                         JalaliDateTime.to_jalali(datetime(1988, 5, 4, 14, 0, 0, 0)))
        self.assertEqual(JalaliDateTime(1369, 7, 1, 14, 14, 1, 1111),
                         JalaliDateTime(datetime(1990, 9, 23, 14, 14, 1, 1111)))
        self.assertEqual(JalaliDateTime(1369, 7, 1, 14, 14, 1, 9111),
                         JalaliDateTime(JalaliDateTime(1369, 7, 1, 14, 14, 1, 9111)))

        g = JalaliDateTime.now()
        self.assertEqual(g.time(), _time(g.hour, g.minute, g.second, g.microsecond))

        g = g.replace(tzinfo=pytz.timezone("America/Los_Angeles"))
        self.assertEqual(g.timetz(),
                         _time(g.hour, g.minute, g.second, g.microsecond, pytz.timezone("America/Los_Angeles")))

        self.assertEqual(JalaliDateTime.fromtimestamp(578723400, pytz.utc),
                         JalaliDateTime(1367, 2, 14, 4, 30, 0, 0, pytz.utc))
        self.assertEqual(JalaliDateTime.utcfromtimestamp(578723400), JalaliDateTime(1367, 2, 14, 4, 30, 0, 0))

        try:
            JalaliDateTime._check_time_fields(20, 1, 61, 1000)
        except ValueError:
            assert True
        else:
            assert False

        try:
            JalaliDateTime._check_time_fields("20", 1, 61, 1000)
        except TypeError:
            assert True
        else:
            assert False
    def __add__(self, other):
        if not isinstance(other, timedelta):
            raise NotImplementedError

        delta = timedelta(
            self.toordinal(),
            hours=self._hour,
            minutes=self._minute,
            seconds=self._second,
            microseconds=self._microsecond,
        )
        delta += other
        hour, rem = divmod(delta.seconds, 3600)
        minute, second = divmod(rem, 60)

        if 0 < delta.days <= _MAXORDINAL:
            return JalaliDateTime.combine(
                JalaliDate.fromordinal(delta.days),
                _time(hour,
                      minute,
                      second,
                      delta.microseconds,
                      tzinfo=self._tzinfo),
            )

        raise OverflowError("result out of range")
    def to_jalali(
        cls,
        year,
        month=None,
        day=None,
        hour=None,
        minute=None,
        second=None,
        microsecond=None,
        tzinfo=None,
    ):
        if month is None and isinstance(year, dt):
            month = year.month
            day = year.day
            hour = year.hour
            minute = year.minute
            second = year.second
            microsecond = year.microsecond
            tzinfo = year.tzinfo
            year = year.year

        j_date = JalaliDate.to_jalali(year, month, day)

        return cls.combine(
            j_date,
            _time(hour=hour,
                  minute=minute,
                  second=second,
                  microsecond=microsecond,
                  tzinfo=tzinfo),
        )
Esempio n. 4
0
def time(ctx, hours, minutes, seconds):
    """
    Defines a time value
    """
    return _time(conversions.to_integer(hours, ctx),
                 conversions.to_integer(minutes, ctx),
                 conversions.to_integer(seconds, ctx))
Esempio n. 5
0
    def to_gregorian(self):
        g_date = super(JalaliDateTime, self).to_gregorian()

        return dt.combine(g_date, _time(hour=self._hour, minute=self._minute,
                                        second=self._second,
                                        microsecond=self._microsecond,
                                        tzinfo=self._tzinfo))
Esempio n. 6
0
    def test_base(self):
        self.assertEqual(
            JalaliDateTime(1369, 7, 1, 14, 14, 1, 9111),
            JalaliDateTime(JalaliDateTime(1369, 7, 1, 14, 14, 1, 9111)),
        )

        g = JalaliDateTime.now()
        self.assertEqual(g.time(),
                         _time(g.hour, g.minute, g.second, g.microsecond))

        g = g.replace(tzinfo=pytz.timezone("America/Los_Angeles"))
        self.assertEqual(
            g.timetz(),
            _time(
                g.hour,
                g.minute,
                g.second,
                g.microsecond,
                pytz.timezone("America/Los_Angeles"),
            ),
        )

        self.assertEqual(
            JalaliDateTime.fromtimestamp(578723400, pytz.utc),
            JalaliDateTime(1367, 2, 14, 4, 30, 0, 0, pytz.utc),
        )
        self.assertEqual(
            JalaliDateTime.utcfromtimestamp(578723400),
            JalaliDateTime(1367, 2, 14, 4, 30, 0, 0),
        )

        with pytest.raises(TypeError):
            JalaliDateTime._check_time_fields("20", 1, 61, 1000)

        with pytest.raises(ValueError):
            JalaliDateTime(1367, 2, 14, 25, 0, 0, 0)

        with pytest.raises(ValueError):
            JalaliDateTime(1367, 2, 14, 22, 61, 0, 0)

        with pytest.raises(ValueError):
            JalaliDateTime(1367, 2, 14, 22, 1, 722, 0)

        with pytest.raises(ValueError):
            JalaliDateTime(1367, 2, 14, 22, 1, 0, 1000000)
    def to_gregorian(self):
        g_date = super(JalaliDateTime, self).to_gregorian()

        return dt.combine(
            g_date,
            _time(
                hour=self._hour,
                minute=self._minute,
                second=self._second,
                microsecond=self._microsecond,
                tzinfo=self._tzinfo,
            ),
        )
Esempio n. 8
0
 def __init__(_, dtime=None, **sets):
     super().__init__(**sets)
     if some(dtime) and issubclass(type(dtime), _time): _._time = dtime
     else:
         d, h, m, s, ms = _._default('d', 0), _._default(
             'h', 0), _._default('m',
                                 0), _._default('s',
                                                0), _._default('ms', 0)
         if not any([d, h, m, s, ms]):
             now = datetime.now()
             _._time = now - datetime(now.year, now.month, now.day)
         else:
             _._time = _time(days=d,
                             hours=h,
                             minutes=m,
                             seconds=s,
                             milliseconds=ms)
     _.clear('d', 'h', 'm', 's', 'ms')
Esempio n. 9
0
    def to_jalali(cls, year, month=None, day=None, hour=None, minute=None,
                  second=None, microsecond=None, tzinfo=None):
        if month is None and isinstance(year, dt):
            month = year.month
            day = year.day
            hour = year.hour
            minute = year.minute
            second = year.second
            microsecond = year.microsecond
            tzinfo = year.tzinfo
            year = year.year

        j_date = JalaliDate.to_jalali(year, month, day)

        return cls.combine(j_date,
                           _time(hour=hour, minute=minute, second=second,
                                 microsecond=microsecond,
                                 tzinfo=tzinfo))
Esempio n. 10
0
    def __add__(self, other):
        if not isinstance(other, timedelta):
            return NotImplemented

        delta = timedelta(self.toordinal(),
                          hours=self._hour,
                          minutes=self._minute,
                          seconds=self._second,
                          microseconds=self._microsecond)
        delta += other
        hour, rem = divmod(delta.seconds, 3600)
        minute, second = divmod(rem, 60)

        if 0 < delta.days <= _MAXORDINAL:
            return JalaliDateTime.combine(JalaliDate.fromordinal(delta.days),
                                          _time(hour, minute, second,
                                                delta.microseconds,
                                                tzinfo=self._tzinfo))

        raise OverflowError("result out of range")
Esempio n. 11
0
def convert_to_datetime(src):
    if src is None:
        return

    elif isinstance(src, datetime):
        dst = src

    elif isinstance(src, date):
        dst = datetime.combine(src, _time())
        
    elif isinstance(src, basestring):
        m = _DATE_REGEX.match(src)
        if not m:
            raise ValueError('Invalid date string')
        values = [(k, int(v or 0)) for k, v in m.groupdict().items()]
        values = dict(values)
        dst = datetime(**values)

    else:
        raise TypeError('Unsupported type for %s: %s' % (arg_name, src.__class__.__name__))

    return dst
    def run(self):
        # Check that ElasticSearch is alive
        self.check_index()

        # If the user specified the --REBUILD flag, recreate the index
        if self.options['rebuild']:
            self.rebuild_index()

        # Connect to the repository
        registry = MetadataRegistry()
        registry.registerReader(self.settings["metadata_format"], self.settings["metadata_reader"])

        client = Client(self.settings["uri"], registry)
        identity = client.identify()

        print "Connected to repository: %s" % identity.repositoryName()

        # got to update granularity or we barf with: 
        # oaipmh.error.BadArgumentError: Max granularity is YYYY-MM-DD:2003-04-10T00:00:00Z
        client.updateGranularity()

        # Initialise some variables
        batcher = Batch.Batch()
        total_records = 0
        start = time.time()
        
        # Now do the synchonisation
        
        # If the user specified an identifier, then synchronise this record
        if (self.options['identifier'] is not None):
            total_records += self.synchronise_record(client, batcher, self.options['identifier'])
        else:
            # Else, synchronise using the date-range provided by the user, or failing that, 
            # the date-range based on the last sync

            # Get the synchronisation config record
            synchronisation_config = self.get_synchronisation_config()

            
            if self.options["from_date"] is not None:
                # If the user specified a from-date argument, use it
                from_date = self.options["from_date"] # already a date (not a datetime)
            elif synchronisation_config is not None and "to_date" in synchronisation_config:
                # Else read the last synchronised to_date from the config, and add on a day
                from_date = dateutil.parser.parse(synchronisation_config["to_date"]).date() + timedelta(days=1)
            else:
                # Else use the default_from_date in the config
                from_date = dateutil.parser.parse(self.settings['default_from_date']).date()

            if self.options["to_date"] is not None:
                to_date = self.options["to_date"] # already a date (not a datetime)
            else:
                to_date = (date.today() - timedelta(days=1))
            
            # Force the from_date to use time 00:00:00
            from_date = datetime.combine(from_date, _time(hour=0, minute=0, second=0, microsecond=0))

            # Force the to_date to use time 23:59:59
            to_date = datetime.combine(to_date, _time(hour=23, minute=59, second=59, microsecond=0))


            print "Synchronising from %s - %s" % (from_date, to_date)

            while from_date < to_date:
                next_date = datetime.combine(from_date.date() + timedelta(days=(self.settings['delta_days'] - 1)), _time(hour=23, minute=59, second=59, microsecond=0))
                number_of_records = self.synchronise_period(client, batcher, from_date, next_date)
                batcher.clear() #Store the records in elasticsearch
                self.put_synchronisation_config(from_date, next_date, number_of_records)
                from_date += timedelta(days=(self.settings['delta_days']))
                total_records += number_of_records

                # Pause so as not to get banned.
                to = 20
                print "Sleeping for %i seconds so as not to get banned." % to
                time.sleep(to)

            
        # Store the records in the index
        batcher.clear()
        
        # Print out some statistics
        time_spent = time.time() - start
        print 'Total time spent: %d seconds' % (time_spent)

        if time_spent > 0.001: # careful as its not an integer
            print 'Total records synchronised: %i records (%d records/second)' % (total_records, (total_records/time_spent))
        else:
            print 'Total records synchronised: %i records' % (total_records)
        return total_records

        sys.exit()
Esempio n. 13
0
    def __process(self, data):
        """
        Process data to fill properties of Burp object.
        """
        self.host = data.get('host', None)
        self.ip_address = data.get('ip_address', None)

        self._request.update({
             'method': data['request'].get('method'),
             'path': data['request'].get('path'),
             'version': data['request'].get('version'),
             'headers': parse_headers(data['request'].get('headers', CaseInsensitiveDict())),
             'body': data['request'].get('body', ""),
            })

        self._response.update({
             'version': data['response'].get('version'),
             'status': int(data['response'].get('status', 0)),
             'reason': data['response'].get('reason'),
             'headers': parse_headers(data['response'].get('headers', CaseInsensitiveDict())),
             'body': data['response'].get('body', ""),
            })

        if self.get_response_header('Date'):
            # NOTE: the HTTP-date should represent the best available
            # approximation of the date and time of message generation.
            # See: http://tools.ietf.org/html/rfc2616#section-14.18
            #
            # This doesn't always indicate the exact datetime the response
            # was served, i.e., cached pages might have a Date header
            # that occurrs in the past.
            req_date = self.get_response_header('Date')

            try:
                self.datetime = _datetime.strptime(req_date,
                                                   '%a, %d %b %Y %H:%M:%S %Z')
            except (ValueError, TypeError):
                logger.exception("Invalid time struct %r", req_date)
                self.datetime = None

        self.burptime = data.get('time', None)

        if self.burptime:
            # Let's take Burp's recorded time and stuff that into a 
            # datetime.time object.
            try:
                r_time, am_pm = self.burptime.split()
                hour, minute, second = map(int, r_time.split(":"))
                if hour < 12 and am_pm == 'PM':
                    hour += 12
                elif hour == 12 and am_pm == 'AM':
                    hour = 0

                self.time = _time(hour, minute, second)
            except ValueError:
                logger.exception("Invalid time struct %r", self.burptime)
                self.time = _time()

        self.url = urlparse(urljoin(self.host, self._request.get('path', '/')))
        self.parameters = parse_parameters(self)

        logger.debug("Loading cookies: %s", self.get_request_header('Cookie'))

        try:
            self.cookies.load(self.get_request_header('Cookie'))
        except Cookie.CookieError:
            logger.exception("Failed to load Cookie: %r into cookie jar",
                             self.get_request_header('Cookie'))

        if self.cookies:
            logger.debug("Added following cookies: %s",
                         ', '.join(self.cookies.keys()))

        # During parsing, we may parse an extra CRLF or two.  So to account
        # for that, we'll just grab the actual content-length from the
        # HTTP header and slice the request/response body appropriately.
        if self.get_response_header(HTTP_CONTENT_LENGTH):
            content_length = int(self.get_response_header(HTTP_CONTENT_LENGTH))
            if len(self) != content_length:
                #logger.debug("Response content-length differs by %d",
                #             len(self) - content_length)
                self._response['body'] = self._response['body'][:content_length]

        if self.get_request_header(HTTP_CONTENT_LENGTH):
            content_length = int(self.get_request_header(HTTP_CONTENT_LENGTH))
            if len(self.get_request_body()) != content_length and \
                'amf' not in self.get_request_header(HTTP_CONTENT_TYPE):
                #logger.debug("Request content-length differs by %d",
                #             len(self.get_request_body()) - content_length)
                self._request['body'] = self._request['body'][:content_length]
Esempio n. 14
0
def f_time(hours, minutes, seconds):
    """
    Defines a time value
    """
    return _time(val_to_integer(hours), val_to_integer(minutes), val_to_integer(seconds))
 def timetz(self):
     return _time(self.hour, self.minute, self.second, self.microsecond,
                  self.tzinfo)
Esempio n. 16
0
def time(ctx, hours, minutes, seconds):
    """
    Defines a time value
    """
    return _time(conversions.to_integer(hours, ctx), conversions.to_integer(minutes, ctx), conversions.to_integer(seconds, ctx))
Esempio n. 17
0
 def zero():
     return at(_time())
Esempio n. 18
0
    def __process(self, data):
        """
        Process data to fill properties of Burp object.
        """
        self.host = data.get("host", None)
        self.ip_address = data.get("ip_address", None)

        self._request.update(
            {
                "method": data["request"].get("method"),
                "path": data["request"].get("path"),
                "version": data["request"].get("version"),
                "headers": parse_headers(data["request"].get("headers", CaseInsensitiveDict())),
                "body": data["request"].get("body", ""),
            }
        )

        self._response.update(
            {
                "version": data["response"].get("version"),
                "status": int(data["response"].get("status", 0)),
                "reason": data["response"].get("reason"),
                "headers": parse_headers(data["response"].get("headers", CaseInsensitiveDict())),
                "body": data["response"].get("body", ""),
            }
        )

        if self.get_response_header("Date"):
            # NOTE: the HTTP-date should represent the best available
            # approximation of the date and time of message generation.
            # See: http://tools.ietf.org/html/rfc2616#section-14.18
            #
            # This doesn't always indicate the exact datetime the response
            # was served, i.e., cached pages might have a Date header
            # that occurrs in the past.
            req_date = self.get_response_header("Date")

            try:
                self.datetime = _datetime.strptime(req_date, "%a, %d %b %Y %H:%M:%S %Z")
            except (ValueError, TypeError):
                logger.exception("Invalid time struct %r", req_date)
                self.datetime = None

        self.burptime = data.get("time", None)

        if self.burptime:
            # Let's take Burp's recorded time and stuff that into a
            # datetime.time object.
            try:
                r_time, am_pm = self.burptime.split()
                hour, minute, second = map(int, r_time.split(":"))
                if hour < 12 and am_pm == "PM":
                    hour += 12
                elif hour == 12 and am_pm == "AM":
                    hour = 0

                self.time = _time(hour, minute, second)
            except ValueError:
                logger.exception("Invalid time struct %r", self.burptime)
                self.time = _time()

        self.url = urlparse(urljoin(self.host, self._request.get("path", "/")))
        self.parameters = parse_parameters(self)

        logger.debug("Loading cookies: %s", self.get_request_header("Cookie"))

        try:
            self.cookies.load(self.get_request_header("Cookie"))
        except Cookie.CookieError:
            logger.exception("Failed to load Cookie: %r into cookie jar", self.get_request_header("Cookie"))

        if self.cookies:
            logger.debug("Added following cookies: %s", ", ".join(self.cookies.keys()))

        # During parsing, we may parse an extra CRLF or two.  So to account
        # for that, we'll just grab the actual content-length from the
        # HTTP header and slice the request/response body appropriately.
        if self.get_response_header(HTTP_CONTENT_LENGTH):
            content_length = int(self.get_response_header(HTTP_CONTENT_LENGTH))
            if len(self) != content_length:
                # logger.debug("Response content-length differs by %d",
                #             len(self) - content_length)
                self._response["body"] = self._response["body"][:content_length]

        if self.get_request_header(HTTP_CONTENT_LENGTH):
            content_length = int(self.get_request_header(HTTP_CONTENT_LENGTH))
            if len(self.get_request_body()) != content_length and "amf" not in self.get_request_header(
                HTTP_CONTENT_TYPE
            ):
                # logger.debug("Request content-length differs by %d",
                #             len(self.get_request_body()) - content_length)
                self._request["body"] = self._request["body"][:content_length]
Esempio n. 19
0
    def __process(self, data):
        """
        Process data to fill properties of Burp object.
        """
        self.host = data.get('host', None)
        self.ip_address = data.get('ip_address', None)

        self._request.update({
             'method': data['request'].get('method'),
             'path': data['request'].get('path'),
             'version': data['request'].get('version'),
             'headers': parse_headers(data['request'].get('headers', {})),
             'body': data['request'].get('body', ""),
            })

        self._response.update({
             'version': data['response'].get('version'),
             'status': int(data['response'].get('status', 0)),
             'reason': data['response'].get('reason'),
             'headers': parse_headers(data['response'].get('headers', {})),
             'body': data['response'].get('body', ""),
            })

        if 'Date' in self.response_headers:
            # NOTE: the HTTP-date should represent the best available
            # approximation of the date and time of message generation.
            # See: http://tools.ietf.org/html/rfc2616#section-14.18
            #
            # This doesn't always indicate the exact datetime the response
            # was served, i.e., cached pages might have a Date header
            # that occurrs in the past.
            req_date = self.get_response_header('Date')

            try:
                self.datetime = _datetime.strptime(req_date,
                                                   '%a, %d %b %Y %H:%M:%S %Z')
            except (ValueError, TypeError):
                LOGGER.exception("Invalid time struct %r", req_date)
                self.datetime = None

        self.burptime = data.get('time', None)

        if self.burptime:
            # Let's take Burp's recorded time and stuff that into a 
            # datetime.time object.
            try:
                r_time, am_pm = self.burptime.split()
                hour, minute, second = map(int, r_time.split(":"))
                if hour < 12 and am_pm == 'PM':
                    hour += 12
                elif hour == 12 and am_pm == 'AM':
                    hour = 0

                self.time = _time(hour, minute, second)
            except ValueError:
                LOGGER.exception("Invalid time struct %r", self.burptime)
                self.time = _time()

        self.url = urlparse(urljoin(self.host, self._request.get('path', '/')))
        self.parameters = parse_parameters(self)

        # During parsing, we may parse an extra CRLF or two.  So to account
        # for that, we'll just grab the actual content-length from the
        # HTTP header and slice the request/response body appropriately.
        if self.get_response_header(HTTP_CONTENT_LENGTH):
            content_length = int(self.get_response_header(HTTP_CONTENT_LENGTH))
            if len(self) != content_length:
                #LOGGER.debug("Response content-length differs by %d", len(self) - content_length)
                self._response['body'] = self._response['body'][:content_length]

        if self.get_request_header(HTTP_CONTENT_LENGTH):
            content_length = int(self.get_request_header(HTTP_CONTENT_LENGTH))
            if len(self.get_request_body()) != content_length and 'amf' not in \
                self.get_request_header(HTTP_CONTENT_LENGTH):
                #LOGGER.debug("Request content-length differs by %d", len(self.get_request_body()) - content_length)
                self._request['body'] = self._request['body'][:content_length]
Esempio n. 20
0
 def timetz(self):
     return _time(self.hour, self.minute, self.second, self.microsecond,
                  self.tzinfo)
Esempio n. 21
0
def last_monday(dt):
    '''Returns datetime of UTC midnight of most recent monday
    '''
    today = dt.date()
    return datetime.combine(today - timedelta(days=today.weekday()),
                            _time(0, 0))
Esempio n. 22
0
def last_monday(dt):
    '''Returns datetime of UTC midnight of most recent monday
    '''
    today = dt.date()
    return datetime.combine(today - timedelta(days=today.weekday()), _time(0,0))
Esempio n. 23
0
def f_time(hours, minutes, seconds):
    """
    Defines a time value
    """
    return _time(val_to_integer(hours), val_to_integer(minutes),
                 val_to_integer(seconds))
    def run(self):
        # Check that ElasticSearch is alive
        self.check_index()

        # If the user specified the --REBUILD flag, recreate the index
        if self.options['rebuild']:
            self.rebuild_index()

        # Connect to the repository
        registry = MetadataRegistry()
        registry.registerReader(self.settings["metadata_format"],
                                self.settings["metadata_reader"])

        client = Client(self.settings["uri"], registry)
        identity = client.identify()

        print "Connected to repository: %s" % identity.repositoryName()

        # got to update granularity or we barf with:
        # oaipmh.error.BadArgumentError: Max granularity is YYYY-MM-DD:2003-04-10T00:00:00Z
        client.updateGranularity()

        # Initialise some variables
        batcher = Batch.Batch()
        total_records = 0
        start = time.time()

        # Now do the synchonisation

        # If the user specified an identifier, then synchronise this record
        if (self.options['identifier'] is not None):
            total_records += self.synchronise_record(
                client, batcher, self.options['identifier'])
        else:
            # Else, synchronise using the date-range provided by the user, or failing that,
            # the date-range based on the last sync

            # Get the synchronisation config record
            synchronisation_config = self.get_synchronisation_config()

            if self.options["from_date"] is not None:
                # If the user specified a from-date argument, use it
                from_date = self.options[
                    "from_date"]  # already a date (not a datetime)
            elif synchronisation_config is not None and "to_date" in synchronisation_config:
                # Else read the last synchronised to_date from the config, and add on a day
                from_date = dateutil.parser.parse(
                    synchronisation_config["to_date"]).date() + timedelta(
                        days=1)
            else:
                # Else use the default_from_date in the config
                from_date = dateutil.parser.parse(
                    self.settings['default_from_date']).date()

            if self.options["to_date"] is not None:
                to_date = self.options[
                    "to_date"]  # already a date (not a datetime)
            else:
                to_date = (date.today() - timedelta(days=1))

            # Force the from_date to use time 00:00:00
            from_date = datetime.combine(
                from_date, _time(hour=0, minute=0, second=0, microsecond=0))

            # Force the to_date to use time 23:59:59
            to_date = datetime.combine(
                to_date, _time(hour=23, minute=59, second=59, microsecond=0))

            print "Synchronising from %s - %s" % (from_date, to_date)

            while from_date < to_date:
                next_date = datetime.combine(
                    from_date.date() +
                    timedelta(days=(self.settings['delta_days'] - 1)),
                    _time(hour=23, minute=59, second=59, microsecond=0))
                number_of_records = self.synchronise_period(
                    client, batcher, from_date, next_date)
                batcher.clear()  #Store the records in elasticsearch
                self.put_synchronisation_config(from_date, next_date,
                                                number_of_records)
                from_date += timedelta(days=(self.settings['delta_days']))
                total_records += number_of_records

                # Pause so as not to get banned.
                to = 20
                print "Sleeping for %i seconds so as not to get banned." % to
                time.sleep(to)

        # Store the records in the index
        batcher.clear()

        # Print out some statistics
        time_spent = time.time() - start
        print 'Total time spent: %d seconds' % (time_spent)

        if time_spent > 0.001:  # careful as its not an integer
            print 'Total records synchronised: %i records (%d records/second)' % (
                total_records, (total_records / time_spent))
        else:
            print 'Total records synchronised: %i records' % (total_records)
        return total_records

        sys.exit()