def test_base(self):
self.assertEqual(JalaliDateTime(1367, 2, 14, 14, 0, 0, 0),
JalaliDateTime.to_jalali(datetime(1988, 5, 4, 14, 0, 0, 0)))
self.assertEqual(JalaliDateTime(1369, 7, 1, 14, 14, 1, 1111),
JalaliDateTime(datetime(1990, 9, 23, 14, 14, 1, 1111)))
self.assertEqual(JalaliDateTime(1369, 7, 1, 14, 14, 1, 9111),
JalaliDateTime(JalaliDateTime(1369, 7, 1, 14, 14, 1, 9111)))
g = JalaliDateTime.now()
self.assertEqual(g.time(), _time(g.hour, g.minute, g.second, g.microsecond))
g = g.replace(tzinfo=pytz.timezone("America/Los_Angeles"))
self.assertEqual(g.timetz(),
_time(g.hour, g.minute, g.second, g.microsecond, pytz.timezone("America/Los_Angeles")))
self.assertEqual(JalaliDateTime.fromtimestamp(578723400, pytz.utc),
JalaliDateTime(1367, 2, 14, 4, 30, 0, 0, pytz.utc))
self.assertEqual(JalaliDateTime.utcfromtimestamp(578723400), JalaliDateTime(1367, 2, 14, 4, 30, 0, 0))
try:
JalaliDateTime._check_time_fields(20, 1, 61, 1000)
except ValueError:
assert True
else:
assert False
try:
JalaliDateTime._check_time_fields("20", 1, 61, 1000)
except TypeError:
assert True
else:
assert False
def __add__(self, other):
if not isinstance(other, timedelta):
raise NotImplementedError
delta = timedelta(
self.toordinal(),
hours=self._hour,
minutes=self._minute,
seconds=self._second,
microseconds=self._microsecond,
)
delta += other
hour, rem = divmod(delta.seconds, 3600)
minute, second = divmod(rem, 60)
if 0 < delta.days <= _MAXORDINAL:
return JalaliDateTime.combine(
JalaliDate.fromordinal(delta.days),
_time(hour,
minute,
second,
delta.microseconds,
tzinfo=self._tzinfo),
)
raise OverflowError("result out of range")
def to_jalali(
cls,
year,
month=None,
day=None,
hour=None,
minute=None,
second=None,
microsecond=None,
tzinfo=None,
):
if month is None and isinstance(year, dt):
month = year.month
day = year.day
hour = year.hour
minute = year.minute
second = year.second
microsecond = year.microsecond
tzinfo = year.tzinfo
year = year.year
j_date = JalaliDate.to_jalali(year, month, day)
return cls.combine(
j_date,
_time(hour=hour,
minute=minute,
second=second,
microsecond=microsecond,
tzinfo=tzinfo),
)
def time(ctx, hours, minutes, seconds):
"""
Defines a time value
"""
return _time(conversions.to_integer(hours, ctx),
conversions.to_integer(minutes, ctx),
conversions.to_integer(seconds, ctx))
def to_gregorian(self):
g_date = super(JalaliDateTime, self).to_gregorian()
return dt.combine(g_date, _time(hour=self._hour, minute=self._minute,
second=self._second,
microsecond=self._microsecond,
tzinfo=self._tzinfo))
def test_base(self):
self.assertEqual(
JalaliDateTime(1369, 7, 1, 14, 14, 1, 9111),
JalaliDateTime(JalaliDateTime(1369, 7, 1, 14, 14, 1, 9111)),
)
g = JalaliDateTime.now()
self.assertEqual(g.time(),
_time(g.hour, g.minute, g.second, g.microsecond))
g = g.replace(tzinfo=pytz.timezone("America/Los_Angeles"))
self.assertEqual(
g.timetz(),
_time(
g.hour,
g.minute,
g.second,
g.microsecond,
pytz.timezone("America/Los_Angeles"),
),
)
self.assertEqual(
JalaliDateTime.fromtimestamp(578723400, pytz.utc),
JalaliDateTime(1367, 2, 14, 4, 30, 0, 0, pytz.utc),
)
self.assertEqual(
JalaliDateTime.utcfromtimestamp(578723400),
JalaliDateTime(1367, 2, 14, 4, 30, 0, 0),
)
with pytest.raises(TypeError):
JalaliDateTime._check_time_fields("20", 1, 61, 1000)
with pytest.raises(ValueError):
JalaliDateTime(1367, 2, 14, 25, 0, 0, 0)
with pytest.raises(ValueError):
JalaliDateTime(1367, 2, 14, 22, 61, 0, 0)
with pytest.raises(ValueError):
JalaliDateTime(1367, 2, 14, 22, 1, 722, 0)
with pytest.raises(ValueError):
JalaliDateTime(1367, 2, 14, 22, 1, 0, 1000000)
def to_gregorian(self):
g_date = super(JalaliDateTime, self).to_gregorian()
return dt.combine(
g_date,
_time(
hour=self._hour,
minute=self._minute,
second=self._second,
microsecond=self._microsecond,
tzinfo=self._tzinfo,
),
)
def __init__(_, dtime=None, **sets):
super().__init__(**sets)
if some(dtime) and issubclass(type(dtime), _time): _._time = dtime
else:
d, h, m, s, ms = _._default('d', 0), _._default(
'h', 0), _._default('m',
0), _._default('s',
0), _._default('ms', 0)
if not any([d, h, m, s, ms]):
now = datetime.now()
_._time = now - datetime(now.year, now.month, now.day)
else:
_._time = _time(days=d,
hours=h,
minutes=m,
seconds=s,
milliseconds=ms)
_.clear('d', 'h', 'm', 's', 'ms')
def to_jalali(cls, year, month=None, day=None, hour=None, minute=None,
second=None, microsecond=None, tzinfo=None):
if month is None and isinstance(year, dt):
month = year.month
day = year.day
hour = year.hour
minute = year.minute
second = year.second
microsecond = year.microsecond
tzinfo = year.tzinfo
year = year.year
j_date = JalaliDate.to_jalali(year, month, day)
return cls.combine(j_date,
_time(hour=hour, minute=minute, second=second,
microsecond=microsecond,
tzinfo=tzinfo))
def __add__(self, other):
if not isinstance(other, timedelta):
return NotImplemented
delta = timedelta(self.toordinal(),
hours=self._hour,
minutes=self._minute,
seconds=self._second,
microseconds=self._microsecond)
delta += other
hour, rem = divmod(delta.seconds, 3600)
minute, second = divmod(rem, 60)
if 0 < delta.days <= _MAXORDINAL:
return JalaliDateTime.combine(JalaliDate.fromordinal(delta.days),
_time(hour, minute, second,
delta.microseconds,
tzinfo=self._tzinfo))
raise OverflowError("result out of range")
def convert_to_datetime(src):
if src is None:
return
elif isinstance(src, datetime):
dst = src
elif isinstance(src, date):
dst = datetime.combine(src, _time())
elif isinstance(src, basestring):
m = _DATE_REGEX.match(src)
if not m:
raise ValueError('Invalid date string')
values = [(k, int(v or 0)) for k, v in m.groupdict().items()]
values = dict(values)
dst = datetime(**values)
else:
raise TypeError('Unsupported type for %s: %s' % (arg_name, src.__class__.__name__))
return dst
def run(self):
# Check that ElasticSearch is alive
self.check_index()
# If the user specified the --REBUILD flag, recreate the index
if self.options['rebuild']:
self.rebuild_index()
# Connect to the repository
registry = MetadataRegistry()
registry.registerReader(self.settings["metadata_format"], self.settings["metadata_reader"])
client = Client(self.settings["uri"], registry)
identity = client.identify()
print "Connected to repository: %s" % identity.repositoryName()
# got to update granularity or we barf with:
# oaipmh.error.BadArgumentError: Max granularity is YYYY-MM-DD:2003-04-10T00:00:00Z
client.updateGranularity()
# Initialise some variables
batcher = Batch.Batch()
total_records = 0
start = time.time()
# Now do the synchonisation
# If the user specified an identifier, then synchronise this record
if (self.options['identifier'] is not None):
total_records += self.synchronise_record(client, batcher, self.options['identifier'])
else:
# Else, synchronise using the date-range provided by the user, or failing that,
# the date-range based on the last sync
# Get the synchronisation config record
synchronisation_config = self.get_synchronisation_config()
if self.options["from_date"] is not None:
# If the user specified a from-date argument, use it
from_date = self.options["from_date"] # already a date (not a datetime)
elif synchronisation_config is not None and "to_date" in synchronisation_config:
# Else read the last synchronised to_date from the config, and add on a day
from_date = dateutil.parser.parse(synchronisation_config["to_date"]).date() + timedelta(days=1)
else:
# Else use the default_from_date in the config
from_date = dateutil.parser.parse(self.settings['default_from_date']).date()
if self.options["to_date"] is not None:
to_date = self.options["to_date"] # already a date (not a datetime)
else:
to_date = (date.today() - timedelta(days=1))
# Force the from_date to use time 00:00:00
from_date = datetime.combine(from_date, _time(hour=0, minute=0, second=0, microsecond=0))
# Force the to_date to use time 23:59:59
to_date = datetime.combine(to_date, _time(hour=23, minute=59, second=59, microsecond=0))
print "Synchronising from %s - %s" % (from_date, to_date)
while from_date < to_date:
next_date = datetime.combine(from_date.date() + timedelta(days=(self.settings['delta_days'] - 1)), _time(hour=23, minute=59, second=59, microsecond=0))
number_of_records = self.synchronise_period(client, batcher, from_date, next_date)
batcher.clear() #Store the records in elasticsearch
self.put_synchronisation_config(from_date, next_date, number_of_records)
from_date += timedelta(days=(self.settings['delta_days']))
total_records += number_of_records
# Pause so as not to get banned.
to = 20
print "Sleeping for %i seconds so as not to get banned." % to
time.sleep(to)
# Store the records in the index
batcher.clear()
# Print out some statistics
time_spent = time.time() - start
print 'Total time spent: %d seconds' % (time_spent)
if time_spent > 0.001: # careful as its not an integer
print 'Total records synchronised: %i records (%d records/second)' % (total_records, (total_records/time_spent))
else:
print 'Total records synchronised: %i records' % (total_records)
return total_records
sys.exit()
def __process(self, data):
"""
Process data to fill properties of Burp object.
"""
self.host = data.get('host', None)
self.ip_address = data.get('ip_address', None)
self._request.update({
'method': data['request'].get('method'),
'path': data['request'].get('path'),
'version': data['request'].get('version'),
'headers': parse_headers(data['request'].get('headers', CaseInsensitiveDict())),
'body': data['request'].get('body', ""),
})
self._response.update({
'version': data['response'].get('version'),
'status': int(data['response'].get('status', 0)),
'reason': data['response'].get('reason'),
'headers': parse_headers(data['response'].get('headers', CaseInsensitiveDict())),
'body': data['response'].get('body', ""),
})
if self.get_response_header('Date'):
# NOTE: the HTTP-date should represent the best available
# approximation of the date and time of message generation.
# See: http://tools.ietf.org/html/rfc2616#section-14.18
#
# This doesn't always indicate the exact datetime the response
# was served, i.e., cached pages might have a Date header
# that occurrs in the past.
req_date = self.get_response_header('Date')
try:
self.datetime = _datetime.strptime(req_date,
'%a, %d %b %Y %H:%M:%S %Z')
except (ValueError, TypeError):
logger.exception("Invalid time struct %r", req_date)
self.datetime = None
self.burptime = data.get('time', None)
if self.burptime:
# Let's take Burp's recorded time and stuff that into a
# datetime.time object.
try:
r_time, am_pm = self.burptime.split()
hour, minute, second = map(int, r_time.split(":"))
if hour < 12 and am_pm == 'PM':
hour += 12
elif hour == 12 and am_pm == 'AM':
hour = 0
self.time = _time(hour, minute, second)
except ValueError:
logger.exception("Invalid time struct %r", self.burptime)
self.time = _time()
self.url = urlparse(urljoin(self.host, self._request.get('path', '/')))
self.parameters = parse_parameters(self)
logger.debug("Loading cookies: %s", self.get_request_header('Cookie'))
try:
self.cookies.load(self.get_request_header('Cookie'))
except Cookie.CookieError:
logger.exception("Failed to load Cookie: %r into cookie jar",
self.get_request_header('Cookie'))
if self.cookies:
logger.debug("Added following cookies: %s",
', '.join(self.cookies.keys()))
# During parsing, we may parse an extra CRLF or two. So to account
# for that, we'll just grab the actual content-length from the
# HTTP header and slice the request/response body appropriately.
if self.get_response_header(HTTP_CONTENT_LENGTH):
content_length = int(self.get_response_header(HTTP_CONTENT_LENGTH))
if len(self) != content_length:
#logger.debug("Response content-length differs by %d",
# len(self) - content_length)
self._response['body'] = self._response['body'][:content_length]
if self.get_request_header(HTTP_CONTENT_LENGTH):
content_length = int(self.get_request_header(HTTP_CONTENT_LENGTH))
if len(self.get_request_body()) != content_length and \
'amf' not in self.get_request_header(HTTP_CONTENT_TYPE):
#logger.debug("Request content-length differs by %d",
# len(self.get_request_body()) - content_length)
self._request['body'] = self._request['body'][:content_length]
def f_time(hours, minutes, seconds):
"""
Defines a time value
"""
return _time(val_to_integer(hours), val_to_integer(minutes), val_to_integer(seconds))
def timetz(self):
return _time(self.hour, self.minute, self.second, self.microsecond,
self.tzinfo)
def time(ctx, hours, minutes, seconds):
"""
Defines a time value
"""
return _time(conversions.to_integer(hours, ctx), conversions.to_integer(minutes, ctx), conversions.to_integer(seconds, ctx))
def zero():
return at(_time())
def __process(self, data):
"""
Process data to fill properties of Burp object.
"""
self.host = data.get("host", None)
self.ip_address = data.get("ip_address", None)
self._request.update(
{
"method": data["request"].get("method"),
"path": data["request"].get("path"),
"version": data["request"].get("version"),
"headers": parse_headers(data["request"].get("headers", CaseInsensitiveDict())),
"body": data["request"].get("body", ""),
}
)
self._response.update(
{
"version": data["response"].get("version"),
"status": int(data["response"].get("status", 0)),
"reason": data["response"].get("reason"),
"headers": parse_headers(data["response"].get("headers", CaseInsensitiveDict())),
"body": data["response"].get("body", ""),
}
)
if self.get_response_header("Date"):
# NOTE: the HTTP-date should represent the best available
# approximation of the date and time of message generation.
# See: http://tools.ietf.org/html/rfc2616#section-14.18
#
# This doesn't always indicate the exact datetime the response
# was served, i.e., cached pages might have a Date header
# that occurrs in the past.
req_date = self.get_response_header("Date")
try:
self.datetime = _datetime.strptime(req_date, "%a, %d %b %Y %H:%M:%S %Z")
except (ValueError, TypeError):
logger.exception("Invalid time struct %r", req_date)
self.datetime = None
self.burptime = data.get("time", None)
if self.burptime:
# Let's take Burp's recorded time and stuff that into a
# datetime.time object.
try:
r_time, am_pm = self.burptime.split()
hour, minute, second = map(int, r_time.split(":"))
if hour < 12 and am_pm == "PM":
hour += 12
elif hour == 12 and am_pm == "AM":
hour = 0
self.time = _time(hour, minute, second)
except ValueError:
logger.exception("Invalid time struct %r", self.burptime)
self.time = _time()
self.url = urlparse(urljoin(self.host, self._request.get("path", "/")))
self.parameters = parse_parameters(self)
logger.debug("Loading cookies: %s", self.get_request_header("Cookie"))
try:
self.cookies.load(self.get_request_header("Cookie"))
except Cookie.CookieError:
logger.exception("Failed to load Cookie: %r into cookie jar", self.get_request_header("Cookie"))
if self.cookies:
logger.debug("Added following cookies: %s", ", ".join(self.cookies.keys()))
# During parsing, we may parse an extra CRLF or two. So to account
# for that, we'll just grab the actual content-length from the
# HTTP header and slice the request/response body appropriately.
if self.get_response_header(HTTP_CONTENT_LENGTH):
content_length = int(self.get_response_header(HTTP_CONTENT_LENGTH))
if len(self) != content_length:
# logger.debug("Response content-length differs by %d",
# len(self) - content_length)
self._response["body"] = self._response["body"][:content_length]
if self.get_request_header(HTTP_CONTENT_LENGTH):
content_length = int(self.get_request_header(HTTP_CONTENT_LENGTH))
if len(self.get_request_body()) != content_length and "amf" not in self.get_request_header(
HTTP_CONTENT_TYPE
):
# logger.debug("Request content-length differs by %d",
# len(self.get_request_body()) - content_length)
self._request["body"] = self._request["body"][:content_length]
def __process(self, data):
"""
Process data to fill properties of Burp object.
"""
self.host = data.get('host', None)
self.ip_address = data.get('ip_address', None)
self._request.update({
'method': data['request'].get('method'),
'path': data['request'].get('path'),
'version': data['request'].get('version'),
'headers': parse_headers(data['request'].get('headers', {})),
'body': data['request'].get('body', ""),
})
self._response.update({
'version': data['response'].get('version'),
'status': int(data['response'].get('status', 0)),
'reason': data['response'].get('reason'),
'headers': parse_headers(data['response'].get('headers', {})),
'body': data['response'].get('body', ""),
})
if 'Date' in self.response_headers:
# NOTE: the HTTP-date should represent the best available
# approximation of the date and time of message generation.
# See: http://tools.ietf.org/html/rfc2616#section-14.18
#
# This doesn't always indicate the exact datetime the response
# was served, i.e., cached pages might have a Date header
# that occurrs in the past.
req_date = self.get_response_header('Date')
try:
self.datetime = _datetime.strptime(req_date,
'%a, %d %b %Y %H:%M:%S %Z')
except (ValueError, TypeError):
LOGGER.exception("Invalid time struct %r", req_date)
self.datetime = None
self.burptime = data.get('time', None)
if self.burptime:
# Let's take Burp's recorded time and stuff that into a
# datetime.time object.
try:
r_time, am_pm = self.burptime.split()
hour, minute, second = map(int, r_time.split(":"))
if hour < 12 and am_pm == 'PM':
hour += 12
elif hour == 12 and am_pm == 'AM':
hour = 0
self.time = _time(hour, minute, second)
except ValueError:
LOGGER.exception("Invalid time struct %r", self.burptime)
self.time = _time()
self.url = urlparse(urljoin(self.host, self._request.get('path', '/')))
self.parameters = parse_parameters(self)
# During parsing, we may parse an extra CRLF or two. So to account
# for that, we'll just grab the actual content-length from the
# HTTP header and slice the request/response body appropriately.
if self.get_response_header(HTTP_CONTENT_LENGTH):
content_length = int(self.get_response_header(HTTP_CONTENT_LENGTH))
if len(self) != content_length:
#LOGGER.debug("Response content-length differs by %d", len(self) - content_length)
self._response['body'] = self._response['body'][:content_length]
if self.get_request_header(HTTP_CONTENT_LENGTH):
content_length = int(self.get_request_header(HTTP_CONTENT_LENGTH))
if len(self.get_request_body()) != content_length and 'amf' not in \
self.get_request_header(HTTP_CONTENT_LENGTH):
#LOGGER.debug("Request content-length differs by %d", len(self.get_request_body()) - content_length)
self._request['body'] = self._request['body'][:content_length]
def timetz(self):
return _time(self.hour, self.minute, self.second, self.microsecond,
self.tzinfo)
def last_monday(dt):
'''Returns datetime of UTC midnight of most recent monday
'''
today = dt.date()
return datetime.combine(today - timedelta(days=today.weekday()),
_time(0, 0))
def last_monday(dt):
'''Returns datetime of UTC midnight of most recent monday
'''
today = dt.date()
return datetime.combine(today - timedelta(days=today.weekday()), _time(0,0))
def f_time(hours, minutes, seconds):
"""
Defines a time value
"""
return _time(val_to_integer(hours), val_to_integer(minutes),
val_to_integer(seconds))
def run(self):
# Check that ElasticSearch is alive
self.check_index()
# If the user specified the --REBUILD flag, recreate the index
if self.options['rebuild']:
self.rebuild_index()
# Connect to the repository
registry = MetadataRegistry()
registry.registerReader(self.settings["metadata_format"],
self.settings["metadata_reader"])
client = Client(self.settings["uri"], registry)
identity = client.identify()
print "Connected to repository: %s" % identity.repositoryName()
# got to update granularity or we barf with:
# oaipmh.error.BadArgumentError: Max granularity is YYYY-MM-DD:2003-04-10T00:00:00Z
client.updateGranularity()
# Initialise some variables
batcher = Batch.Batch()
total_records = 0
start = time.time()
# Now do the synchonisation
# If the user specified an identifier, then synchronise this record
if (self.options['identifier'] is not None):
total_records += self.synchronise_record(
client, batcher, self.options['identifier'])
else:
# Else, synchronise using the date-range provided by the user, or failing that,
# the date-range based on the last sync
# Get the synchronisation config record
synchronisation_config = self.get_synchronisation_config()
if self.options["from_date"] is not None:
# If the user specified a from-date argument, use it
from_date = self.options[
"from_date"] # already a date (not a datetime)
elif synchronisation_config is not None and "to_date" in synchronisation_config:
# Else read the last synchronised to_date from the config, and add on a day
from_date = dateutil.parser.parse(
synchronisation_config["to_date"]).date() + timedelta(
days=1)
else:
# Else use the default_from_date in the config
from_date = dateutil.parser.parse(
self.settings['default_from_date']).date()
if self.options["to_date"] is not None:
to_date = self.options[
"to_date"] # already a date (not a datetime)
else:
to_date = (date.today() - timedelta(days=1))
# Force the from_date to use time 00:00:00
from_date = datetime.combine(
from_date, _time(hour=0, minute=0, second=0, microsecond=0))
# Force the to_date to use time 23:59:59
to_date = datetime.combine(
to_date, _time(hour=23, minute=59, second=59, microsecond=0))
print "Synchronising from %s - %s" % (from_date, to_date)
while from_date < to_date:
next_date = datetime.combine(
from_date.date() +
timedelta(days=(self.settings['delta_days'] - 1)),
_time(hour=23, minute=59, second=59, microsecond=0))
number_of_records = self.synchronise_period(
client, batcher, from_date, next_date)
batcher.clear() #Store the records in elasticsearch
self.put_synchronisation_config(from_date, next_date,
number_of_records)
from_date += timedelta(days=(self.settings['delta_days']))
total_records += number_of_records
# Pause so as not to get banned.
to = 20
print "Sleeping for %i seconds so as not to get banned." % to
time.sleep(to)
# Store the records in the index
batcher.clear()
# Print out some statistics
time_spent = time.time() - start
print 'Total time spent: %d seconds' % (time_spent)
if time_spent > 0.001: # careful as its not an integer
print 'Total records synchronised: %i records (%d records/second)' % (
total_records, (total_records / time_spent))
else:
print 'Total records synchronised: %i records' % (total_records)
return total_records
sys.exit()
