def check_type(param, datatype):
"""
Make sure that param is of type datatype and return it.
If param is None, return it.
If param is an instance of datatype, return it.
If param is not an instance of datatype and is not None, cast it as
datatype and return it.
"""
if param is None:
return param
if datatype == "str" and not isinstance(param, basestring):
try:
param = str(param)
except ValueError:
param = str()
elif datatype == "int" and not isinstance(param, int):
try:
param = int(param)
except ValueError:
param = int()
elif datatype == "bool" and not isinstance(param, bool):
param = str(param).lower() in ("true", "t", "1", "y", "yes")
elif datatype == "datetime" and not isinstance(param, datetime):
try:
param = dtutil.string_to_datetime(param)
except ValueError:
param = None
elif datatype == "date" and not isinstance(param, date):
try:
param = dtutil.string_to_datetime(param).date()
except ValueError:
param = None
elif datatype == "timedelta" and not isinstance(param, timedelta):
try:
param = dtutil.strHoursToTimeDelta(param)
except ValueError:
param = None
elif datatype == "json" and isinstance(param, basestring):
try:
param = json.loads(param)
except ValueError:
param = None
return param
def test_get_index_for_crash_dynamic_name(self):
"""Test a dynamic (date-based) index name.
"""
# The crashstorage class looks for '%' in the index name; if that
# symbol is present, it will attempt to generate a new date-based
# index name. Since the test base config doesn't use this pattern,
# we need to specify it now.
modified_config = self.get_tuned_config(
ESCrashStorage,
{'resource.elasticsearch.elasticsearch_index':
'socorro_integration_test_reports%Y%m%d'}
)
es_storage = ESCrashStorage(config=modified_config)
# The date is used to generate the name of the index; it must be a
# datetime object.
date = string_to_datetime(
a_processed_crash['client_crash_date']
)
index = es_storage.get_index_for_crash(date)
# The base index name is obtained from the test base class and the
# date is appended to it according to pattern specified above.
ok_(type(index) is str)
eq_(index, 'socorro_integration_test_reports20120408')
def convert_to_type(value, data_type):
if data_type == 'str' and not isinstance(value, basestring):
value = str(value)
# yes, 'enum' is being converted to a string
elif data_type == 'enum' and not isinstance(value, basestring):
value = str(value)
elif data_type == 'int' and not isinstance(value, int):
value = int(value)
elif data_type == 'bool' and not isinstance(value, bool):
value = str(value).lower() in ('true', 't', '1', 'y', 'yes')
elif data_type == 'datetime' and not isinstance(value, datetime.datetime):
value = datetimeutil.string_to_datetime(value)
elif data_type == 'date' and not isinstance(value, datetime.date):
value = datetimeutil.string_to_datetime(value).date()
elif data_type == 'json' and isinstance(value, basestring):
value = json.loads(value)
return value
def convert_to_type(value, data_type):
if data_type == 'str' and not isinstance(value, basestring):
value = str(value)
# yes, 'enum' is being converted to a string
elif data_type == 'enum' and not isinstance(value, basestring):
value = str(value)
elif data_type == 'int' and not isinstance(value, int):
value = int(value)
elif data_type == 'bool' and not isinstance(value, bool):
value = str(value).lower() in ('true', 't', '1', 'y', 'yes')
elif data_type == 'datetime' and not isinstance(value, datetime.datetime):
value = datetimeutil.string_to_datetime(value)
elif data_type == 'date' and not isinstance(value, datetime.date):
value = datetimeutil.string_to_datetime(value).date()
elif data_type == 'json' and isinstance(value, basestring):
value = json.loads(value)
return value
def test_string_to_datetime():
"""
Test datetimeutil.string_to_datetime()
"""
# Empty date
date = ""
try:
res = datetimeutil.string_to_datetime(date)
raise AssertionError("expect this to raise ValueError")
except ValueError:
pass
# already a date
date = datetime.datetime.utcnow()
res = datetimeutil.string_to_datetime(date)
eq_(res, date.replace(tzinfo=UTC))
eq_(res.strftime('%Z'), 'UTC')
eq_(res.strftime('%z'), '+0000')
# YY-mm-dd date
date = "2001-11-03"
res = datetimeutil.string_to_datetime(date)
eq_(res, datetime.datetime(2001, 11, 3, tzinfo=UTC))
eq_(res.strftime('%Z'), 'UTC') # timezone aware
# and naughty YY-m-d date
date = "2001-1-3"
res = datetimeutil.string_to_datetime(date)
eq_(res, datetime.datetime(2001, 1, 3, tzinfo=UTC))
eq_(res.strftime('%Z'), 'UTC') # timezone aware
# Commented out because I don't thing `YY-mm-dd+HH:ii:ss` is a
# valid date format.
## YY-mm-dd+HH:ii:ss date
#date = "2001-11-30+12:34:56"
#try:
# res = datetimeutil.string_to_datetime(date)
#except ValueError:
# res = None
#expected = datetime(2001, 11, 30, 12, 34, 56)
#assert res == expected, "Date is %s, %s expected." % (date, expected)
# YY-mm-dd HH:ii:ss.S date
date = "2001-11-30 12:34:56.123456"
res = datetimeutil.string_to_datetime(date)
eq_(res, datetime.datetime(2001, 11, 30, 12, 34, 56, 123456, tzinfo=UTC))
# Separated date
date = ["2001-11-30", "12:34:56"]
res = datetimeutil.string_to_datetime(date)
eq_(res, datetime.datetime(2001, 11, 30, 12, 34, 56, tzinfo=UTC))
# Invalid date
date = "2001-11-32"
try:
res = datetimeutil.string_to_datetime(date)
raise AssertionError("should have raise a ValueError")
except ValueError:
pass
def clean(self, value):
if any(itertools.imap(value.startswith, ('>=', '<='))):
op = value[:2]
value = value[2:]
elif any(itertools.imap(value.startswith, ('=', '>', '<'))):
op = value[:1]
value = value[1:]
else:
op = '='
return (op, string_to_datetime(value).date())
def clean(self, value):
if any(itertools.imap(value.startswith, ('>=', '<='))):
op = value[:2]
value = value[2:]
elif any(itertools.imap(value.startswith, ('=', '>', '<'))):
op = value[:1]
value = value[1:]
else:
op = '='
return (op, string_to_datetime(value).date())
def test_string_datetime_with_timezone():
date = "2001-11-30T12:34:56Z"
res = datetimeutil.string_to_datetime(date)
eq_(res, datetime.datetime(2001, 11, 30, 12, 34, 56, tzinfo=UTC))
eq_(res.strftime('%H'), '12')
# because it's a timezone aware datetime
ok_(res.tzname())
eq_(res.strftime('%Z'), 'UTC')
eq_(res.strftime('%z'), '+0000')
# plus 3 hours east of Zulu means minus 3 hours on UTC
date = "2001-11-30T12:10:56+03:00"
res = datetimeutil.string_to_datetime(date)
expected = datetime.datetime(2001, 11, 30, 12 - 3, 10, 56, tzinfo=UTC)
eq_(res, expected)
# similar example
date = "2001-11-30T12:10:56-01:30"
res = datetimeutil.string_to_datetime(date)
eq_(res, datetime.datetime(2001, 11, 30, 12 + 1, 10 + 30, 56, tzinfo=UTC))
# YY-mm-dd+HH:ii:ss.S date
date = "2001-11-30 12:34:56.123456Z"
res = datetimeutil.string_to_datetime(date)
eq_(res, datetime.datetime(2001, 11, 30, 12, 34, 56, 123456, tzinfo=UTC))
docstring = """
* 2012-01-10T12:13:14
* 2012-01-10T12:13:14.98765
* 2012-01-10T12:13:14.98765+03:00
* 2012-01-10T12:13:14.98765Z
* 2012-01-10 12:13:14
* 2012-01-10 12:13:14.98765
* 2012-01-10 12:13:14.98765+03:00
* 2012-01-10 12:13:14.98765Z
""".strip().splitlines()
examples = [x.replace('*', '').strip() for x in docstring]
for example in examples:
res = datetimeutil.string_to_datetime(example)
ok_(res.tzinfo)
ok_(isinstance(res, datetime.datetime))
def reconstitute_datetimes(processed_crash):
datetime_fields = [
'submitted_timestamp',
'date_processed',
'client_crash_date',
'started_datetime',
'startedDateTime',
'completed_datetime',
'completeddatetime',
]
for a_key in datetime_fields:
try:
processed_crash[a_key] = string_to_datetime(
processed_crash[a_key])
except KeyError:
# not there? we don't care
pass
def reconstitute_datetimes(processed_crash):
datetime_fields = [
'submitted_timestamp',
'date_processed',
'client_crash_date',
'started_datetime',
'startedDateTime',
'completed_datetime',
'completeddatetime',
]
for a_key in datetime_fields:
try:
processed_crash[a_key] = string_to_datetime(
processed_crash[a_key]
)
except KeyError:
# not there? we don't care
pass
def test_get_index_for_crash_dynamic_name(self):
"""Test a dynamic (date-based) index name.
"""
# The crashstorage class looks for '%' in the index name; if that
# symbol is present, it will attempt to generate a new date-based
# index name. Since the test base config doesn't use this pattern,
# we need to specify it now.
modified_config = self.get_tuned_config(
ESCrashStorage, {
'resource.elasticsearch.elasticsearch_index':
'socorro_integration_test_reports%Y%m%d'
})
es_storage = ESCrashStorage(config=modified_config)
# The date is used to generate the name of the index; it must be a
# datetime object.
date = string_to_datetime(a_processed_crash['client_crash_date'])
index = es_storage.get_index_for_crash(date)
# The base index name is obtained from the test base class and the
# date is appended to it according to pattern specified above.
ok_(type(index) is str)
eq_(index, 'socorro_integration_test_reports20120408')
'json_dump': 'stackwalker output',
},
'upload_file_minidump_flash2': {
'things': 'untouched',
'json_dump': 'stackwalker output',
},
'upload_file_minidump_browser': {
'things': 'untouched',
'json_dump': 'stackwalker output',
},
}
a_processed_crash_with_no_stackwalker = deepcopy(a_processed_crash)
a_processed_crash_with_no_stackwalker['date_processed'] = \
string_to_datetime('2012-04-08 10:56:41.558922')
a_processed_crash_with_no_stackwalker['client_crash_date'] = \
string_to_datetime('2012-04-08 10:52:42.0')
a_processed_crash_with_no_stackwalker['completeddatetime'] = \
string_to_datetime('2012-04-08 10:56:50.902884')
a_processed_crash_with_no_stackwalker['started_datetime'] = \
string_to_datetime('2012-04-08 10:56:50.440752')
a_processed_crash_with_no_stackwalker['startedDateTime'] = \
string_to_datetime('2012-04-08 10:56:50.440752')
del a_processed_crash_with_no_stackwalker['json_dump']
del a_processed_crash_with_no_stackwalker['upload_file_minidump_flash1'][
'json_dump'
]
del a_processed_crash_with_no_stackwalker['upload_file_minidump_flash2'][
'json_dump'
def check_type(param, datatype):
"""
Make sure that param is of type datatype and return it.
If param is None, return it.
If param is an instance of datatype, return it.
If param is not an instance of datatype and is not None, cast it as
datatype and return it.
"""
if param is None:
return param
if getattr(datatype, 'clean', None) and callable(datatype.clean):
try:
return datatype.clean(param)
except ValueError:
raise BadArgumentError(param)
elif isinstance(datatype, str):
# You've given it something like `'bool'` as a string.
# This is the legacy way of doing it.
datatype = {
'str': str,
'bool': bool,
'float': float,
'date': datetime.date,
'datetime': datetime.datetime,
'timedelta': datetime.timedelta,
'json': 'json', # exception
'int': int,
}[datatype]
if datatype is str and not isinstance(param, basestring):
try:
param = str(param)
except ValueError:
param = str()
elif datatype is int and not isinstance(param, int):
try:
param = int(param)
except ValueError:
param = int()
elif datatype is bool and not isinstance(param, bool):
param = str(param).lower() in ("true", "t", "1", "y", "yes")
elif (datatype is datetime.datetime
and not isinstance(param, datetime.datetime)):
try:
param = dtutil.string_to_datetime(param)
except ValueError:
param = None
elif datatype is datetime.date and not isinstance(param, datetime.date):
try:
param = dtutil.string_to_datetime(param).date()
except ValueError:
param = None
elif (datatype is datetime.timedelta
and not isinstance(param, datetime.timedelta)):
try:
param = dtutil.strHoursToTimeDelta(param)
except ValueError:
param = None
elif datatype == "json" and isinstance(param, basestring):
try:
param = json.loads(param)
except ValueError:
param = None
return param
'json_dump': 'stackwalker output',
},
'upload_file_minidump_flash2': {
'things': 'untouched',
'json_dump': 'stackwalker output',
},
'upload_file_minidump_browser': {
'things': 'untouched',
'json_dump': 'stackwalker output',
},
}
a_processed_crash_with_no_stackwalker = deepcopy(a_processed_crash)
a_processed_crash_with_no_stackwalker['date_processed'] = \
string_to_datetime('2012-04-08 10:56:41.558922')
a_processed_crash_with_no_stackwalker['client_crash_date'] = \
string_to_datetime('2012-04-08 10:52:42.0')
a_processed_crash_with_no_stackwalker['completeddatetime'] = \
string_to_datetime('2012-04-08 10:56:50.902884')
a_processed_crash_with_no_stackwalker['started_datetime'] = \
string_to_datetime('2012-04-08 10:56:50.440752')
a_processed_crash_with_no_stackwalker['startedDateTime'] = \
string_to_datetime('2012-04-08 10:56:50.440752')
del a_processed_crash_with_no_stackwalker['json_dump']
del a_processed_crash_with_no_stackwalker['upload_file_minidump_flash1'][
'json_dump']
del a_processed_crash_with_no_stackwalker['upload_file_minidump_flash2'][
'json_dump']
del a_processed_crash_with_no_stackwalker['upload_file_minidump_browser'][
def check_type(param, datatype):
"""
Make sure that param is of type datatype and return it.
If param is None, return it.
If param is an instance of datatype, return it.
If param is not an instance of datatype and is not None, cast it as
datatype and return it.
"""
if param is None:
return param
if getattr(datatype, 'clean', None) and callable(datatype.clean):
try:
return datatype.clean(param)
except ValueError:
raise BadArgumentError(param)
elif isinstance(datatype, str):
# You've given it something like `'bool'` as a string.
# This is the legacy way of doing it.
datatype = {
'str': str,
'bool': bool,
'float': float,
'date': datetime.date,
'datetime': datetime.datetime,
'timedelta': datetime.timedelta,
'json': 'json', # exception
'int': int,
}[datatype]
if datatype is str and not isinstance(param, basestring):
try:
param = str(param)
except ValueError:
param = str()
elif datatype is int and not isinstance(param, int):
try:
param = int(param)
except ValueError:
param = int()
elif datatype is bool and not isinstance(param, bool):
param = str(param).lower() in ("true", "t", "1", "y", "yes")
elif (
datatype is datetime.datetime and
not isinstance(param, datetime.datetime)
):
try:
param = dtutil.string_to_datetime(param)
except ValueError:
param = None
elif datatype is datetime.date and not isinstance(param, datetime.date):
try:
param = dtutil.string_to_datetime(param).date()
except ValueError:
param = None
elif (
datatype is datetime.timedelta and
not isinstance(param, datetime.timedelta)
):
try:
param = dtutil.strHoursToTimeDelta(param)
except ValueError:
param = None
elif datatype == "json" and isinstance(param, basestring):
try:
param = json.loads(param)
except ValueError:
param = None
return param
