def run_transform(data_set_config, transform, earliest, latest):
data_set = DataSet.from_group_and_type(
config.BACKDROP_READ_URL,
data_set_config['data_group'],
data_set_config['data_type'],
)
earliest = parse_time_as_utc(earliest)
latest = parse_time_as_utc(latest)
data = data_set.get(
query_parameters=get_query_parameters(transform, earliest, latest)
)
transform_function = get_transform_function(transform)
transformed_data = transform_function(data['data'],
transform,
data_set_config)
if 'additionalFields' in transform['options']:
additionalFields = transform['options']['additionalFields']
transformed_data = [
merge_additional_fields(datum, additionalFields) for datum in transformed_data]
output_data_set = get_or_get_and_create_output_dataset(
transform,
data_set_config)
output_data_set.post(transformed_data)
stats_client.incr('run_transform.success')
def run_transform(data_set_config, transform, earliest, latest):
data_set = DataSet.from_group_and_type(
config.BACKDROP_READ_URL,
data_set_config['data_group'],
data_set_config['data_type'],
)
earliest = parse_time_as_utc(earliest)
latest = parse_time_as_utc(latest)
data = data_set.get(
query_parameters=get_query_parameters(transform, earliest, latest))
transform_function = get_transform_function(transform)
transformed_data = transform_function(data['data'], transform,
data_set_config)
if 'additionalFields' in transform['options']:
additionalFields = transform['options']['additionalFields']
transformed_data = [
merge_additional_fields(datum, additionalFields)
for datum in transformed_data
]
output_data_set = get_or_get_and_create_output_dataset(
transform, data_set_config)
output_data_set.post(transformed_data)
stats_client.incr('run_transform.success')
def channel_volumetrics(rows):
rows = list(rows)
yield [
"_timestamp", "_id", "successful_agent", "successful_ivr",
"successful_web", "total_agent", "total_ivr", "total_web"
]
for column in range(1, 8):
all = rows[5][column]
if all == 0:
return
date = rows[1][column]
agent_successful = rows[2][column]
ivr_successful = rows[3][column]
web_successful = rows[4][column]
agent_total = rows[6][column]
ivr_total = rows[7][column]
web_total = rows[8][column]
yield [
date,
parse_time_as_utc(date).date().isoformat(), agent_successful,
ivr_successful, web_successful, agent_total, ivr_total, web_total
]
def test_that_date_strings_get_converted_to_utc(self):
some_datetime = parse_time_as_utc(
'2014-01-02T06:04:05+03:30')
assert_that(isinstance(some_datetime, datetime), is_(True))
assert_that(some_datetime,
is_(datetime(2014, 1, 2, 2, 34, 5, tzinfo=pytz.utc)))
def channel_volumetrics(rows):
rows = list(rows)
yield ["_timestamp", "_id",
"successful_agent", "successful_ivr", "successful_web",
"total_agent", "total_ivr", "total_web"]
for column in range(1, 8):
all = rows[5][column]
if all == 0:
return
date = rows[1][column]
agent_successful = rows[2][column]
ivr_successful = rows[3][column]
web_successful = rows[4][column]
agent_total = rows[6][column]
ivr_total = rows[7][column]
web_total = rows[8][column]
yield [date, parse_time_as_utc(date).date().isoformat(),
agent_successful, ivr_successful, web_successful,
agent_total, ivr_total, web_total]
def test_datetime_is_converted_to_utc(self):
us_eastern_time = d_tz(2012,
12,
12,
12,
tzinfo=pytz.timezone("US/Eastern"))
assert_that(parse_time_as_utc(us_eastern_time),
equal_to(d_tz(2012, 12, 12, 17)))
def parse(datum):
if '_timestamp' in datum:
try:
datum['_timestamp'] = parse_time_as_utc(
datum['_timestamp'])
except ValueError:
raise ParseError(
'_timestamp is not a valid timestamp, it must be ISO8601')
return Record(datum)
def parse(datum):
if '_timestamp' in datum:
try:
datum['_timestamp'] = parse_time_as_utc(
datum['_timestamp'])
except ValueError:
raise ParseError(
'_timestamp is not a valid timestamp, it must be ISO8601')
return Record(datum)
def parse_timestamps(record):
"""Parses a timestamp in a record
>>> parse_timestamps({'_timestamp': '2012-12-12T00:00:00'})
({'_timestamp': datetime.datetime(2012, 12, 12, 0, 0, tzinfo=<UTC>)}, None)
>>> parse_timestamps({'_start_at': '2012-12-12T00:00:00'})
({'_start_at': datetime.datetime(2012, 12, 12, 0, 0, tzinfo=<UTC>)}, None)
>>> parse_timestamps({'_end_at': '2012-12-12T00:00:00'})
({'_end_at': datetime.datetime(2012, 12, 12, 0, 0, tzinfo=<UTC>)}, None)
>>> parse_timestamps({})
({}, None)
>>> record, error = parse_timestamps({'_timestamp': 'invalid'})
>>> record
{'_timestamp': 'invalid'}
>>> error
'_timestamp is not a valid timestamp, it must be ISO8601'
>>> record, error = parse_timestamps({'_start_at': 'invalid'})
>>> record
{'_start_at': 'invalid'}
>>> error
'_start_at is not a valid timestamp, it must be ISO8601'
"""
error = None
if '_timestamp' in record:
try:
record['_timestamp'] = parse_time_as_utc(record['_timestamp'])
except (TypeError, ValueError):
error = '_timestamp is not a valid timestamp, it must be ISO8601'
if '_start_at' in record:
try:
record['_start_at'] = parse_time_as_utc(record['_start_at'])
except (TypeError, ValueError):
error = '_start_at is not a valid timestamp, it must be ISO8601'
if '_end_at' in record:
try:
record['_end_at'] = parse_time_as_utc(record['_end_at'])
except (TypeError, ValueError):
error = '_end_at is not a valid timestamp, it must be ISO8601'
return (record, error)
def service_volumetrics(rows):
rows = list(rows)
yield ["_timestamp", "_id", "timeSpan", "successful_tax_disc",
"successful_sorn"]
timestamp = rows[2][1]
taxDiskApplications = rows[24][2]
sornApplications = rows[25][2]
yield [timestamp, parse_time_as_utc(timestamp).date().isoformat(), "day",
taxDiskApplications, sornApplications]
def service_volumetrics(rows):
rows = list(rows)
yield [
"_timestamp", "_id", "timeSpan", "successful_tax_disc",
"successful_sorn"
]
timestamp = rows[2][1]
taxDiskApplications = rows[24][2]
sornApplications = rows[25][2]
yield [
timestamp,
parse_time_as_utc(timestamp).date().isoformat(), "day",
taxDiskApplications, sornApplications
]
def test_datetime_with_no_timezone_is_given_utc(self):
assert_that(parse_time_as_utc(d(2012, 12, 12, 12)),
equal_to(d_tz(2012, 12, 12, 12)))
def test_time_string_is_converted_to_utc(self):
assert_that(parse_time_as_utc("2012-12-12T12:12:12+01:00"),
equal_to(d_tz(2012, 12, 12, 11, 12, 12)))
def failure(service_type, reason, failures, description, timestamp):
date = parse_time_as_utc(timestamp).date()
id = "%s.%s.%s" % (date.isoformat(), service_type, reason)
return [timestamp, id, service_type, reason, failures, description]
def parse_timestamps(record):
"""Parses a timestamp in a record
>>> parse_timestamps({'_timestamp': '2012-12-12T00:00:00'})
({'_timestamp': datetime.datetime(2012, 12, 12, 0, 0, tzinfo=<UTC>)}, None)
>>> parse_timestamps({'_start_at': '2012-12-12T00:00:00'})
({'_start_at': datetime.datetime(2012, 12, 12, 0, 0, tzinfo=<UTC>)}, None)
>>> parse_timestamps({'_end_at': '2012-12-12T00:00:00'})
({'_end_at': datetime.datetime(2012, 12, 12, 0, 0, tzinfo=<UTC>)}, None)
>>> parse_timestamps({})
({}, None)
>>> record, error = parse_timestamps({'_timestamp': 'invalid'})
>>> record
{'_timestamp': 'invalid'}
>>> error
'the _timestamp must be a date in the format yyyy-MM-ddT00:00:00Z'
>>> record, error = parse_timestamps({'_timestamp': '31-11-12T00:00:00'})
>>> record
{'_timestamp': '31-11-12T00:00:00'}
>>> error
'31-11-12T00:00:00 in the _timestamp field is not a valid date'
>>> record, error = parse_timestamps({'_start_at': 'invalid'})
>>> record
{'_start_at': 'invalid'}
>>> error
'_start_at must be a date in the format yyyy-MM-ddT00:00:00Z'
>>> record, error = parse_timestamps({'_start_at': '31-11-12T00:00:00'})
>>> record
{'_start_at': '31-11-12T00:00:00'}
>>> error
'31-11-12T00:00:00 in the _start_at field is not a valid date'
>>> record, error = parse_timestamps({'_end_at': 'invalid'})
>>> record
{'_end_at': 'invalid'}
>>> error
'_end_at must be a date in the format yyyy-MM-ddT00:00:00Z'
>>> record, error = parse_timestamps({'_end_at': '31-11-12T00:00:00'})
>>> record
{'_end_at': '31-11-12T00:00:00'}
>>> error
'31-11-12T00:00:00 in the _end_at field is not a valid date'
"""
error = None
if '_timestamp' in record:
try:
record['_timestamp'] = parse_time_as_utc(record['_timestamp'])
except TypeError:
error = 'the _timestamp must be a date in the format ' \
'yyyy-MM-ddT00:00:00Z'
except ValueError:
error = '{} in the _timestamp field is not a valid date'.format(
record['_timestamp'])
if '_start_at' in record:
try:
record['_start_at'] = parse_time_as_utc(record['_start_at'])
except TypeError:
error = '_start_at must be a date in the format ' \
'yyyy-MM-ddT00:00:00Z'
except ValueError:
error = '{} in the _start_at field is not a valid date'.format(
record['_start_at'])
if '_end_at' in record:
try:
record['_end_at'] = parse_time_as_utc(record['_end_at'])
except TypeError:
error = '_end_at must be a date in the format yyyy-MM-ddT00:00:00Z'
except ValueError:
error = '{} in the _end_at field is not a valid date'.format(
record['_end_at'])
return (record, error)
def date_or_none(string):
try:
return parse_time_as_utc(string)
except (TypeError, ValueError):
return None
def ceg_date(rows, column):
try:
return parse_time_as_utc(rows[3][column])
except IndexError:
return None
def ceg_date(rows, column):
try:
return parse_time_as_utc(rows[3][column])
except IndexError:
return None
def date_or_none(string):
try:
return parse_time_as_utc(string)
except (TypeError, ValueError):
return None
def failure(service_type, reason, failures, description, timestamp):
date = parse_time_as_utc(timestamp).date()
id = "%s.%s.%s" % (date.isoformat(), service_type, reason)
return [timestamp, id, service_type, reason, failures, description]
def test_that_date_strings_get_converted_to_utc(self):
some_datetime = parse_time_as_utc('2014-01-02T06:04:05+03:30')
assert_that(isinstance(some_datetime, datetime), is_(True))
assert_that(some_datetime,
is_(datetime(2014, 1, 2, 2, 34, 5, tzinfo=pytz.utc)))
def test_datetime_with_no_timezone_is_given_utc(self):
assert_that(parse_time_as_utc(d(2012, 12, 12, 12)),
equal_to(d_tz(2012, 12, 12, 12)))
def test_datetime_is_converted_to_utc(self):
us_eastern_time = d_tz(2012, 12, 12, 12,
tzinfo=pytz.timezone("US/Eastern"))
assert_that(parse_time_as_utc(us_eastern_time),
equal_to(d_tz(2012, 12, 12, 17)))
def test_time_string_is_converted_to_utc(self):
assert_that(parse_time_as_utc("2012-12-12T12:12:12+01:00"),
equal_to(d_tz(2012, 12, 12, 11, 12, 12)))
def test_valid_time_string_is_parsed(self):
assert_that(parse_time_as_utc("2012-12-12T12:12:12+00:00"),
equal_to(d_tz(2012, 12, 12, 12, 12, 12)))
def test_valid_time_string_is_parsed(self):
assert_that(parse_time_as_utc("2012-12-12T12:12:12+00:00"),
equal_to(d_tz(2012, 12, 12, 12, 12, 12)))
def parse_timestamps(record):
"""Parses a timestamp in a record
>>> parse_timestamps({'_timestamp': '2012-12-12T00:00:00'})
({'_timestamp': datetime.datetime(2012, 12, 12, 0, 0, tzinfo=<UTC>)}, None)
>>> parse_timestamps({'_start_at': '2012-12-12T00:00:00'})
({'_start_at': datetime.datetime(2012, 12, 12, 0, 0, tzinfo=<UTC>)}, None)
>>> parse_timestamps({'_end_at': '2012-12-12T00:00:00'})
({'_end_at': datetime.datetime(2012, 12, 12, 0, 0, tzinfo=<UTC>)}, None)
>>> parse_timestamps({})
({}, None)
>>> record, error = parse_timestamps({'_timestamp': 'invalid'})
>>> record
{'_timestamp': 'invalid'}
>>> error
'the _timestamp must be a date in the format yyyy-MM-ddT00:00:00Z'
>>> record, error = parse_timestamps({'_timestamp': '31-11-12T00:00:00'})
>>> record
{'_timestamp': '31-11-12T00:00:00'}
>>> error
'31-11-12T00:00:00 in the _timestamp field is not a valid date'
>>> record, error = parse_timestamps({'_start_at': 'invalid'})
>>> record
{'_start_at': 'invalid'}
>>> error
'_start_at must be a date in the format yyyy-MM-ddT00:00:00Z'
>>> record, error = parse_timestamps({'_start_at': '31-11-12T00:00:00'})
>>> record
{'_start_at': '31-11-12T00:00:00'}
>>> error
'31-11-12T00:00:00 in the _start_at field is not a valid date'
>>> record, error = parse_timestamps({'_end_at': 'invalid'})
>>> record
{'_end_at': 'invalid'}
>>> error
'_end_at must be a date in the format yyyy-MM-ddT00:00:00Z'
>>> record, error = parse_timestamps({'_end_at': '31-11-12T00:00:00'})
>>> record
{'_end_at': '31-11-12T00:00:00'}
>>> error
'31-11-12T00:00:00 in the _end_at field is not a valid date'
"""
error = None
if '_timestamp' in record:
try:
record['_timestamp'] = parse_time_as_utc(record['_timestamp'])
except TypeError:
error = 'the _timestamp must be a date in the format ' \
'yyyy-MM-ddT00:00:00Z'
except ValueError:
error = '{} in the _timestamp field is not a valid date'.format(
record['_timestamp'])
if '_start_at' in record:
try:
record['_start_at'] = parse_time_as_utc(record['_start_at'])
except TypeError:
error = '_start_at must be a date in the format ' \
'yyyy-MM-ddT00:00:00Z'
except ValueError:
error = '{} in the _start_at field is not a valid date'.format(
record['_start_at'])
if '_end_at' in record:
try:
record['_end_at'] = parse_time_as_utc(record['_end_at'])
except TypeError:
error = '_end_at must be a date in the format yyyy-MM-ddT00:00:00Z'
except ValueError:
error = '{} in the _end_at field is not a valid date'.format(
record['_end_at'])
return (record, error)
