def run_transform(data_set_config, transform, earliest, latest): data_set = DataSet.from_group_and_type( config.BACKDROP_READ_URL, data_set_config['data_group'], data_set_config['data_type'], ) earliest = parse_time_as_utc(earliest) latest = parse_time_as_utc(latest) data = data_set.get( query_parameters=get_query_parameters(transform, earliest, latest) ) transform_function = get_transform_function(transform) transformed_data = transform_function(data['data'], transform, data_set_config) if 'additionalFields' in transform['options']: additionalFields = transform['options']['additionalFields'] transformed_data = [ merge_additional_fields(datum, additionalFields) for datum in transformed_data] output_data_set = get_or_get_and_create_output_dataset( transform, data_set_config) output_data_set.post(transformed_data) stats_client.incr('run_transform.success')
def run_transform(data_set_config, transform, earliest, latest): data_set = DataSet.from_group_and_type( config.BACKDROP_READ_URL, data_set_config['data_group'], data_set_config['data_type'], ) earliest = parse_time_as_utc(earliest) latest = parse_time_as_utc(latest) data = data_set.get( query_parameters=get_query_parameters(transform, earliest, latest)) transform_function = get_transform_function(transform) transformed_data = transform_function(data['data'], transform, data_set_config) if 'additionalFields' in transform['options']: additionalFields = transform['options']['additionalFields'] transformed_data = [ merge_additional_fields(datum, additionalFields) for datum in transformed_data ] output_data_set = get_or_get_and_create_output_dataset( transform, data_set_config) output_data_set.post(transformed_data) stats_client.incr('run_transform.success')
def channel_volumetrics(rows): rows = list(rows) yield [ "_timestamp", "_id", "successful_agent", "successful_ivr", "successful_web", "total_agent", "total_ivr", "total_web" ] for column in range(1, 8): all = rows[5][column] if all == 0: return date = rows[1][column] agent_successful = rows[2][column] ivr_successful = rows[3][column] web_successful = rows[4][column] agent_total = rows[6][column] ivr_total = rows[7][column] web_total = rows[8][column] yield [ date, parse_time_as_utc(date).date().isoformat(), agent_successful, ivr_successful, web_successful, agent_total, ivr_total, web_total ]
def test_that_date_strings_get_converted_to_utc(self): some_datetime = parse_time_as_utc( '2014-01-02T06:04:05+03:30') assert_that(isinstance(some_datetime, datetime), is_(True)) assert_that(some_datetime, is_(datetime(2014, 1, 2, 2, 34, 5, tzinfo=pytz.utc)))
def channel_volumetrics(rows): rows = list(rows) yield ["_timestamp", "_id", "successful_agent", "successful_ivr", "successful_web", "total_agent", "total_ivr", "total_web"] for column in range(1, 8): all = rows[5][column] if all == 0: return date = rows[1][column] agent_successful = rows[2][column] ivr_successful = rows[3][column] web_successful = rows[4][column] agent_total = rows[6][column] ivr_total = rows[7][column] web_total = rows[8][column] yield [date, parse_time_as_utc(date).date().isoformat(), agent_successful, ivr_successful, web_successful, agent_total, ivr_total, web_total]
def test_datetime_is_converted_to_utc(self): us_eastern_time = d_tz(2012, 12, 12, 12, tzinfo=pytz.timezone("US/Eastern")) assert_that(parse_time_as_utc(us_eastern_time), equal_to(d_tz(2012, 12, 12, 17)))
def parse(datum): if '_timestamp' in datum: try: datum['_timestamp'] = parse_time_as_utc( datum['_timestamp']) except ValueError: raise ParseError( '_timestamp is not a valid timestamp, it must be ISO8601') return Record(datum)
def parse_timestamps(record): """Parses a timestamp in a record >>> parse_timestamps({'_timestamp': '2012-12-12T00:00:00'}) ({'_timestamp': datetime.datetime(2012, 12, 12, 0, 0, tzinfo=<UTC>)}, None) >>> parse_timestamps({'_start_at': '2012-12-12T00:00:00'}) ({'_start_at': datetime.datetime(2012, 12, 12, 0, 0, tzinfo=<UTC>)}, None) >>> parse_timestamps({'_end_at': '2012-12-12T00:00:00'}) ({'_end_at': datetime.datetime(2012, 12, 12, 0, 0, tzinfo=<UTC>)}, None) >>> parse_timestamps({}) ({}, None) >>> record, error = parse_timestamps({'_timestamp': 'invalid'}) >>> record {'_timestamp': 'invalid'} >>> error '_timestamp is not a valid timestamp, it must be ISO8601' >>> record, error = parse_timestamps({'_start_at': 'invalid'}) >>> record {'_start_at': 'invalid'} >>> error '_start_at is not a valid timestamp, it must be ISO8601' """ error = None if '_timestamp' in record: try: record['_timestamp'] = parse_time_as_utc(record['_timestamp']) except (TypeError, ValueError): error = '_timestamp is not a valid timestamp, it must be ISO8601' if '_start_at' in record: try: record['_start_at'] = parse_time_as_utc(record['_start_at']) except (TypeError, ValueError): error = '_start_at is not a valid timestamp, it must be ISO8601' if '_end_at' in record: try: record['_end_at'] = parse_time_as_utc(record['_end_at']) except (TypeError, ValueError): error = '_end_at is not a valid timestamp, it must be ISO8601' return (record, error)
def service_volumetrics(rows): rows = list(rows) yield ["_timestamp", "_id", "timeSpan", "successful_tax_disc", "successful_sorn"] timestamp = rows[2][1] taxDiskApplications = rows[24][2] sornApplications = rows[25][2] yield [timestamp, parse_time_as_utc(timestamp).date().isoformat(), "day", taxDiskApplications, sornApplications]
def service_volumetrics(rows): rows = list(rows) yield [ "_timestamp", "_id", "timeSpan", "successful_tax_disc", "successful_sorn" ] timestamp = rows[2][1] taxDiskApplications = rows[24][2] sornApplications = rows[25][2] yield [ timestamp, parse_time_as_utc(timestamp).date().isoformat(), "day", taxDiskApplications, sornApplications ]
def test_datetime_with_no_timezone_is_given_utc(self): assert_that(parse_time_as_utc(d(2012, 12, 12, 12)), equal_to(d_tz(2012, 12, 12, 12)))
def test_time_string_is_converted_to_utc(self): assert_that(parse_time_as_utc("2012-12-12T12:12:12+01:00"), equal_to(d_tz(2012, 12, 12, 11, 12, 12)))
def failure(service_type, reason, failures, description, timestamp): date = parse_time_as_utc(timestamp).date() id = "%s.%s.%s" % (date.isoformat(), service_type, reason) return [timestamp, id, service_type, reason, failures, description]
def parse_timestamps(record): """Parses a timestamp in a record >>> parse_timestamps({'_timestamp': '2012-12-12T00:00:00'}) ({'_timestamp': datetime.datetime(2012, 12, 12, 0, 0, tzinfo=<UTC>)}, None) >>> parse_timestamps({'_start_at': '2012-12-12T00:00:00'}) ({'_start_at': datetime.datetime(2012, 12, 12, 0, 0, tzinfo=<UTC>)}, None) >>> parse_timestamps({'_end_at': '2012-12-12T00:00:00'}) ({'_end_at': datetime.datetime(2012, 12, 12, 0, 0, tzinfo=<UTC>)}, None) >>> parse_timestamps({}) ({}, None) >>> record, error = parse_timestamps({'_timestamp': 'invalid'}) >>> record {'_timestamp': 'invalid'} >>> error 'the _timestamp must be a date in the format yyyy-MM-ddT00:00:00Z' >>> record, error = parse_timestamps({'_timestamp': '31-11-12T00:00:00'}) >>> record {'_timestamp': '31-11-12T00:00:00'} >>> error '31-11-12T00:00:00 in the _timestamp field is not a valid date' >>> record, error = parse_timestamps({'_start_at': 'invalid'}) >>> record {'_start_at': 'invalid'} >>> error '_start_at must be a date in the format yyyy-MM-ddT00:00:00Z' >>> record, error = parse_timestamps({'_start_at': '31-11-12T00:00:00'}) >>> record {'_start_at': '31-11-12T00:00:00'} >>> error '31-11-12T00:00:00 in the _start_at field is not a valid date' >>> record, error = parse_timestamps({'_end_at': 'invalid'}) >>> record {'_end_at': 'invalid'} >>> error '_end_at must be a date in the format yyyy-MM-ddT00:00:00Z' >>> record, error = parse_timestamps({'_end_at': '31-11-12T00:00:00'}) >>> record {'_end_at': '31-11-12T00:00:00'} >>> error '31-11-12T00:00:00 in the _end_at field is not a valid date' """ error = None if '_timestamp' in record: try: record['_timestamp'] = parse_time_as_utc(record['_timestamp']) except TypeError: error = 'the _timestamp must be a date in the format ' \ 'yyyy-MM-ddT00:00:00Z' except ValueError: error = '{} in the _timestamp field is not a valid date'.format( record['_timestamp']) if '_start_at' in record: try: record['_start_at'] = parse_time_as_utc(record['_start_at']) except TypeError: error = '_start_at must be a date in the format ' \ 'yyyy-MM-ddT00:00:00Z' except ValueError: error = '{} in the _start_at field is not a valid date'.format( record['_start_at']) if '_end_at' in record: try: record['_end_at'] = parse_time_as_utc(record['_end_at']) except TypeError: error = '_end_at must be a date in the format yyyy-MM-ddT00:00:00Z' except ValueError: error = '{} in the _end_at field is not a valid date'.format( record['_end_at']) return (record, error)
def date_or_none(string): try: return parse_time_as_utc(string) except (TypeError, ValueError): return None
def ceg_date(rows, column): try: return parse_time_as_utc(rows[3][column]) except IndexError: return None
def test_that_date_strings_get_converted_to_utc(self): some_datetime = parse_time_as_utc('2014-01-02T06:04:05+03:30') assert_that(isinstance(some_datetime, datetime), is_(True)) assert_that(some_datetime, is_(datetime(2014, 1, 2, 2, 34, 5, tzinfo=pytz.utc)))
def test_valid_time_string_is_parsed(self): assert_that(parse_time_as_utc("2012-12-12T12:12:12+00:00"), equal_to(d_tz(2012, 12, 12, 12, 12, 12)))