def test_filter_with_unexisting_filter_type(self):
filters = {'unexisting': 'filter'}
data_filter = DataFilter(filters)
filtered_data = data_filter.filter(self.data)
self.assertDictEqual(self.data, filtered_data)
def test_filter_start_date_earlier_than_all_dates(self):
filters = {'start_date': 0}
data_filter = DataFilter(filters)
filtered_data = data_filter.filter(self.data)
self.assertDictEqual(self.data, filtered_data)
def test_data_consistency_after_segment_filtering(self):
filters = {'countries': 'US',
'devices': 'DO'}
data_filter = DataFilter(filters)
filtered_data = data_filter.filter(self.data)
for tuple, segment in filtered_data.items():
self.assertEqual(self.data[tuple], segment)
def test_filter_by_not_contained_country(self):
filters = {'countries': 'not contained'}
data_filter = DataFilter(filters)
filtered_data = data_filter.filter(self.data)
# empty dictionaries evaluate to False
self.assertFalse(filtered_data)
def test_filter_end_date_later_than_all_dates(self):
next_year = datetime.now() + timedelta(days=365)
filters = {'end_date': next_year.timestamp()}
data_filter = DataFilter(filters)
filtered_data = data_filter.filter(self.data)
self.assertDictEqual(self.data, filtered_data)
def test_filter_end_date_earlier_than_all_dates(self):
filters = {'end_date': 0}
data_filter = DataFilter(filters)
filtered_data = data_filter.filter(self.data)
for segment in filtered_data.values():
self.assertFalse(segment.inventory_volumes)
self.assertFalse(segment.dates)
def test_filter_by_segment(self):
devices = ['MT', 'DO']
for device in devices:
filters = {'devices': device}
data_filter = DataFilter(filters)
filtered_data = data_filter.filter(self.data)
devices_in_filtered_data = [tuple[1] for tuple in list(filtered_data.keys())]
for device_in_filtered_data in devices_in_filtered_data:
self.assertIn(device_in_filtered_data, [device])
def test_filter_by_country(self):
filters = {'countries': 'US,ES'}
data_filter = DataFilter(filters)
filtered_data = data_filter.filter(self.data)
countries_in_filtered_data = [tuple[0] for tuple in list(filtered_data.keys())]
for country_in_filtered_data in countries_in_filtered_data:
self.assertIn(country_in_filtered_data, ['US', 'ES'])
def test_filter_start_date_later_than_all_dates(self):
next_year = datetime.now() + timedelta(days=365)
filters = {'start_date': next_year.timestamp()}
data_filter = DataFilter(filters)
filtered_data = data_filter.filter(self.data)
for segment in filtered_data.values():
self.assertFalse(segment.inventory_volumes)
self.assertFalse(segment.dates)
def test_filter_end_date_check_all_dates_are_smaller(self):
time = (self.current_time + timedelta(days=randint(0, 10))).timestamp()
filters = {'end_date': time}
data_filter = DataFilter(filters)
filtered_data = data_filter.filter(self.data)
for segment in filtered_data.values():
for date in segment.dates:
self.assertLess(date.seconds, time)
def get(self, filename):
try:
page = int(request.args.get('page')) if request.args.get('page') else 0
per_page = int(request.args.get('per_page')) if request.args.get('per_page') else config.per_page
except ValueError:
abort(400, 'page and per_page values should be numbers')
data_filter = DataFilter(request.args)
filename = secure_filename(filename)
data, countries, devices = get_data(os.path.join(config.UPLOAD_FOLDER, filename))
time_period = request.args.get('time_period') if request.args.get('time_period') else config.time_period
grouped_data = group_segment_data_by_time_period(data, time_period)
filtered_data = data_filter.filter(grouped_data)
order_type = request.args.get('order_by') if request.args.get('order_by') else config.order_by
ordered_data = sort_data_by_order_type(list(filtered_data.values()), order_type)
response = data_pb2.SegmentedTimelineDataResponse()
response.data.extend(ordered_data[(page * per_page):(page * per_page + per_page)])
response.countries.extend(countries)
response.devices.extend(devices)
return MessageToDict(response)
def get(self, original_filename, filename_for_comparison):
try:
page = int(
request.args.get('page')) if request.args.get('page') else 0
per_page = int(request.args.get('per_page')) if request.args.get(
'per_page') else config.per_page
except ValueError:
abort(400, 'page and per_page values should be numbers')
filename1 = secure_filename(original_filename)
filename2 = secure_filename(filename_for_comparison)
original_data, _, _ = get_data(
os.path.join(config.UPLOAD_FOLDER, filename1))
comparison_data, _, _ = get_data(
os.path.join(config.UPLOAD_FOLDER, filename2))
time_period = request.args.get('time_period') if request.args.get(
'time_period') else config.time_period
grouped_original_data = group_segment_data_by_time_period(
original_data, time_period)
grouped_comparison_data = group_segment_data_by_time_period(
comparison_data, time_period)
data_filter = DataFilter(request.args)
filtered_original_data = data_filter.filter(grouped_original_data)
filtered_comparison_data = data_filter.filter(grouped_comparison_data)
original_segmented_timeline_data = data_pb2.SegmentedTimelineDataResponse(
)
comparison_segmented_timeline_data = data_pb2.SegmentedTimelineDataResponse(
)
if (filename1, filename2) not in error_importance_by_files:
errors = sorted([
get_error(original_data[tuple], comparison_data[tuple],
time_period) for tuple in original_data.keys()
],
key=lambda x: x.median,
reverse=True)
order_type = request.args.get('order_by') if request.args.get(
'order_by') else config.error_order_by
sorted_errors = sort_data_by_order_type(errors, order_type)
error_importance_by_files[(filename1, filename2)] = [
(error.country, error.device) for error in sorted_errors
]
error_importance = error_importance_by_files[(filename1, filename2)]
sorted_segment_keys_by_importance = [
segment_key for segment_key in error_importance
if segment_key in filtered_original_data.keys()
]
sorted_segment_keys_by_importance_for_page = sorted_segment_keys_by_importance[
(page * per_page):(page * per_page + per_page)]
original_segmented_timeline_data.data.extend([
filtered_original_data[key]
for key in sorted_segment_keys_by_importance_for_page
])
comparison_segmented_timeline_data.data.extend([
filtered_comparison_data[key]
for key in sorted_segment_keys_by_importance_for_page
])
response = data_pb2.SegmentedTimelineCompareResponse(
original_data=original_segmented_timeline_data,
comparison_data=comparison_segmented_timeline_data)
return MessageToDict(response)