def test_no_query(self):
filter_request = FeedFilterRequest(self.test_file_path)
filter_response = filter_request.filter()
self.assertEqual(filter_response.status_code, FAILURE_CODE)
self.assertIsNotNone(filter_response.message)
self.assertIsNone(filter_response.file_path)
self.assertEqual(len(filter_response.applied_filters), 0)
def test_dir_file_path(self):
filter_request = FeedFilterRequest('../tests/test-data')
filter_response = filter_request.filter()
self.assertEqual(filter_response.status_code, FAILURE_CODE)
self.assertIsNotNone(filter_response.message)
self.assertIsNone(filter_response.file_path)
self.assertEqual(len(filter_response.applied_filters), 0)
def __test_apply_filters(self):
filter_request = FeedFilterRequest(self.test_file_path,
price_upper_limit=10,
any_query=self.test_any_query)
filter_response = filter_request.filter(keep_db=False)
self.assertEqual(filter_response.status_code, SUCCESS_CODE)
self.assertIsNotNone(filter_response.message)
self.assertEqual(len(filter_request.queries), 2)
self.assertEqual(len(filter_response.applied_filters), 2)
self.assertTrue(filter_request.number_of_records > 0)
self.assertTrue(filter_request.number_of_filtered_records > 0)
self.assertIsNotNone(filter_request.filtered_file_path)
self.assertTrue(isfile(filter_request.filtered_file_path))
self.assertEqual(filter_request.filtered_file_path,
filter_response.file_path)
# clean up
remove(filter_request.filtered_file_path)
def test_default_values(self):
filter_request = FeedFilterRequest(self.test_file_path)
self.assertIsNone(filter_request.item_ids)
self.assertIsNone(filter_request.leaf_category_ids)
self.assertIsNone(filter_request.seller_names)
self.assertIsNone(filter_request.gtins)
self.assertIsNone(filter_request.epids)
self.assertIsNone(filter_request.price_upper_limit)
self.assertIsNone(filter_request.price_lower_limit)
self.assertIsNone(filter_request.item_location_countries)
self.assertIsNone(filter_request.inferred_epids)
self.assertIsNone(filter_request.item_location_countries)
self.assertIsNone(filter_request.any_query)
self.assertIsNone(filter_request.filtered_file_path)
self.assertEqual(filter_request.compression_type,
FileFormat.GZIP.value)
self.assertEqual(filter_request.separator, '\t')
self.assertEqual(filter_request.encoding, FileEncoding.UTF8.value)
self.assertEqual(filter_request.rows_chunk_size, DATA_FRAME_CHUNK_SIZE)
self.assertEqual(filter_request.number_of_records, 0)
self.assertEqual(filter_request.number_of_filtered_records, 0)
self.assertEqual(len(filter_request.queries), 0)
def __create_requests(self, token):
if ConfigField.REQUESTS.value not in self.__config_json_obj:
raise ConfigError('No \"%s\" field exists in the config file %s' %
(str(ConfigField.REQUESTS), self.file_path))
for req in self.__config_json_obj[ConfigField.REQUESTS.value]:
feed_obj = None
feed_field = req.get(ConfigField.FEED_REQUEST.value)
if feed_field:
feed_obj = Feed(
feed_field.get(FeedField.TYPE.value),
feed_field.get(FeedField.SCOPE.value),
feed_field.get(FeedField.CATEGORY_ID.value),
feed_field.get(FeedField.MARKETPLACE_ID.value), token,
feed_field.get(FeedField.DATE.value),
feed_field.get(FeedField.ENVIRONMENT.value),
feed_field.get(FeedField.DOWNLOAD_LOCATION.value),
feed_field.get(FeedField.FILE_FORMAT.value))
filter_request_obj = None
filter_field = req.get(ConfigField.FILTER_REQUEST.value)
if filter_field:
filter_request_obj = FeedFilterRequest(
str(filter_field.get(FilterField.INPUT_FILE_PATH.value)),
filter_field.get(FilterField.ITEM_IDS.value),
filter_field.get(FilterField.LEAF_CATEGORY_IDS.value),
filter_field.get(FilterField.SELLER_NAMES.value),
filter_field.get(FilterField.GTINS.value),
filter_field.get(FilterField.EPIDS.value),
filter_field.get(FilterField.PRICE_LOWER_LIMIT.value),
filter_field.get(FilterField.PRICE_UPPER_LIMIT.value),
filter_field.get(
FilterField.ITEM_LOCATION_COUNTRIES.value),
filter_field.get(FilterField.INFERRED_EPIDS.value),
filter_field.get(FilterField.ANY_QUERY.value),
filter_field.get(FilterField.FILE_FORMAT.value))
config_request_obj = ConfigRequest(feed_obj, filter_request_obj)
self.requests.append(config_request_obj)
'46103', '33738', '33728', '33730', '33729', '33731', '33732', '33727', '171115', '171117', '33733', '33740', '33741', '33742', '33744', '42614', '33746', '180090', '179696', '66471', '33747', '66479', '43961', '170141', '33749', '122154', '43953' ] feed_filter_obj = FeedFilterRequest(input_file_path='./output-1-0.gz') file_path = feed_filter_obj.store_mysql(leaf_category_ids=leaf_category_ids)
# any query to filter the feed file
parser.add_argument(
'-qf',
help=
'any other query to filter the feed file. See Python dataframe query format'
)
# parse the arguments
args = parser.parse_args()
start = time.time()
if args.filteronly:
# create the filtered file
feed_filter_obj = FeedFilterRequest(args.downloadlocation, args.itemf,
args.lf, args.sellerf, args.gtinf,
args.epidf, args.pricelf, args.priceuf,
args.locf, args.iepidf, args.qf,
args.format)
filter_response = feed_filter_obj.filter()
if filter_response.status_code != SUCCESS_CODE:
print(filter_response.message)
else:
# download the feed file if --filteronly option is not set
feed_obj = Feed(FeedType.ITEM.value, args.scope, args.c1, args.mkt,
args.token, args.dt, args.env, args.downloadlocation,
args.format)
get_response = feed_obj.get()
if get_response.status_code != SUCCESS_CODE:
logger.error(
'Exception in downloading feed. Cannot proceed\nFile path: %s\n Error message: %s\n',
def test_any_query_format(self):
filter_request = FeedFilterRequest(self.test_file_path,
any_query=self.test_any_query)
self.assertEqual(filter_request.any_query,
'(' + self.test_any_query + ')')