def test_save_geartypes(self): # Grab a list of node_ids from config file guitar_nodes = config_utils.fetch_data_config('guitars') print('Fetching the following nodes: {0}'.format(guitar_nodes)) # Execute function of test self.cat_retriever.save_geartypes(node_ids=guitar_nodes)
def test_search_items(self): # Fetch a node to search items for search_node = config_utils.fetch_data_config( config_name='computer_recording')[0] items = self.retriever.search_items(browse_node_id=search_node, price_limit=10000) print len(items) self.assertIsNotNone(items) self.assertTrue(len(items) > 0)
def __init__(self, amazon_config, bot_db_configs, app_db_configs): self.logger = logging.getLogger('gearstack_rbt.{0}'.format( GearTypeRetriever.__name__)) self.amazon_config = amazon_config self.geartype_retrieved_dao = GearTypeRetrievedDAO( bot_db_configs=bot_db_configs) self.geartype_dao = GearTypeDAO(app_db_configs=app_db_configs) self.amazon = AmazonAPI( aws_key=amazon_config['api_access_key'], aws_secret=amazon_config['api_secret_key'], aws_associate_tag=amazon_config['associate_tag']) self.cats_to_ignore = config_utils.fetch_data_config('ignore_cats')
def setUpClass(cls): configs = config_utils.fetch_bot_config(env='local') cls.retriever = GearRetriever( amazon_config=configs['amazon_config'], app_db_config=configs['db_config']['app_db_props'], exclusion_keys=config_utils.fetch_data_config('excluded_keywords'))
def main(argv): # Gather command line args env = None log_level = None usage = 'geartype_rbt_runner.py --env <local | dev | prod> --log_level <info | debug | warning | error>' try: opts, args = getopt.getopt(argv, "h", ["env=", "log_level="]) except getopt.GetoptError: print('An invalid argument was specified!') sys.exit(2) for opt, arg in opts: if opt == '-h': print(usage) sys.exit(0) elif opt == '--env': env = arg elif opt == '--log_level': log_level = arg # Validate command line args try: # env arg if env.lower() not in ('local', 'dev', 'prod'): print('An invalid value {0} was specified for arg [--env]'.format( env)) print(usage) sys.exit(2) # log_level arg if log_level.lower() not in ('info', 'debug', 'warning', 'error', 'critical'): print('An invalid value {0} was specified for arg [--log_level]'. format(log_level)) print(usage) sys.exit(2) except AttributeError: print('An invalid argument was specified!') print(usage) sys.exit(2) # Create logger and log options specified logger = log_utils.init_root_logger(logger_name="gearstack_rbt", log_path='../log/geartype_rbt.log', log_level=log_level) logger.info('Gearstack Gear Type Bot') logger.info('Environment: {0}'.format(env.upper())) logger.info('Log Level: {0}'.format(log_level.upper())) # Fetch configuration data for specified environment bot_configs = config_utils.fetch_bot_config(env) types = config_utils.fetch_data_config('cat_data_keys') # Instantiate retriever object type_retriever = GearTypeRetriever( amazon_config=bot_configs['amazon_config'], bot_db_configs=bot_configs['db_config']['bot_db_props'], app_db_configs=bot_configs['db_config']['app_db_props']) # Iterate through types list and persist try: for type in types: nodes_for_type = config_utils.fetch_data_config(config_name=type) type_retriever.save_geartypes(node_ids=nodes_for_type) logger.info('Finished persisting gear types!') except Exception: logger.exception('An error occurred during persistence of gear types!') finally: log_utils.close_logger(logger)
def test_fetch_cat_nodes(self): # Attempt to retrieve cat_nodes for a category nodes = config.fetch_data_config('guitars') self.assertIsNotNone(nodes) self.assertTrue(len(nodes) == 4)
def main(argv): # Gather command line args env = None log_level = None browse_node = None usage = "gear_rbt_runner.py --env <local|dev|prod> --log_level <info|debug|warning|error> " \ "--browse_node <browse_node_id>" try: opts, args = getopt.getopt(argv, "h", ["env=", "log_level=", "browse_node="]) except getopt.GetoptError: print('An invalid argument was specified!') sys.exit(2) for opt, arg in opts: if opt == '-h': print(usage) sys.exit(0) elif opt == '--env': env = arg elif opt == '--log_level': log_level = arg elif opt == '--browse_node': browse_node = arg # Validate command line args try: # env arg if env.lower() not in ('local', 'dev', 'prod'): print('An invalid value {0} was specified for arg [--env]'.format( env)) print(usage) sys.exit(2) # log_level arg if log_level.lower() not in ('info', 'debug', 'warning', 'error', 'critical'): print('An invalid value {0} was specified for arg [--log_level]'. format(log_level)) print(usage) sys.exit(2) # browse_node arg browse_node = int(browse_node) except (AttributeError, ValueError): print('An invalid argument was specified!') print(usage) sys.exit(2) # Create logger and log options specified logger = log_utils.init_root_logger(logger_name="gear_rbt_logger", log_path='../log/gear_rbt.log', log_level=log_level) logger.info('Gearstack Gear Bot') logger.info('Environment: {0}'.format(env.upper())) logger.info('Log Level: {0}'.format(log_level.upper())) logger.info('Amazon Browse Node: {0}'.format(browse_node)) # Fetch configuration data for specified environment bot_configs = config_utils.fetch_bot_config(env) # Instantiate gear retriever instance gear_retriever = GearRetriever( amazon_config=bot_configs['amazon_config'], app_db_config=bot_configs['db_config']['app_db_props'], exclusion_keys=config_utils.fetch_data_config('excluded_keywords')) # Loop through nodes_to_search list and execute amazon API search # and item save try: fetched_items = gear_retriever.search_items(browse_node_id=browse_node) gear_retriever.save_gear(item_list=fetched_items, browse_node=browse_node) logger.info("Finished DB population for browse node {0}!!!".format( browse_node)) except Exception: logger.exception( 'Gear Retriever bot encountered an exception during execution!') finally: log_utils.close_logger(logger)