def fetch_gear_by_id(self, gear_id): try: sql = "SELECT * from gear WHERE id = %s;" self.logger.debug('Executing SQL: {0}'.format(sql)) with self.db_conn.cursor() as cursor: cursor.execute(sql, [gear_id]) record = cursor.fetchone() if record: gear = Gear(name=record[1], description=record[2], thumb_url=record[3], amazon_link=record[4], image_url=record[5], manufacturer=record[6], type_id=record[7], features=record[8]) return gear else: return None except (db_utils.psycopg2.DatabaseError, Exception) as err: self.logger.exception( 'An error occurred during fetch by id operation on gear table!' ) log_utils.close_logger(self.logger) db_utils.close_db_connection(self.db_conn) raise err
def delete_gear(self, gear_id): try: sql = "DELETE FROM gear WHERE id = %s;" self.logger.debug('Executing SQL: {0}'.format(sql)) with self.db_conn.cursor() as cursor: cursor.execute(sql, [gear_id]) self.db_conn.commit() except (db_utils.psycopg2.DatabaseError, Exception) as err: self.logger.exception( 'An error occurred during delete operation on gear table!') log_utils.close_logger(self.logger) db_utils.close_db_connection(self.db_conn) raise err
def insert_geartype_retrieved(self, gear_type_retrieved): try: ret_id = None sql = "INSERT INTO gear_type_retrieved " \ "(amazon_node_id) " \ "VALUES (%s) RETURNING amazon_node_id;" self.logger.debug('Executing SQL: {0}'.format(sql)) with self.db_conn.cursor() as cursor: cursor.execute(sql, [gear_type_retrieved.amazon_node_id]) ret_id = cursor.fetchone()[0] self.db_conn.commit() return ret_id except (db_utils.psycopg2.DatabaseError, Exception) as db_err: self.logger.exception( 'An error occurred during insert operation on gear_type_retrieved table!' ) log_utils.close_logger(self.logger) db_utils.close_db_connection(self.db_conn) raise db_err
def fetch_geartype_retrieved_by_id(self, node_id): try: sql = "SELECT * FROM gear_type_retrieved WHERE amazon_node_id = %s;" self.logger.debug('Executing SQL: {0}'.format(sql)) with self.db_conn.cursor() as cursor: cursor.execute(sql, [node_id]) row = cursor.fetchone() if row: # Create GearTypeRetrieved instance from fetched row values gear_type_retrieved = GearTypeRetrieved(amazon_node_id=row[0]) return gear_type_retrieved else: return None except (db_utils.psycopg2.DatabaseError, Exception) as db_err: self.logger.exception( 'An error occurred during fetch operation on gear_type_retrieved table!' ) log_utils.close_logger(self.logger) db_utils.close_db_connection(self.db_conn) raise db_err
def fetch_geartype_by_name(self, name): try: sql = "SELECT * FROM gear_type WHERE cat_name = %s;" self.logger.debug('Executing SQL: {0}'.format(sql)) with self.db_conn.cursor() as cursor: cursor.execute(sql, [name]) row = cursor.fetchone() if row: gear_type = GearType(amazon_node_id=row[0], name=row[1], parent_node_id=row[2], is_leaf_node=row[3]) return gear_type else: return None except (db_utils.psycopg2.DatabaseError, Exception) as err: self.logger.exception( 'An error occurred during fetch by name operation on gear_type table!' ) log_utils.close_logger(self.logger) db_utils.close_db_connection(self.db_conn) raise err
def insert_geartype(self, gear_type): try: sql = "INSERT INTO gear_type " \ "(amazon_node_id, cat_name, parent_node_id, is_leaf_node) " \ "VALUES (%s, %s, %s, %s) RETURNING amazon_node_id;" self.logger.debug('Executing SQL: {0}'.format(sql)) with self.db_conn.cursor() as cursor: cursor.execute( sql, (gear_type.amazon_node_id, gear_type.name, gear_type.parent_node_id, gear_type.is_leaf_node)) rec = cursor.fetchone() if rec: self.db_conn.commit() return rec[0] else: return None except (db_utils.psycopg2.DatabaseError, Exception) as err: self.logger.exception( 'An error occurred during insert operation on gear_type table!' ) log_utils.close_logger(self.logger) db_utils.close_db_connection(self.db_conn) raise err
def save_gear(self, gear): try: sql = "INSERT INTO gear (name, description, thumb_url, amazon_link, " \ "image_url, manufacturer, type_id, features) " \ "VALUES (%s, %s, %s, %s, %s, %s, %s, %s) RETURNING id;" self.logger.debug('Executing SQL: {0}'.format(sql)) with self.db_conn.cursor() as cursor: cursor.execute( sql, (gear.name, gear.description, gear.thumb_url, gear.amazon_link, gear.image_url, gear.manufacturer, gear.type_id, gear.features)) rec = cursor.fetchone() if rec: self.db_conn.commit() return rec[0] else: return None except (db_utils.psycopg2.DatabaseError, Exception) as err: self.logger.exception( 'An error occurred during save operation on gear table!') log_utils.close_logger(self.logger) db_utils.close_db_connection(self.db_conn) raise err
log_level="DEBUG") gear_logger = log_utils.init_root_logger( logger_name="gear_rbt_logger", log_path="../log/gear_rbt.log", log_level="DEBUG") # Define list of test case classes to run tests = [ ConfigUtilsTest, DBUtilsTest, GearTypeRetrieverTest, GearTypeRetrievedDAOTest, GearTypeDAOTest ] suites_list = [] # Initialize test loader loader = unittest.TestLoader() # Loop through test lists to load tests from test cases for test in tests: suites_list.append(loader.loadTestsFromTestCase(test)) # Create big_suite of all test suites and execute it big_suite = unittest.TestSuite(suites_list) unittest.TextTestRunner(verbosity=2).run(big_suite) finally: if geartype_logger: log_utils.close_logger(geartype_logger) if gear_logger: log_utils.close_logger(gear_logger)
ret_id = self.dao.insert_geartype_retrieved(gear_type_retrieved=gear_type_retrieved) self.assertIsNotNone(ret_id) self.assertEquals(ret_id, 123456) # Fetch GearTypeRetrieved record by id fetched_by_id = self.dao.fetch_geartype_retrieved_by_id(node_id=ret_id) self.assertIsNotNone(fetched_by_id) self.assertEquals(fetched_by_id.amazon_node_id, 123456) # Delete record from database self.dao.delete_geartype_retrieved(node_id=ret_id) self.assertIsNone(self.dao.fetch_geartype_retrieved_by_id(node_id=ret_id)) """ MAIN - Execute unit tests. """ if __name__ == '__main__': root_logger = None try: # Initialize root logger here for logging to work during tests root_logger = log_utils.init_root_logger(logger_name="gearstack_rbt", log_path='../log/geartype_rbt.log', log_level="DEBUG") # Define and execute test suite suite = unittest.TestLoader().loadTestsFromTestCase(GearTypeRetrievedDAOTest) unittest.TextTestRunner(verbosity=2).run(suite) finally: if root_logger: log_utils.close_logger(root_logger)
def main(argv): # Gather command line args env = None log_level = None usage = 'geartype_rbt_runner.py --env <local | dev | prod> --log_level <info | debug | warning | error>' try: opts, args = getopt.getopt(argv, "h", ["env=", "log_level="]) except getopt.GetoptError: print('An invalid argument was specified!') sys.exit(2) for opt, arg in opts: if opt == '-h': print(usage) sys.exit(0) elif opt == '--env': env = arg elif opt == '--log_level': log_level = arg # Validate command line args try: # env arg if env.lower() not in ('local', 'dev', 'prod'): print('An invalid value {0} was specified for arg [--env]'.format( env)) print(usage) sys.exit(2) # log_level arg if log_level.lower() not in ('info', 'debug', 'warning', 'error', 'critical'): print('An invalid value {0} was specified for arg [--log_level]'. format(log_level)) print(usage) sys.exit(2) except AttributeError: print('An invalid argument was specified!') print(usage) sys.exit(2) # Create logger and log options specified logger = log_utils.init_root_logger(logger_name="gearstack_rbt", log_path='../log/geartype_rbt.log', log_level=log_level) logger.info('Gearstack Gear Type Bot') logger.info('Environment: {0}'.format(env.upper())) logger.info('Log Level: {0}'.format(log_level.upper())) # Fetch configuration data for specified environment bot_configs = config_utils.fetch_bot_config(env) types = config_utils.fetch_data_config('cat_data_keys') # Instantiate retriever object type_retriever = GearTypeRetriever( amazon_config=bot_configs['amazon_config'], bot_db_configs=bot_configs['db_config']['bot_db_props'], app_db_configs=bot_configs['db_config']['app_db_props']) # Iterate through types list and persist try: for type in types: nodes_for_type = config_utils.fetch_data_config(config_name=type) type_retriever.save_geartypes(node_ids=nodes_for_type) logger.info('Finished persisting gear types!') except Exception: logger.exception('An error occurred during persistence of gear types!') finally: log_utils.close_logger(logger)
def main(argv): # Gather command line args env = None log_level = None browse_node = None usage = "gear_rbt_runner.py --env <local|dev|prod> --log_level <info|debug|warning|error> " \ "--browse_node <browse_node_id>" try: opts, args = getopt.getopt(argv, "h", ["env=", "log_level=", "browse_node="]) except getopt.GetoptError: print('An invalid argument was specified!') sys.exit(2) for opt, arg in opts: if opt == '-h': print(usage) sys.exit(0) elif opt == '--env': env = arg elif opt == '--log_level': log_level = arg elif opt == '--browse_node': browse_node = arg # Validate command line args try: # env arg if env.lower() not in ('local', 'dev', 'prod'): print('An invalid value {0} was specified for arg [--env]'.format( env)) print(usage) sys.exit(2) # log_level arg if log_level.lower() not in ('info', 'debug', 'warning', 'error', 'critical'): print('An invalid value {0} was specified for arg [--log_level]'. format(log_level)) print(usage) sys.exit(2) # browse_node arg browse_node = int(browse_node) except (AttributeError, ValueError): print('An invalid argument was specified!') print(usage) sys.exit(2) # Create logger and log options specified logger = log_utils.init_root_logger(logger_name="gear_rbt_logger", log_path='../log/gear_rbt.log', log_level=log_level) logger.info('Gearstack Gear Bot') logger.info('Environment: {0}'.format(env.upper())) logger.info('Log Level: {0}'.format(log_level.upper())) logger.info('Amazon Browse Node: {0}'.format(browse_node)) # Fetch configuration data for specified environment bot_configs = config_utils.fetch_bot_config(env) # Instantiate gear retriever instance gear_retriever = GearRetriever( amazon_config=bot_configs['amazon_config'], app_db_config=bot_configs['db_config']['app_db_props'], exclusion_keys=config_utils.fetch_data_config('excluded_keywords')) # Loop through nodes_to_search list and execute amazon API search # and item save try: fetched_items = gear_retriever.search_items(browse_node_id=browse_node) gear_retriever.save_gear(item_list=fetched_items, browse_node=browse_node) logger.info("Finished DB population for browse node {0}!!!".format( browse_node)) except Exception: logger.exception( 'Gear Retriever bot encountered an exception during execution!') finally: log_utils.close_logger(logger)