def test_list_ingested(self, mock_aws): conf = read_conf('example-config.json') ingester = Ingester(conf) self.assertIsNotNone(ingester) # 200 with empty [] should be OK with patch('requests.get', return_value=Mock(status_code=200)) as mock_get: mock_get.return_value.json.return_value = [] empty = ingester.list_ingested() self.assertIsInstance(empty, list) self.assertEqual(len(empty), 0) # 404 with empty body should be OK with patch('requests.get', return_value=Mock(status_code=404)) as mock_get: mock_get.return_value.json.return_value = '' empty = ingester.list_ingested() self.assertIsInstance(empty, list) self.assertEqual(len(empty), 0) with patch('requests.get', return_value=Mock(status_code=500)) as mock_get: mock_get.return_value.json.return_value = '' with self.assertRaises(OSError) as cm: empty = ingester.list_ingested() self.assertEqual(str(cm.exception), 'HTTP 500') with patch('requests.get', return_value=Mock(status_code=200)) as mock_get: mock_get.return_value.json.return_value = [{'name': 'one'}, {'name': 'two'}] result = ingester.list_ingested() self.assertIsInstance(result, list) self.assertEqual(len(result), 2)
def parse_command(description='Ingest records from HCP to Database through API server'): parser = ArgumentParser(description=description) parser.add_argument('name', help='Path to the json.xz file to be ingested') parser.add_argument('-t', '--tracker', default='', help='A string, commonly name of an object in store') parser.add_argument('--conf', default='debugger_api_conf.json', help='Path to config.json. Default = debugger_api_conf.json') args = parser.parse_args() if args.tracker: return args.name, args.tracker, args.conf else: return args.name, args.name, args.conf if __name__ == "__main__": #logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG) logging.basicConfig(format='%(levelname)s:%(message)s') # It only needs DB_API of a normal conf file: API end point, secret and optional schema # But currently Ingester is not flexible for it. try: file_name, tracker, conf_file = parse_command('Ingest records from HCP to Database through API server') except Exception as e: logging.error(e) sys.exit(2) logging.debug("Ingesting %s tracked as %s with connection conf in %s" % (file_name, tracker, conf_file)) conf = read_conf(conf_file) ingester = Ingester(conf) #logging.debug(ingester.check_input(tracker)) ingester.put_local_xz(file_name, tracker, True)
#!/usr/bin/env python3 import sys import logging from ingest import parse_command, Ingester logger = logging.getLogger('reporting-ingest') if __name__ == "__main__": try: conf = parse_command('Ingest records from HCP to Database through API server') except Exception as e: logger.error(e) sys.exit(2) end_point = conf['DB_API']['ENDPOINT'] log_file = "%s.log" % end_point[end_point.index('/')+2:].replace('/', '_') # In case there is a / at the end of end point log_file = log_file.replace('_.log', '.log') file_handler = logging.handlers.RotatingFileHandler(log_file, maxBytes=30000000) file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s')) logger.addHandler(file_handler) logger.setLevel(logging.DEBUG) logger.debug('Start an ingest job') ingester = Ingester(conf) ingester.batch()
import logging from ingest import parse_command, Ingester logger = logging.getLogger('reporting-ingest') if __name__ == "__main__": try: conf = parse_command('Ingest records from AWS to Database through API server') except Exception as e: logger.error(e) sys.exit(2) end_point = conf['DB_API']['ENDPOINT'] log_file = "%s.log" % end_point[end_point.index('/') + 2:].replace('/', '_') # In case there is a / at the end of end point log_file = log_file.replace('_.log', '.log') file_handler = logging.handlers.RotatingFileHandler(log_file, maxBytes=30000000) file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s')) logger.addHandler(file_handler) logger.setLevel(logging.DEBUG) logger.debug('Start an ingest job') ingester = Ingester(conf) try: ingester.batch() except Exception as err: logger.error('Batch ingest failed. %s', err)