def test_list_ingested(self, mock_aws):
conf = read_conf('example-config.json')
ingester = Ingester(conf)
self.assertIsNotNone(ingester)
# 200 with empty [] should be OK
with patch('requests.get', return_value=Mock(status_code=200)) as mock_get:
mock_get.return_value.json.return_value = []
empty = ingester.list_ingested()
self.assertIsInstance(empty, list)
self.assertEqual(len(empty), 0)
# 404 with empty body should be OK
with patch('requests.get', return_value=Mock(status_code=404)) as mock_get:
mock_get.return_value.json.return_value = ''
empty = ingester.list_ingested()
self.assertIsInstance(empty, list)
self.assertEqual(len(empty), 0)
with patch('requests.get', return_value=Mock(status_code=500)) as mock_get:
mock_get.return_value.json.return_value = ''
with self.assertRaises(OSError) as cm:
empty = ingester.list_ingested()
self.assertEqual(str(cm.exception), 'HTTP 500')
with patch('requests.get', return_value=Mock(status_code=200)) as mock_get:
mock_get.return_value.json.return_value = [{'name': 'one'}, {'name': 'two'}]
result = ingester.list_ingested()
self.assertIsInstance(result, list)
self.assertEqual(len(result), 2)
def parse_command(description='Ingest records from HCP to Database through API server'):
parser = ArgumentParser(description=description)
parser.add_argument('name', help='Path to the json.xz file to be ingested')
parser.add_argument('-t', '--tracker', default='', help='A string, commonly name of an object in store')
parser.add_argument('--conf', default='debugger_api_conf.json', help='Path to config.json. Default = debugger_api_conf.json')
args = parser.parse_args()
if args.tracker:
return args.name, args.tracker, args.conf
else:
return args.name, args.name, args.conf
if __name__ == "__main__":
#logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG)
logging.basicConfig(format='%(levelname)s:%(message)s')
# It only needs DB_API of a normal conf file: API end point, secret and optional schema
# But currently Ingester is not flexible for it.
try:
file_name, tracker, conf_file = parse_command('Ingest records from HCP to Database through API server')
except Exception as e:
logging.error(e)
sys.exit(2)
logging.debug("Ingesting %s tracked as %s with connection conf in %s" % (file_name, tracker, conf_file))
conf = read_conf(conf_file)
ingester = Ingester(conf)
#logging.debug(ingester.check_input(tracker))
ingester.put_local_xz(file_name, tracker, True)
#!/usr/bin/env python3
import sys
import logging
from ingest import parse_command, Ingester
logger = logging.getLogger('reporting-ingest')
if __name__ == "__main__":
try:
conf = parse_command('Ingest records from HCP to Database through API server')
except Exception as e:
logger.error(e)
sys.exit(2)
end_point = conf['DB_API']['ENDPOINT']
log_file = "%s.log" % end_point[end_point.index('/')+2:].replace('/', '_')
# In case there is a / at the end of end point
log_file = log_file.replace('_.log', '.log')
file_handler = logging.handlers.RotatingFileHandler(log_file, maxBytes=30000000)
file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
logger.addHandler(file_handler)
logger.setLevel(logging.DEBUG)
logger.debug('Start an ingest job')
ingester = Ingester(conf)
ingester.batch()
import logging
from ingest import parse_command, Ingester
logger = logging.getLogger('reporting-ingest')
if __name__ == "__main__":
try:
conf = parse_command('Ingest records from AWS to Database through API server')
except Exception as e:
logger.error(e)
sys.exit(2)
end_point = conf['DB_API']['ENDPOINT']
log_file = "%s.log" % end_point[end_point.index('/') + 2:].replace('/', '_')
# In case there is a / at the end of end point
log_file = log_file.replace('_.log', '.log')
file_handler = logging.handlers.RotatingFileHandler(log_file, maxBytes=30000000)
file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
logger.addHandler(file_handler)
logger.setLevel(logging.DEBUG)
logger.debug('Start an ingest job')
ingester = Ingester(conf)
try:
ingester.batch()
except Exception as err:
logger.error('Batch ingest failed. %s', err)
