def ingest_to_graph():
"""Import e/p/v data and generate response for the POST request to /api/v1/ingest_to_graph."""
input_json = request.get_json()
current_app.logger.info("Ingesting the given list of EPVs - " +
json.dumps(input_json))
expected_keys = set(['ecosystem', 'name', 'version'])
for epv in input_json:
if not expected_keys.issubset(set(epv.keys())):
response = {
'message':
'Invalid keys found in input: ' + ','.join(epv.keys())
}
return flask.jsonify(response), 400
report = data_importer.import_epv_from_s3_http(list_epv=input_json)
response = {
'message': report.get('message'),
'epv': input_json,
'count_imported_EPVs': report.get('count_imported_EPVs')
}
print(response)
# TODO the previous code can raise a runtime exception, does not we need to handle that?
if report.get('status') is not 'Success':
return flask.jsonify(response), 500
else:
return flask.jsonify(response)
def sync_all():
"""Generate response for the GET request to /api/v1/sync_all."""
current_app.logger.info("/api/v1/sync_all - %s" % dict(request.args))
ecosystem_name = request.args.get('ecosystem', None)
package_name = request.args.get('package', None)
version_id = request.args.get('version', None)
limit = request.args.get('limit', None)
offset = request.args.get('offset', None)
params = {
"ecosystem": ecosystem_name,
"package": package_name,
"version": version_id,
"limit": limit,
"offset": offset
}
current_app.logger.info("params - %s" % params)
data = data_importer.PostgresHandler().fetch_pending_epvs(**params)
try:
pending_list = data["pending_list"]
report = data_importer.import_epv_from_s3_http(list_epv=pending_list)
response = {
'message': report.get('message'),
'epv': pending_list,
'count_imported_EPVs': report.get('count_imported_EPVs')
}
if report.get('status') is not 'Success':
return flask.jsonify(response), 500
else:
return flask.jsonify(response)
except RuntimeError:
response = {'message': 'RuntimeError encountered', 'epv': pending_list}
return flask.jsonify(response), 500
def selective_ingest():
input_json = request.get_json()
if input_json.get('package_list') is None or len(input_json.get('package_list')) == 0:
return flask.jsonify(message='No Packages provided. Nothing to be ingested'), 400
expected_keys = set(['ecosystem', 'name'])
for epv in input_json.get('package_list'):
if not expected_keys.issubset(set(epv.keys())):
response = {'message': 'Invalid keys found in input: ' + ','.join(epv.keys())}
return flask.jsonify(response), 400
app.logger.info("Selective Ingestion with payload - " + json.dumps(input_json))
report = data_importer.import_epv_from_s3_http(list_epv=input_json.get('package_list'),
select_doc=input_json.get('select_ingest', None))
response = {'message': report.get('message'),
'epv': input_json,
'count_imported_EPVs': report.get('count_imported_EPVs')}
app.logger.info(response)
if report.get('status') is not 'Success':
return flask.jsonify(response), 500
else:
return flask.jsonify(response)
def selective_ingest():
"""Import e/p/v data and generate response for the POST request to /api/v1/selective."""
input_json = request.get_json()
if input_json.get('package_list') is None or len(input_json.get('package_list')) == 0:
return flask.jsonify(message='No Packages provided. Nothing to be ingested'), 400
expected_keys = set(['ecosystem', 'name'])
for epv in input_json.get('package_list'):
if not expected_keys.issubset(set(epv.keys())):
response = {'message': 'Invalid keys found in input: ' + ','.join(epv.keys())}
return flask.jsonify(response), 400
app.logger.info("Selective Ingestion with payload - " + json.dumps(input_json))
report = data_importer.import_epv_from_s3_http(list_epv=input_json.get('package_list'),
select_doc=input_json.get('select_ingest', None))
response = {'message': report.get('message'),
'epv': input_json,
'count_imported_EPVs': report.get('count_imported_EPVs')}
app.logger.info(response)
# TODO the previous code can raise a runtime exception, does not we need to handle that?
if report.get('status') is not 'Success':
return flask.jsonify(response), 500
else:
return flask.jsonify(response)
def ingest_to_graph():
input_json = request.get_json()
app.logger.info("Ingesting the given list of EPVs - " +
json.dumps(input_json))
expected_keys = set(['ecosystem', 'name', 'version'])
for epv in input_json:
if expected_keys != set(epv.keys()):
response = {
'message':
'Invalid keys found in input: ' + ','.join(epv.keys())
}
return flask.jsonify(response), 400
report = data_importer.import_epv_from_s3_http(list_epv=input_json)
response = {
'message': report.get('message'),
'epv': input_json,
'count_imported_EPVs': report.get('count_imported_EPVs')
}
print(response)
if report.get('status') is not 'Success':
return flask.jsonify(response), 500
else:
return flask.jsonify(response)
def test_insertion():
"""Test if the stored e/p/v data can be retrieved back."""
list_epv = [{
"version": "0.4.59",
"name": "access_points",
"ecosystem": "pypi"
}]
try:
report = import_epv_from_s3_http(list_epv)
logger.info(report)
assert report['status'] == "Success"
assert report["epv"] == ["pypi:access_points:0.4.59"]
assert report["count_imported_EPVs"] == 1
except Exception:
tb = traceback.format_exc()
logger.error("Traceback for latest failure in import call: %s" % tb)
def test_insertion():
"""Test if the stored e/p/v data can be retrieved back."""
list_epv = [{
"version": "0.4.59",
"name": "access_points",
"ecosystem": "pypi"
}]
try:
report = import_epv_from_s3_http(list_epv)
logger.info(report)
assert report['status'] == "Success"
# TODO Need to enable this test with new changes
# assert report["epv"] == ["pypi:access_points:0.4.59"]
assert report["count_imported_EPVs"] == 1
except Exception:
# TODO this is probably bad approach how to handle/ignore exceptions
# see https://github.com/openshiftio/openshift.io/issues/2263
tb = traceback.format_exc()
logger.error("Traceback for latest failure in import call: %s" % tb)
