async def store_buckets(request):
PATH = '/bucket'
async with request.app['db'].acquire() as conn:
service_config = await get_service_config_by_action(conn=conn, action='store', media_type='default')
config = service_config.get('config')
endpoint = service_config.get('endpoint')
if request.method == 'GET':
try:
data = request.query
config.update({'bucket_name': data.get('bucket_name')})
fd = FormData()
fd.add_field('config', json.dumps(config), content_type='application/json')
async with new_request(method='GET', url=endpoint+PATH, data=fd) as resp:
try:
resp_json = await resp.json()
except Exception as err:
return web.json_response({'err': str(err), 'resp': await resp.text()})
return web.json_response({'resp': resp_json})
except Exception as err:
return web.json_response({'origin': 'gateway', 'err': str(err)})
if request.method == 'POST':
try:
data = await request.json()
fd = FormData()
fd.add_field('config', json.dumps(config), content_type='application/json')
fd.add_field('data', json.dumps(data), content_type='application/json')
# payload = dict({'config': config, 'data': data})
async with new_request(method='POST', url=endpoint+PATH, data=fd) as resp:
resp_json = await resp.json()
return web.json_response({'resp': resp_json})
except Exception as err:
return web.json_response({'origin': 'gateway', 'err': str(err)})
async def trigger_publish(conn, data):
media_type = data.get('media_type')
service_config = await get_service_config_by_action(conn=conn,
action='publish',
media_type=media_type)
logging.debug(msg='service_config: {}'.format(str(service_config)))
service_name = service_config.get('name')
did = data.get('id')
form = data.get('form')
metadata = extract_data_from_form(form)
data = {}
data.update({'metadata': metadata})
fd = FormData()
fd.add_field('data', json.dumps(data), content_type='application/json')
service_config = await db.get_service_config(conn=conn, name=service_name)
if service_config:
endpoint = service_config.get('endpoint')
config = service_config.get('config')
fd.add_field('config',
json.dumps(config),
content_type='application/json')
with open(TMP_FILE_LOCATION.format(did), 'rb') as f:
fd.add_field('file',
f,
filename=did,
content_type='application/octet-stream')
async with new_request(method='POST', url=endpoint,
data=fd) as resp:
resp_json = await resp.json()
location = resp_json.get('uri')
return location
async def trigger_store(conn, did):
PATH = '/object'
service_config = await get_service_config_by_action(conn=conn,
action='store',
media_type='default')
config = dict(service_config.get('config'))
endpoint = service_config.get('endpoint')
bucket_name = dict({'bucket_name': '3deposit'})
config.update(bucket_name)
with open(TMP_FILE_LOCATION + did, 'rb') as f:
try:
deposit_id = dict({'deposit_id': did})
fd = FormData()
fd.add_field('file',
f,
filename=did,
content_type='application/octet-stream')
fd.add_field('config',
json.dumps(config),
content_type='application/json')
fd.add_field('data',
json.dumps(deposit_id),
content_type='application/json')
async with new_request(method='POST', url=endpoint + PATH,
data=fd) as resp:
resp_json = await resp.json()
etag = resp_json.get('etag')
return etag
except Exception as err:
return web.json_response({'origin': 'gateway', 'err': str(err)})
async def trigger_metadata(data):
did = data.get('id')
form = data.get('form')
media_type = data.get('media_type')
form_data = extract_data_from_form(form)
deposit_date = data.get('deposit_date')
form_data.update({'deposit_date': deposit_date})
deposit_metadata = dict({'deposit_metadata': form_data})
deposit_id = dict({'deposit_id': did})
data = {}
data.update({'deposit_date': deposit_date})
data.update(deposit_id)
data.update(deposit_metadata)
data.update({'media_type': media_type})
fd = FormData()
fd.add_field('data', json.dumps(data), content_type='application/json')
with open(TMP_FILE_LOCATION + did, 'rb') as f:
fd.add_field('file',
f,
filename=did,
content_type='application/octet-stream')
async with new_request(method='POST',
url='http://metadata-service:5000/',
data=fd) as resp:
if resp.content_type == 'application/json':
resp_json = await resp.json()
else:
resp_json = await resp.text()
logging.debug(
f'trigger_metadata, metadata-service returned: {str(resp_json)}'
)
technical_metadata = resp_json.get('technical_metadata')
if technical_metadata:
logging.debug(f'technical_metadata: {technical_metadata}')
data.update({'technical_metadata': technical_metadata})
fd = FormData()
fd.add_field('data', json.dumps(data), content_type='application/json')
async with new_request(method='POST',
url='http://mongo-service:5000/objects',
data=fd) as resp:
resp_json = await resp.json()
mongo_id = resp_json.get('mongo_id')
return mongo_id
async def store_objects(request):
PATH = '/object'
async with request.app['db'].acquire() as conn:
service_config = await get_service_config_by_action(conn=conn, action='store', media_type='default')
logging.debug(msg='store_objects service_config: {}'.format(str(service_config)))
config = dict(service_config.get('config'))
endpoint = service_config.get('endpoint')
bucket_name = dict({'bucket_name': '3deposit'})
config.update(bucket_name)
if request.method == 'GET':
try:
data = request.query
config.update({'bucket_name': data.get('bucket_name')})
payload = dict({'config': config})
async with new_request(method='GET', url=endpoint+PATH, json=payload) as resp:
try:
resp_json = await resp.json()
except Exception as err:
return web.json_response({'err': str(err), 'resp': await resp.text()})
return web.json_response({'resp': resp_json, 'payload': payload})
except Exception as err:
return web.json_response({'origin': 'gateway', 'err': str(err)})
if request.method == 'POST':
try:
q = dict(request.query)
fd = FormData()
reader = await request.multipart()
while True:
part = await reader.next()
if part is None:
break
if part.name == 'file':
fd.add_field(name='file', value=await part.read(), filename=q.get('deposit_id'), content_type='application/octet-stream')
else:
continue
fd.add_field('config', json.dumps(config), content_type='application/json')
fd.add_field('data', json.dumps(q), content_type='application/json')
async with new_request(method='POST', url=endpoint+PATH, data=fd) as resp:
resp_json = await resp.json()
return web.json_response(resp_json)
except Exception as err:
return web.json_response({'origin': 'gateway', 'err': str(err)})
async def trigger_metadata(data):
did = data.get('id')
form = data.get('form')
form_data = extract_data_from_form(form)
deposit_metadata = dict({'deposit_metadata': form_data})
deposit_id = dict({'deposit_id': did})
data = {}
data.update(deposit_id)
data.update(deposit_metadata)
fd = FormData()
fd.add_field('data', json.dumps(data), content_type='application/json')
async with new_request(method='POST', url='http://mongo-service:5000/objects', data=fd) as resp:
resp_json = await resp.json()
mongo_id = resp_json.get('mongo_id')
return mongo_id
async def publish_models(request):
PATH = '/models'
async with request.app['db'].acquire() as conn:
service_config = await get_service_config_by_action(conn=conn,
action='publish',
media_type='model')
logging.debug(msg='service_config: {}'.format(str(service_config)))
service_name = service_config.get('name')
if request.method == 'POST':
try:
q = request.query
did = q.get('deposit_id')
metadata = await request.json()
data = {}
data.update({'metadata': metadata})
fd = FormData()
fd.add_field('data',
json.dumps(data),
content_type='application/json')
async with request.app['db'].acquire() as conn:
service_config = await db.get_service_config(conn=conn,
name=service_name)
if service_config:
endpoint = service_config.get('endpoint')
config = service_config.get('config')
fd.add_field('config',
json.dumps(config),
content_type='application/json')
with open('./data/{}'.format(did), 'rb') as f:
fd.add_field('file',
f,
filename=did,
content_type='application/octet-stream')
async with new_request(method='POST',
url=endpoint + PATH,
data=fd) as resp:
resp_json = await resp.json()
uri = resp_json.get('uri')
return web.json_response(uri)
else:
return web.json_response({
'err':
'could not retrieve config for service: {}'.format(
service_name)
})
except Exception as err:
return web.json_response({'err': str(err)})
async def metadata(request):
headers = {
'ACCESS-CONTROL-ALLOW-ORIGIN': '*',
'Access-Control-Allow-Headers': 'content-type'
}
if request.method == 'GET':
try:
q = request.query
fd = FormData()
config = dict({"deposit_id": q.get('deposit_id')})
fd.add_field('config', json.dumps(config), content_type='application/json')
async with new_request(method='GET', url='http://mongo-service:5000/objects', data=fd) as resp:
logging.debug("MONGO ERROR:"+await resp.text())
resp_json = await resp.json()
return web.json_response(resp_json, headers=headers)
except Exception as err:
return web.json_response({ 'err': str(err) }, headers=headers)
async def publications(request):
headers = {
'ACCESS-CONTROL-ALLOW-ORIGIN': '*',
'Access-Control-Allow-Headers': 'content-type'
}
# PATH = '/models'
try:
if request.method == 'GET':
q = request.query
data = {
'resource_id': q.get('resource_id')
}
media_type = q.get('media_type')
async with request.app['db'].acquire() as conn:
service_config = await get_service_config_by_action(conn=conn, action='publish', media_type=media_type)
logging.debug(msg='service_config: {}'.format(str(service_config)))
endpoint = service_config.get('endpoint')
config = service_config.get('config')
payload = {}
payload.update({'data': json.dumps(data)})
payload.update({'config': json.dumps(config)})
logging.debug(msg="PAYLOAD LOG: "+str(payload))
async with new_request(method='GET', url=endpoint, data=payload) as resp:
logging.debug(await resp.text())
return web.json_response(await resp.json(), headers=headers)
else:
return web.Response(status=200, headers=headers)
except Exception as err:
return web.json_response({"err": str(err)}, headers=headers)
async def start_deposit_processing_task(data):
logging.debug(f'start_deposit_processing_task with data: {str(data)}')
try:
deposit_id = data.get('id')
if deposit_id:
logging.info(msg='start_deposit_processing_task' + str(data))
assemble_chunks = create_task(
start_reassembling_chunks(deposit_id))
chunks_assembled = await assemble_chunks
if chunks_assembled:
engine = await get_service_engine()
media_type = data.get('media_type')
async with engine.acquire() as conn:
# insert deposit_id
await db.add_deposit_by_id(conn, deposit_id)
await db.update_deposit_by_id(
conn,
deposit_id=deposit_id,
media_type=media_type,
deposit_date=data.get('deposit_date'))
# update with etag
etag = await trigger_store(conn, deposit_id)
await db.update_deposit_by_id(conn,
deposit_id=deposit_id,
etag=etag)
# update with mongo_id
mongo_id = await trigger_metadata(data)
await db.update_deposit_by_id(conn,
deposit_id=deposit_id,
mongo_id=mongo_id)
# update with resource_id and location
publish_resp = await trigger_publish(conn, data)
await db.update_deposit_by_id(
conn,
deposit_id=deposit_id,
resource_id=publish_resp.get('resource_id'))
# add new fields to mongo doc
data = {
'etag': etag,
'location': publish_resp.get('location'),
'resource_id': publish_resp.get('resource_id')
}
config = {'deposit_id': deposit_id}
fd = FormData()
fd.add_field('data',
json.dumps(data),
content_type='application/json')
fd.add_field('config',
json.dumps(config),
content_type='application/json')
async with new_request(
method='PATCH',
url='http://mongo-service:5000/objects',
data=fd) as resp:
resp_json = await resp.json()
if resp_json.get('err'):
logging.error(resp_json.get('err'))
else:
logging.info(resp_json.get('log'))
if os.path.exists(TMP_FILE_LOCATION + deposit_id):
os.remove(TMP_FILE_LOCATION + deposit_id)
return True
else:
logging.error('err: no deposit id')
return False
except Exception as err:
logging.error(msg=f'start_deposit_processing_task err: {str(err)}')
return False