def ingest_policy_check(job_id, meta_str):
"""Ingest check to validate metadata at policy."""
success, exception = validate_meta(meta_str)
if not success:
update_state(job_id, 'FAILED', 'Policy Validation', 0, exception)
raise IngestException()
update_state(job_id, 'OK', 'Policy Validation', 100)
def test_update_state(self):
"""Test return and update of unique index."""
return
with test_database(TEST_DB, (BaseModel, IngestState)):
test_object = IngestState.create(job_id=999, state='ERROR', task='unbundling',
task_percent=42.3)
self.assertEqual(test_object.job_id, 999)
update_state(999, 'WORKING', 'validating', 33.2)
record = read_state(999)
self.assertEqual(record.state, 'WORKING')
self.assertEqual(record.task, 'validating')
def test_update_state(self):
"""Test return and update of unique index."""
with test_database(TEST_DB, (BaseModel, IngestState)):
test_object = IngestState.create(job_id=999,
state='ERROR',
task='unbundling',
task_percent=42.3)
self.assertEqual(test_object.job_id, 999)
IngestState.database_close()
update_state(999, 'WORKING', 'validating', 33.2)
record = read_state(999)
self.assertEqual(record.state, 'WORKING')
self.assertEqual(record.task, 'validating')
self.assertEqual(float(record.task_percent), 33.2)
record = read_state(None)
self.assertEqual(record.state, 'DATA_ACCESS_ERROR')
self.assertEqual(record.task, 'read_state')
self.assertEqual(record.task_percent, 0)
def application(environ, start_response):
"""The wsgi callback."""
info = environ['PATH_INFO']
if info and info == '/get_state':
job_id = get_job_id(environ)
record = read_state(job_id)
if record:
status, response_headers, response_body = create_state_return(record)
start_response(status, response_headers)
return [response_body]
elif info and info == '/upload':
#celery_is_alive = ping_celery()
celery_is_alive = True
if celery_is_alive:
# get temporary id from id server
job_id = get_unique_id(1, 'upload_job')
update_state(job_id, 'OK', 'UPLOADING', 0)
try:
filepath = receive(environ, job_id)
except Exception, e:
update_state(job_id, 'FAILED', 'receive bundle', 100)
if filepath != '':
start_ingest(job_id, filepath)
record = read_state(job_id)
else:
record = IngestState()
record.state = 'ERROR: Celery is dead'
record.job_id = -99
if record:
status, response_headers, response_body = create_state_return(record)
start_response(status, response_headers)
return [response_body]
def ingest_metadata(job_id, meta):
"""Ingest metadata to the metadata service."""
update_state(job_id, 'OK', 'ingest metadata', 0)
success, exception = meta.post_metadata()
if not success:
# rollback files
update_state(job_id, 'FAILED', 'ingest metadata', 0, str(exception))
raise IngestException()
update_state(job_id, 'OK', 'ingest metadata', 100)
def ingest_check_tarfile(job_id, filepath):
"""Check the ingest tarfile and return state or set it properly."""
update_state(job_id, 'OK', 'Open tar', 0)
tar = open_tar(filepath)
if tar is None:
update_state(job_id, 'FAILED', 'Bad tarfile', 0)
raise IngestException()
update_state(job_id, 'OK', 'Open tar', 100)
return tar
def ingest_metadata_parser(job_id, tar):
"""Ingest the metadata and set the state appropriately."""
update_state(job_id, 'OK', 'load metadata', 0)
meta = MetaParser()
try:
meta.load_meta(tar, job_id)
# pylint: disable=broad-except
except Exception as ex:
update_state(job_id, 'FAILED', 'load metadata', 0, str(ex))
raise IngestException()
update_state(job_id, 'OK', 'load metadata', 100)
return meta
def ingest_files(job_id, ingest_obj):
"""Ingest the files to the archive interface."""
update_state(job_id, 'OK', 'ingest files', 0)
try:
ingest_obj.ingest()
# pylint: disable=broad-except
except Exception as ex:
# rollback files
stack_dump = traceback.format_exc()
update_state(job_id, 'FAILED', 'ingest files', 0,
u'{}\n{}'.format(stack_dump, str(ex)))
raise IngestException()
update_state(job_id, 'OK', 'ingest files', 100)
def ingest(job_id, filepath):
"""Ingest a tar bundle into the archive."""
update_state(job_id, 'OK', 'Open tar', 0)
tar = open_tar(filepath)
update_state(job_id, 'OK', 'Open tar', 100)
update_state(job_id, 'OK', 'load metadata', 0)
meta = MetaParser()
meta.load_meta(tar)
update_state(job_id, 'OK', 'load metadata', 100)
ingest = TarIngester(tar, meta)
# validate policy
success = validate_meta (meta.meta_str)
if not success:
update_state(job_id, 'FAILED', 'Policy Validation', 0)
return
update_state(job_id, 'OK', 'Policy Validation', 100)
update_state(job_id, 'OK', 'ingest files', 0)
success = ingest.ingest()
if not success:
#rollback files
update_state(job_id, 'FAILED', 'ingest files', 0)
return
update_state(job_id, 'OK', 'ingest files', 100)
update_state(job_id, 'OK', 'ingest metadata', 0)
success = meta.post_metadata()
if not success:
#rollback files
update_state(job_id, 'FAILED', 'ingest metadata', 0)
return
update_state(job_id, 'OK', 'ingest metadata', 100)
def application(environ, start_response):
"""The wsgi callback."""
create_tables()
info = environ['PATH_INFO']
if info and info == '/get_state':
job_id = get_job_id(environ)
try:
record = read_state(job_id)
except peewee.DoesNotExist:
status, response_headers, response_body = create_invalid_return()
start_response(status, response_headers)
return [response_body]
if record:
status, response_headers, response_body = create_state_return(
record)
start_response(status, response_headers)
return [response_body]
elif info and info == '/upload':
# celery_is_alive = ping_celery()
celery_is_alive = True
if celery_is_alive:
# get temporary id from id server
job_id = get_unique_id(1, 'upload_job')
update_state(job_id, 'OK', 'UPLOADING', 0)
try:
filepath = receive(environ, job_id)
# pylint: disable=broad-except
except Exception as exc:
update_state(job_id, 'FAILED', 'receive bundle', 0, str(exc))
status = '500 Internal Server Error'
record = read_state(job_id)
response_body = create_state_response(record)
response_headers = [('Content-Type', 'application/json'),
('Content-Length', str(len(response_body)))
]
start_response(status, response_headers)
return [response_body]
# pylint: enable=broad-except
if filepath != '':
start_ingest(job_id, filepath)
record = read_state(job_id)
else:
record = IngestState()
record.state = 'ERROR: Celery is dead'
record.job_id = -99
if record:
status, response_headers, response_body = create_state_return(
record)
start_response(status, response_headers)
return [response_body]
else:
status, response_headers, response_body = create_invalid_return()
start_response(status, response_headers)
return [response_body]
status, response_headers, response_body = create_invalid_return()
start_response(status, response_headers)
return [response_body]