def load_meta(self, tar, job_id):
"""Load the metadata from a tar file into searchable structures."""
# transaction id is the unique upload job id created by the ingest frontend
self.transaction_id = job_id
string = tar.extractfile('metadata.txt').read()
meta_list = json.loads(string)
# get the start index for the file
self.file_count = file_count(tar)
self.start_id = get_unique_id(self.file_count, 'file')
self.files = {}
# all we care about for now is the hash and the file path
file_id = self.start_id
for meta in meta_list:
if meta['destinationTable'] == 'Files':
meta['_id'] = file_id
self.files[str(file_id)] = meta
file_id += 1
trans = {}
trans['destinationTable'] = 'Transactions._id'
trans['value'] = self.transaction_id
meta_list.append(trans)
self.meta_str = json.dumps(meta_list, sort_keys=True, indent=4)
def application(environ, start_response):
"""The wsgi callback."""
info = environ['PATH_INFO']
if info and info == '/get_state':
job_id = get_job_id(environ)
record = read_state(job_id)
if record:
status, response_headers, response_body = create_state_return(record)
start_response(status, response_headers)
return [response_body]
elif info and info == '/upload':
#celery_is_alive = ping_celery()
celery_is_alive = True
if celery_is_alive:
# get temporary id from id server
job_id = get_unique_id(1, 'upload_job')
update_state(job_id, 'OK', 'UPLOADING', 0)
try:
filepath = receive(environ, job_id)
except Exception, e:
update_state(job_id, 'FAILED', 'receive bundle', 100)
if filepath != '':
start_ingest(job_id, filepath)
record = read_state(job_id)
else:
record = IngestState()
record.state = 'ERROR: Celery is dead'
record.job_id = -99
if record:
status, response_headers, response_body = create_state_return(record)
start_response(status, response_headers)
return [response_body]
def test_tasks(self):
"""Test the ingest task."""
job_id = get_unique_id(1, 'upload_job')
ingest(job_id, 'test_data/good.tar')
self.assertTrue(job_id)
def application(environ, start_response):
"""The wsgi callback."""
create_tables()
info = environ['PATH_INFO']
if info and info == '/get_state':
job_id = get_job_id(environ)
try:
record = read_state(job_id)
except peewee.DoesNotExist:
status, response_headers, response_body = create_invalid_return()
start_response(status, response_headers)
return [response_body]
if record:
status, response_headers, response_body = create_state_return(
record)
start_response(status, response_headers)
return [response_body]
elif info and info == '/upload':
# celery_is_alive = ping_celery()
celery_is_alive = True
if celery_is_alive:
# get temporary id from id server
job_id = get_unique_id(1, 'upload_job')
update_state(job_id, 'OK', 'UPLOADING', 0)
try:
filepath = receive(environ, job_id)
# pylint: disable=broad-except
except Exception as exc:
update_state(job_id, 'FAILED', 'receive bundle', 0, str(exc))
status = '500 Internal Server Error'
record = read_state(job_id)
response_body = create_state_response(record)
response_headers = [('Content-Type', 'application/json'),
('Content-Length', str(len(response_body)))
]
start_response(status, response_headers)
return [response_body]
# pylint: enable=broad-except
if filepath != '':
start_ingest(job_id, filepath)
record = read_state(job_id)
else:
record = IngestState()
record.state = 'ERROR: Celery is dead'
record.job_id = -99
if record:
status, response_headers, response_body = create_state_return(
record)
start_response(status, response_headers)
return [response_body]
else:
status, response_headers, response_body = create_invalid_return()
start_response(status, response_headers)
return [response_body]
status, response_headers, response_body = create_invalid_return()
start_response(status, response_headers)
return [response_body]
def test_tasks(self):
job_id = get_unique_id(1, 'upload_job')
ingest(job_id, 'test_data/baby.tar')