def load_meta(self, tar, job_id): """Load the metadata from a tar file into searchable structures.""" # transaction id is the unique upload job id created by the ingest frontend self.transaction_id = job_id string = tar.extractfile('metadata.txt').read() meta_list = json.loads(string) # get the start index for the file self.file_count = file_count(tar) self.start_id = get_unique_id(self.file_count, 'file') self.files = {} # all we care about for now is the hash and the file path file_id = self.start_id for meta in meta_list: if meta['destinationTable'] == 'Files': meta['_id'] = file_id self.files[str(file_id)] = meta file_id += 1 trans = {} trans['destinationTable'] = 'Transactions._id' trans['value'] = self.transaction_id meta_list.append(trans) self.meta_str = json.dumps(meta_list, sort_keys=True, indent=4)
def application(environ, start_response): """The wsgi callback.""" info = environ['PATH_INFO'] if info and info == '/get_state': job_id = get_job_id(environ) record = read_state(job_id) if record: status, response_headers, response_body = create_state_return(record) start_response(status, response_headers) return [response_body] elif info and info == '/upload': #celery_is_alive = ping_celery() celery_is_alive = True if celery_is_alive: # get temporary id from id server job_id = get_unique_id(1, 'upload_job') update_state(job_id, 'OK', 'UPLOADING', 0) try: filepath = receive(environ, job_id) except Exception, e: update_state(job_id, 'FAILED', 'receive bundle', 100) if filepath != '': start_ingest(job_id, filepath) record = read_state(job_id) else: record = IngestState() record.state = 'ERROR: Celery is dead' record.job_id = -99 if record: status, response_headers, response_body = create_state_return(record) start_response(status, response_headers) return [response_body]
def test_tasks(self): """Test the ingest task.""" job_id = get_unique_id(1, 'upload_job') ingest(job_id, 'test_data/good.tar') self.assertTrue(job_id)
def application(environ, start_response): """The wsgi callback.""" create_tables() info = environ['PATH_INFO'] if info and info == '/get_state': job_id = get_job_id(environ) try: record = read_state(job_id) except peewee.DoesNotExist: status, response_headers, response_body = create_invalid_return() start_response(status, response_headers) return [response_body] if record: status, response_headers, response_body = create_state_return( record) start_response(status, response_headers) return [response_body] elif info and info == '/upload': # celery_is_alive = ping_celery() celery_is_alive = True if celery_is_alive: # get temporary id from id server job_id = get_unique_id(1, 'upload_job') update_state(job_id, 'OK', 'UPLOADING', 0) try: filepath = receive(environ, job_id) # pylint: disable=broad-except except Exception as exc: update_state(job_id, 'FAILED', 'receive bundle', 0, str(exc)) status = '500 Internal Server Error' record = read_state(job_id) response_body = create_state_response(record) response_headers = [('Content-Type', 'application/json'), ('Content-Length', str(len(response_body))) ] start_response(status, response_headers) return [response_body] # pylint: enable=broad-except if filepath != '': start_ingest(job_id, filepath) record = read_state(job_id) else: record = IngestState() record.state = 'ERROR: Celery is dead' record.job_id = -99 if record: status, response_headers, response_body = create_state_return( record) start_response(status, response_headers) return [response_body] else: status, response_headers, response_body = create_invalid_return() start_response(status, response_headers) return [response_body] status, response_headers, response_body = create_invalid_return() start_response(status, response_headers) return [response_body]
def test_tasks(self): job_id = get_unique_id(1, 'upload_job') ingest(job_id, 'test_data/baby.tar')