def ingest_policy_check(job_id, meta_str): """Ingest check to validate metadata at policy.""" success, exception = validate_meta(meta_str) if not success: update_state(job_id, 'FAILED', 'Policy Validation', 0, exception) raise IngestException() update_state(job_id, 'OK', 'Policy Validation', 100)
def test_update_state(self): """Test return and update of unique index.""" return with test_database(TEST_DB, (BaseModel, IngestState)): test_object = IngestState.create(job_id=999, state='ERROR', task='unbundling', task_percent=42.3) self.assertEqual(test_object.job_id, 999) update_state(999, 'WORKING', 'validating', 33.2) record = read_state(999) self.assertEqual(record.state, 'WORKING') self.assertEqual(record.task, 'validating')
def test_update_state(self): """Test return and update of unique index.""" with test_database(TEST_DB, (BaseModel, IngestState)): test_object = IngestState.create(job_id=999, state='ERROR', task='unbundling', task_percent=42.3) self.assertEqual(test_object.job_id, 999) IngestState.database_close() update_state(999, 'WORKING', 'validating', 33.2) record = read_state(999) self.assertEqual(record.state, 'WORKING') self.assertEqual(record.task, 'validating') self.assertEqual(float(record.task_percent), 33.2) record = read_state(None) self.assertEqual(record.state, 'DATA_ACCESS_ERROR') self.assertEqual(record.task, 'read_state') self.assertEqual(record.task_percent, 0)
def application(environ, start_response): """The wsgi callback.""" info = environ['PATH_INFO'] if info and info == '/get_state': job_id = get_job_id(environ) record = read_state(job_id) if record: status, response_headers, response_body = create_state_return(record) start_response(status, response_headers) return [response_body] elif info and info == '/upload': #celery_is_alive = ping_celery() celery_is_alive = True if celery_is_alive: # get temporary id from id server job_id = get_unique_id(1, 'upload_job') update_state(job_id, 'OK', 'UPLOADING', 0) try: filepath = receive(environ, job_id) except Exception, e: update_state(job_id, 'FAILED', 'receive bundle', 100) if filepath != '': start_ingest(job_id, filepath) record = read_state(job_id) else: record = IngestState() record.state = 'ERROR: Celery is dead' record.job_id = -99 if record: status, response_headers, response_body = create_state_return(record) start_response(status, response_headers) return [response_body]
def ingest_metadata(job_id, meta): """Ingest metadata to the metadata service.""" update_state(job_id, 'OK', 'ingest metadata', 0) success, exception = meta.post_metadata() if not success: # rollback files update_state(job_id, 'FAILED', 'ingest metadata', 0, str(exception)) raise IngestException() update_state(job_id, 'OK', 'ingest metadata', 100)
def ingest_check_tarfile(job_id, filepath): """Check the ingest tarfile and return state or set it properly.""" update_state(job_id, 'OK', 'Open tar', 0) tar = open_tar(filepath) if tar is None: update_state(job_id, 'FAILED', 'Bad tarfile', 0) raise IngestException() update_state(job_id, 'OK', 'Open tar', 100) return tar
def ingest_metadata_parser(job_id, tar): """Ingest the metadata and set the state appropriately.""" update_state(job_id, 'OK', 'load metadata', 0) meta = MetaParser() try: meta.load_meta(tar, job_id) # pylint: disable=broad-except except Exception as ex: update_state(job_id, 'FAILED', 'load metadata', 0, str(ex)) raise IngestException() update_state(job_id, 'OK', 'load metadata', 100) return meta
def ingest_files(job_id, ingest_obj): """Ingest the files to the archive interface.""" update_state(job_id, 'OK', 'ingest files', 0) try: ingest_obj.ingest() # pylint: disable=broad-except except Exception as ex: # rollback files stack_dump = traceback.format_exc() update_state(job_id, 'FAILED', 'ingest files', 0, u'{}\n{}'.format(stack_dump, str(ex))) raise IngestException() update_state(job_id, 'OK', 'ingest files', 100)
def ingest(job_id, filepath): """Ingest a tar bundle into the archive.""" update_state(job_id, 'OK', 'Open tar', 0) tar = open_tar(filepath) update_state(job_id, 'OK', 'Open tar', 100) update_state(job_id, 'OK', 'load metadata', 0) meta = MetaParser() meta.load_meta(tar) update_state(job_id, 'OK', 'load metadata', 100) ingest = TarIngester(tar, meta) # validate policy success = validate_meta (meta.meta_str) if not success: update_state(job_id, 'FAILED', 'Policy Validation', 0) return update_state(job_id, 'OK', 'Policy Validation', 100) update_state(job_id, 'OK', 'ingest files', 0) success = ingest.ingest() if not success: #rollback files update_state(job_id, 'FAILED', 'ingest files', 0) return update_state(job_id, 'OK', 'ingest files', 100) update_state(job_id, 'OK', 'ingest metadata', 0) success = meta.post_metadata() if not success: #rollback files update_state(job_id, 'FAILED', 'ingest metadata', 0) return update_state(job_id, 'OK', 'ingest metadata', 100)
def application(environ, start_response): """The wsgi callback.""" create_tables() info = environ['PATH_INFO'] if info and info == '/get_state': job_id = get_job_id(environ) try: record = read_state(job_id) except peewee.DoesNotExist: status, response_headers, response_body = create_invalid_return() start_response(status, response_headers) return [response_body] if record: status, response_headers, response_body = create_state_return( record) start_response(status, response_headers) return [response_body] elif info and info == '/upload': # celery_is_alive = ping_celery() celery_is_alive = True if celery_is_alive: # get temporary id from id server job_id = get_unique_id(1, 'upload_job') update_state(job_id, 'OK', 'UPLOADING', 0) try: filepath = receive(environ, job_id) # pylint: disable=broad-except except Exception as exc: update_state(job_id, 'FAILED', 'receive bundle', 0, str(exc)) status = '500 Internal Server Error' record = read_state(job_id) response_body = create_state_response(record) response_headers = [('Content-Type', 'application/json'), ('Content-Length', str(len(response_body))) ] start_response(status, response_headers) return [response_body] # pylint: enable=broad-except if filepath != '': start_ingest(job_id, filepath) record = read_state(job_id) else: record = IngestState() record.state = 'ERROR: Celery is dead' record.job_id = -99 if record: status, response_headers, response_body = create_state_return( record) start_response(status, response_headers) return [response_body] else: status, response_headers, response_body = create_invalid_return() start_response(status, response_headers) return [response_body] status, response_headers, response_body = create_invalid_return() start_response(status, response_headers) return [response_body]