def main():
"""Entry point for main index server."""
peewee_logger = logging.getLogger('peewee')
peewee_logger.setLevel(logging.DEBUG)
peewee_logger.addHandler(logging.StreamHandler())
main_logger = logging.getLogger('index_server')
main_logger.setLevel(logging.DEBUG)
main_logger.addHandler(logging.StreamHandler())
msg = {'msg': os.getenv('MYSQL_ENV_MYSQL_DATABASE', 'pacifica_ingest')}
main_logger.info('MYSQL_ENV_MYSQL_DATABASE = %(msg)s', msg)
msg['msg'] = os.getenv('MYSQL_PORT_3306_TCP_ADDR', '127.0.0.1')
main_logger.info('MYSQL_PORT_3306_TCP_ADDR = %(msg)s', msg)
msg['msg'] = os.getenv('MYSQL_PORT_3306_TCP_PORT', '3306')
main_logger.info('MYSQL_PORT_3306_TCP_PORT = %(msg)s', msg)
msg['msg'] = os.getenv('MYSQL_ENV_MYSQL_USER', 'ingest')
main_logger.info('MYSQL_ENV_MYSQL_USER = %(msg)s', msg)
main_logger.info('MYSQL_ENV_MYSQL_PASSWORD = %(msg)s', msg)
msg['msg'] = os.getenv('MYSQL_ENV_MYSQL_PASSWORD', 'ingest')
if not IngestState.table_exists():
IngestState.create_table()
httpd = make_server('0.0.0.0', 8066, application)
httpd.serve_forever()
def test_job_subcommand(self):
"""Test the job subcommand for updating job state."""
rwfd, fname = mkstemp()
os.close(rwfd)
with test_database(SqliteDatabase(fname), (BaseModel, IngestState)):
IngestState.create(job_id=999,
state='ERROR',
task='unbundling',
task_percent=42.3)
IngestState.database_close()
cmd([
'job', '--task=unbundling', '--job-id=999', '--state=OK',
'--task-percent=100.0', '--exception=Badness'
])
record = read_state(999)
self.assertEqual(record.state, 'OK')
self.assertEqual(record.task_percent, 100.0)
def test_update_state(self):
"""Test return and update of unique index."""
with test_database(TEST_DB, (BaseModel, IngestState)):
test_object = IngestState.create(job_id=999,
state='ERROR',
task='unbundling',
task_percent=42.3)
self.assertEqual(test_object.job_id, 999)
IngestState.database_close()
update_state(999, 'WORKING', 'validating', 33.2)
record = read_state(999)
self.assertEqual(record.state, 'WORKING')
self.assertEqual(record.task, 'validating')
self.assertEqual(float(record.task_percent), 33.2)
record = read_state(None)
self.assertEqual(record.state, 'DATA_ACCESS_ERROR')
self.assertEqual(record.task, 'read_state')
self.assertEqual(record.task_percent, 0)
def test_update_state(self):
"""Test return and update of unique index."""
return
with test_database(TEST_DB, (BaseModel, IngestState)):
test_object = IngestState.create(job_id=999, state='ERROR', task='unbundling',
task_percent=42.3)
self.assertEqual(test_object.job_id, 999)
update_state(999, 'WORKING', 'validating', 33.2)
record = read_state(999)
self.assertEqual(record.state, 'WORKING')
self.assertEqual(record.task, 'validating')
def application(environ, start_response):
"""The wsgi callback."""
info = environ['PATH_INFO']
if info and info == '/get_state':
job_id = get_job_id(environ)
record = read_state(job_id)
if record:
status, response_headers, response_body = create_state_return(record)
start_response(status, response_headers)
return [response_body]
elif info and info == '/upload':
#celery_is_alive = ping_celery()
celery_is_alive = True
if celery_is_alive:
# get temporary id from id server
job_id = get_unique_id(1, 'upload_job')
update_state(job_id, 'OK', 'UPLOADING', 0)
try:
filepath = receive(environ, job_id)
except Exception, e:
update_state(job_id, 'FAILED', 'receive bundle', 100)
if filepath != '':
start_ingest(job_id, filepath)
record = read_state(job_id)
else:
record = IngestState()
record.state = 'ERROR: Celery is dead'
record.job_id = -99
if record:
status, response_headers, response_body = create_state_return(record)
start_response(status, response_headers)
return [response_body]
def application(environ, start_response):
"""The wsgi callback."""
create_tables()
info = environ['PATH_INFO']
if info and info == '/get_state':
job_id = get_job_id(environ)
try:
record = read_state(job_id)
except peewee.DoesNotExist:
status, response_headers, response_body = create_invalid_return()
start_response(status, response_headers)
return [response_body]
if record:
status, response_headers, response_body = create_state_return(
record)
start_response(status, response_headers)
return [response_body]
elif info and info == '/upload':
# celery_is_alive = ping_celery()
celery_is_alive = True
if celery_is_alive:
# get temporary id from id server
job_id = get_unique_id(1, 'upload_job')
update_state(job_id, 'OK', 'UPLOADING', 0)
try:
filepath = receive(environ, job_id)
# pylint: disable=broad-except
except Exception as exc:
update_state(job_id, 'FAILED', 'receive bundle', 0, str(exc))
status = '500 Internal Server Error'
record = read_state(job_id)
response_body = create_state_response(record)
response_headers = [('Content-Type', 'application/json'),
('Content-Length', str(len(response_body)))
]
start_response(status, response_headers)
return [response_body]
# pylint: enable=broad-except
if filepath != '':
start_ingest(job_id, filepath)
record = read_state(job_id)
else:
record = IngestState()
record.state = 'ERROR: Celery is dead'
record.job_id = -99
if record:
status, response_headers, response_body = create_state_return(
record)
start_response(status, response_headers)
return [response_body]
else:
status, response_headers, response_body = create_invalid_return()
start_response(status, response_headers)
return [response_body]
status, response_headers, response_body = create_invalid_return()
start_response(status, response_headers)
return [response_body]