Example #1
0
def main():
    """Entry point for main index server."""
    peewee_logger = logging.getLogger('peewee')
    peewee_logger.setLevel(logging.DEBUG)
    peewee_logger.addHandler(logging.StreamHandler())

    main_logger = logging.getLogger('index_server')
    main_logger.setLevel(logging.DEBUG)
    main_logger.addHandler(logging.StreamHandler())

    msg = {'msg': os.getenv('MYSQL_ENV_MYSQL_DATABASE', 'pacifica_ingest')}
    main_logger.info('MYSQL_ENV_MYSQL_DATABASE = %(msg)s', msg)
    msg['msg'] = os.getenv('MYSQL_PORT_3306_TCP_ADDR', '127.0.0.1')
    main_logger.info('MYSQL_PORT_3306_TCP_ADDR = %(msg)s', msg)
    msg['msg'] = os.getenv('MYSQL_PORT_3306_TCP_PORT', '3306')
    main_logger.info('MYSQL_PORT_3306_TCP_PORT = %(msg)s', msg)
    msg['msg'] = os.getenv('MYSQL_ENV_MYSQL_USER', 'ingest')
    main_logger.info('MYSQL_ENV_MYSQL_USER = %(msg)s', msg)
    main_logger.info('MYSQL_ENV_MYSQL_PASSWORD = %(msg)s', msg)
    msg['msg'] = os.getenv('MYSQL_ENV_MYSQL_PASSWORD', 'ingest')

    if not IngestState.table_exists():
        IngestState.create_table()

    httpd = make_server('0.0.0.0', 8066, application)
    httpd.serve_forever()
Example #2
0
 def test_job_subcommand(self):
     """Test the job subcommand for updating job state."""
     rwfd, fname = mkstemp()
     os.close(rwfd)
     with test_database(SqliteDatabase(fname), (BaseModel, IngestState)):
         IngestState.create(job_id=999,
                            state='ERROR',
                            task='unbundling',
                            task_percent=42.3)
         IngestState.database_close()
         cmd([
             'job', '--task=unbundling', '--job-id=999', '--state=OK',
             '--task-percent=100.0', '--exception=Badness'
         ])
         record = read_state(999)
         self.assertEqual(record.state, 'OK')
         self.assertEqual(record.task_percent, 100.0)
Example #3
0
 def test_update_state(self):
     """Test return and update of unique index."""
     with test_database(TEST_DB, (BaseModel, IngestState)):
         test_object = IngestState.create(job_id=999,
                                          state='ERROR',
                                          task='unbundling',
                                          task_percent=42.3)
         self.assertEqual(test_object.job_id, 999)
         IngestState.database_close()
         update_state(999, 'WORKING', 'validating', 33.2)
         record = read_state(999)
         self.assertEqual(record.state, 'WORKING')
         self.assertEqual(record.task, 'validating')
         self.assertEqual(float(record.task_percent), 33.2)
         record = read_state(None)
         self.assertEqual(record.state, 'DATA_ACCESS_ERROR')
         self.assertEqual(record.task, 'read_state')
         self.assertEqual(record.task_percent, 0)
Example #4
0
 def test_update_state(self):
     """Test return and update of unique index."""
     return
     with test_database(TEST_DB, (BaseModel, IngestState)):
         test_object = IngestState.create(job_id=999, state='ERROR', task='unbundling',
                                          task_percent=42.3)
         self.assertEqual(test_object.job_id, 999)
         update_state(999, 'WORKING', 'validating', 33.2)
         record = read_state(999)
         self.assertEqual(record.state, 'WORKING')
         self.assertEqual(record.task, 'validating')
Example #5
0
def application(environ, start_response):
    """The wsgi callback."""
    info = environ['PATH_INFO']
    if info and info == '/get_state':
        job_id = get_job_id(environ)
        record = read_state(job_id)
        if record:
            status, response_headers, response_body = create_state_return(record)
            start_response(status, response_headers)
            return [response_body]
    elif info and info == '/upload':

        #celery_is_alive = ping_celery()

        celery_is_alive = True
        if celery_is_alive:
            # get temporary id from id server
            job_id = get_unique_id(1, 'upload_job')
            update_state(job_id, 'OK', 'UPLOADING', 0)

            try:
                filepath = receive(environ, job_id)
            except Exception, e:                
                update_state(job_id, 'FAILED', 'receive bundle', 100)

            if filepath != '':
                start_ingest(job_id, filepath)

            record = read_state(job_id)

        else:
            record = IngestState()
            record.state = 'ERROR: Celery is dead'
            record.job_id = -99

        if record:
            status, response_headers, response_body = create_state_return(record)
            start_response(status, response_headers)
            return [response_body]
Example #6
0
def application(environ, start_response):
    """The wsgi callback."""
    create_tables()
    info = environ['PATH_INFO']
    if info and info == '/get_state':
        job_id = get_job_id(environ)
        try:
            record = read_state(job_id)
        except peewee.DoesNotExist:
            status, response_headers, response_body = create_invalid_return()
            start_response(status, response_headers)
            return [response_body]
        if record:
            status, response_headers, response_body = create_state_return(
                record)
            start_response(status, response_headers)
            return [response_body]
    elif info and info == '/upload':

        # celery_is_alive = ping_celery()

        celery_is_alive = True
        if celery_is_alive:
            # get temporary id from id server
            job_id = get_unique_id(1, 'upload_job')
            update_state(job_id, 'OK', 'UPLOADING', 0)

            try:
                filepath = receive(environ, job_id)
            # pylint: disable=broad-except
            except Exception as exc:
                update_state(job_id, 'FAILED', 'receive bundle', 0, str(exc))
                status = '500 Internal Server Error'
                record = read_state(job_id)
                response_body = create_state_response(record)
                response_headers = [('Content-Type', 'application/json'),
                                    ('Content-Length', str(len(response_body)))
                                    ]
                start_response(status, response_headers)
                return [response_body]
            # pylint: enable=broad-except

            if filepath != '':
                start_ingest(job_id, filepath)

            record = read_state(job_id)

        else:
            record = IngestState()
            record.state = 'ERROR: Celery is dead'
            record.job_id = -99

        if record:
            status, response_headers, response_body = create_state_return(
                record)
            start_response(status, response_headers)
            return [response_body]

    else:
        status, response_headers, response_body = create_invalid_return()
        start_response(status, response_headers)
        return [response_body]

    status, response_headers, response_body = create_invalid_return()
    start_response(status, response_headers)
    return [response_body]