Exemple #1
0
def match_finished(matched_results_paths,
                   match_job_id,
                   match_start_at,
                   match_complete_at,
                   match_status,
                   match_runtime,
                   upload_id=None):
    try:
        logger.info('Writing to match log')
        write_match_log(db_session=db_session,
                        match_job_id=match_job_id,
                        match_start_at=match_start_at,
                        match_complete_at=match_complete_at,
                        match_status=match_status,
                        match_runtime=match_runtime,
                        upload_id=upload_id)
        logger.info('Writing matches to db')
        for event_type, filename in matched_results_paths.items():
            jurisdiction = filename.split('/')[-3]
            logger.info(
                'Writing matches from event type %s and filename %s to db. Parsed jurisdiction %s out of filename',
                event_type, filename, jurisdiction)
            with open_sesame(filename, 'rb') as matches_filehandle:
                write_matches_to_db(db_engine=engine,
                                    event_type=event_type,
                                    jurisdiction=jurisdiction,
                                    matches_filehandle=matches_filehandle)
    except Exception as e:
        logger.error('Error encountered during match_finished: %s', str(e))

    finally:
        logger.info('All done!')
Exemple #2
0
def backend_runner(operation=None, config=None):

    env.parallel = True
    env.warn_only = True

    env.timeout = config.get('SETTINGS_SSH_TIMEOUT', 30)
    env.pool_size = config.get('SETTINGS_POOL_SIZE', 250)
    env.command_timeout = config.get('SETTINGS_SSH_COMMAND_TIMEOUT', 60)
    env.disable_known_hosts = config.get('SETTINGS_DISABLE_KNOWN_HOSTS', True)

    if config['SETTINGS_FABRIC_FILE_PATH'] not in sys.path:
        sys.path.append(config['SETTINGS_FABRIC_FILE_PATH'])

    _type = operation.get('OPT_OPERATION_TYPE', '')

    if _type == u'ssh_status_detecting':
        ssh_status_detecting(operation, config)

    elif _type == u'ping_connectivity_detecting':
        ping_connectivity_detecting(operation, config)

    elif _type == u'custom_script_execute':
        custom_script_execute(operation, config)

    elif _type == u'fabfile_execute':
        fabfile_execute(operation, config, get_fab_tasks(config['SETTINGS_FABRIC_FILE_PATH']))

    elif _type == u'remote_control':
        remote_control(operation, config)

    else:
        logger.error(u'Error operation type. ID is %s' % operation['OPT_ID'])
def ssh_status_detecting(operation, config):
    """
    :Return Code Description:

        0: 执行中
        1: 已完成
        2: 内部错误

    """

    ID = operation.get('OPT_ID', 0)
    API_URL = '%s/operation/%s' % (config.get('SETTINGS_API_BASIC_URL', None), ID)

    env.user = operation.get('SSH_USERNAME', 'root')
    env.password = operation.get('SSH_PASSWORD', 'password')
    env.port = operation.get('SSH_PORT', 22)
    env.key_filename = generate_private_path(operation.get('SSH_PRIVATE_KEY', 'default.key'),
                                             config.get('SETTINGS_PRIVATE_KEY_PATH'))

    with hide('everything'):

        result = execute(basic_remote_runner, 'ls', dict(),
                         hosts=operation.get('OPT_SERVER_LIST', '').split())

    data = json.dumps(dict(id=ID, status=1, result=result),  ensure_ascii=False)

    response = requests.put(API_URL, data=data, headers={'content-type': 'application/json'})

    if response.status_code != requests.codes.ok:
        message = response.json.get('message', 'unknown errors')
        logger.error(u'UPDATE OPERATION FAILS| Operation ID is %s, Message is %s' % (ID, message))
def ping_connectivity_detecting(operation, config):
    """
    :Return:

        0: 执行中
        1: 已完成
        2: 内部错误

    """

    ID = operation.get('OPT_ID', 0)
    API_URL = '%s/operation/%s' % (config.get('SETTINGS_API_BASIC_URL', None), ID)

    COMMAND = 'ping -c%s -W%s {{ address }}' % \
              (config.get('SETTINGS_PING_COUNT', 4), config.get('SETTINGS_PING_TIMEOUT', 5))

    if API_URL is not None:
        with hide('everything'):
            result = execute(base_local_runner, COMMAND,
                             hosts=operation.get('OPT_SERVER_LIST', '').split())

        data = json.dumps(dict(id=ID, status=1, result=result),  ensure_ascii=False)

        response = requests.put(API_URL, data=data, headers={'content-type': 'application/json'})

        if response.status_code != requests.codes.ok:
            message = response.json.get('message', 'unknown errors')
            logger.error(u'UPDATE OPERATION FAILS|Operation ID is %s, Message is %s' % (ID, message))

    else:
        logger.error(u'CONFIG FAILS|Message is can\'t get API url from config')
Exemple #5
0
def backend_runner(operation=None, config=None):

    env.parallel = True
    env.warn_only = True

    env.timeout = config.get('SETTINGS_SSH_TIMEOUT', 30)
    env.pool_size = config.get('SETTINGS_POOL_SIZE', 250)
    env.command_timeout = config.get('SETTINGS_SSH_COMMAND_TIMEOUT', 60)
    env.disable_known_hosts = config.get('SETTINGS_DISABLE_KNOWN_HOSTS', True)

    if config['SETTINGS_FABRIC_FILE_PATH'] not in sys.path:
        sys.path.append(config['SETTINGS_FABRIC_FILE_PATH'])

    _type = operation.get('OPT_OPERATION_TYPE', '')

    if _type == u'ssh_status_detecting':
        ssh_status_detecting(operation, config)

    elif _type == u'ping_connectivity_detecting':
        ping_connectivity_detecting(operation, config)

    elif _type == u'custom_script_execute':
        custom_script_execute(operation, config)

    elif _type == u'fabfile_execute':
        fabfile_execute(operation, config,
                        get_fab_tasks(config['SETTINGS_FABRIC_FILE_PATH']))

    elif _type == u'remote_control':
        remote_control(operation, config)

    else:
        logger.error(u'Error operation type. ID is %s' % operation['OPT_ID'])
Exemple #6
0
def remote_control(operation, config):
    """
    :Return:

        0: 执行中
        1: 已完成
        2: 内部错误

    """

    ID = operation.get('OPT_ID', 0)
    API_URL = '%s/operation/%s' % (config.get('SETTINGS_API_BASIC_URL',
                                              None), ID)

    ipmi_address_list = list()
    for address in operation.get('OPT_SERVER_LIST', '').split():
        ipmi_address_list.append(generate_ipmi_address(address))

    specifies = '-I lanplus' if operation.get('IPMI_INTERFACE', False) else ''
    COMMAND = 'ipmitool %s -H {{ ipmi_address }} -U %s -P %s chassis power %s' % \
              (specifies, operation.get('IPMI_USERNAME', 'root'),
               operation.get('IPMI_PASSWORD', 'calvin'), operation.get('OPT_SCRIPT_TEMPLATE', 'status'))

    if API_URL is not None:

        with hide('everything'):

            result = execute(basic_local_ipmi_runner,
                             COMMAND,
                             stdout=True,
                             stderr=True,
                             hosts=operation.get('OPT_SERVER_LIST',
                                                 '').split())

        data = json.dumps(dict(id=ID, status=1, result=result),
                          ensure_ascii=False)

        response = requests.put(API_URL,
                                data=data,
                                headers={'content-type': 'application/json'})

        if response.status_code != requests.codes.ok:
            message = response.json.get('message', 'unknown errors')
            logger.error(
                u'UPDATE OPERATION FAILS| Operation ID is %s, Message is %s' %
                (ID, message))

    else:
        logger.error(u'CONFIG FAILS|Message is can\'t get API url from config')
Exemple #7
0
def fabfile_execute(operation, config, tasks):
    """
    :Return:

        0: 执行中
        1: 已完成
        2: 内部错误

    """

    ID = operation.get('OPT_ID', 0)
    API_URL = '%s/operation/%s' % (config.get('SETTINGS_API_BASIC_URL', None), ID)

    env.user = operation.get('SSH_USERNAME', 'root')
    env.password = operation.get('SSH_PASSWORD', 'password')
    env.port = operation.get('SSH_PORT', 22)
    env.key_filename = generate_private_path(operation.get('SSH_PRIVATE_KEY', 'default.key'),
                                             config.get('SETTINGS_PRIVATE_KEY_PATH'))

    task = tasks.get(operation.get('FABFILE_NAME', 'default'), None)

    with hide('everything'):
        connectivity = execute(basic_remote_runner, 'ls', dict(), stderr=True,
                               hosts=operation.get('OPT_SERVER_LIST', '').split())

    connectivity_nodes = list()
    for (k, v) in connectivity.items():
        if v.get('code', 1) == 0:
            connectivity_nodes.append(k)

    with hide('everything'):

        result = execute(task, ext_vars=json.loads(operation.get('OPT_EXT_VARIABLES', dict())),
                         hosts=connectivity_nodes)

    connectivity.update(result)

    data = json.dumps(dict(id=ID, status=1, result=connectivity),  ensure_ascii=False)

    response = requests.put(API_URL, data=data, headers={'content-type': 'application/json'})

    if response.status_code != requests.codes.ok:
        message = response.json.get('message', 'unknown errors')
        logger.error(u'UPDATE OPERATION FAILS| Operation ID is %s, Message is %s' % (ID, message))
Exemple #8
0
def custom_script_execute(operation, config):
    """
    :Return Code Description:

        0: 执行中
        1: 已完成
        2: 内部错误

    """

    ID = operation.get('OPT_ID', 0)
    API_URL = '%s/operation/%s' % (config.get('SETTINGS_API_BASIC_URL',
                                              None), ID)

    env.user = operation.get('SSH_USERNAME', 'root')
    env.password = operation.get('SSH_PASSWORD', 'password')
    env.port = operation.get('SSH_PORT', 22)
    env.key_filename = generate_private_path(
        operation.get('SSH_PRIVATE_KEY', 'default.key'),
        config.get('SETTINGS_PRIVATE_KEY_PATH'))

    with hide('everything'):

        result = execute(basic_remote_runner,
                         operation.get('OPT_SCRIPT_TEMPLATE', 'ls'),
                         json.loads(operation.get('OPT_EXT_VARIABLES',
                                                  dict())),
                         stdout=True,
                         stderr=True,
                         hosts=operation.get('OPT_SERVER_LIST', '').split())

    data = json.dumps(dict(id=ID, status=1, result=result), ensure_ascii=False)

    response = requests.put(API_URL,
                            data=data,
                            headers={'content-type': 'application/json'})

    if response.status_code != requests.codes.ok:
        message = response.json.get('message', 'unknown errors')
        logger.error(
            u'UPDATE OPERATION FAILS| Operation ID is %s, Message is %s' %
            (ID, message))
Exemple #9
0
def merge_file():
    upload_id = request.args.get('uploadId', None)
    if not upload_id:
        return jsonify(status='invalid', reason='uploadId not present')
    has_access = False
    try:
        has_access = can_access_file(upload_id)
        if has_access:
            upload_log = db_session.query(Upload).get(upload_id)
            logger.info('Retrieved upload log, merging raw table to master')
            raw_table_name = 'raw_{}'.format(upload_id)
            logger.info('Merging raw table to master')
            merge_id = upsert_raw_table_to_master(raw_table_name,
                                                  upload_log.jurisdiction_slug,
                                                  upload_log.event_type_slug,
                                                  upload_id, db_session)
            logger.info('Syncing merged file to s3')

            bootstrap_master_tables(upload_log.jurisdiction_slug, db_session)

            sync_merged_file_to_storage(upload_log.jurisdiction_slug,
                                        upload_log.event_type_slug,
                                        db_session.get_bind())
            merge_log = db_session.query(MergeLog).get(merge_id)
            try:
                logger.info('Merge succeeded. Now querying matcher')
                notify_matcher(upload_log.jurisdiction_slug, upload_id)
            except Exception as e:
                logger.error('Error matching: ', e)
                db_session.rollback()
                return make_response(jsonify(status='error'), 500)
            db_session.commit()
            return jsonify(status='success',
                           new_unique_rows=merge_log.new_unique_rows,
                           total_unique_rows=merge_log.total_unique_rows)
        else:
            return jsonify(status='not authorized')
    except ValueError as e:
        logger.error('Error merging: ', e)
        db_session.rollback()
        return make_response(jsonify(status='error'), 500)
Exemple #10
0
def remote_control(operation, config):
    """
    :Return:

        0: 执行中
        1: 已完成
        2: 内部错误

    """

    ID = operation.get('OPT_ID', 0)
    API_URL = '%s/operation/%s' % (config.get('SETTINGS_API_BASIC_URL', None), ID)

    ipmi_address_list = list()
    for address in operation.get('OPT_SERVER_LIST', '').split():
        ipmi_address_list.append(generate_ipmi_address(address))

    specifies = '-I lanplus' if operation.get('IPMI_INTERFACE', False) else ''
    COMMAND = 'ipmitool %s -H {{ ipmi_address }} -U %s -P %s chassis power %s' % \
              (specifies, operation.get('IPMI_USERNAME', 'root'),
               operation.get('IPMI_PASSWORD', 'calvin'), operation.get('OPT_SCRIPT_TEMPLATE', 'status'))

    if API_URL is not None:

        with hide('everything'):

            result = execute(basic_local_ipmi_runner, COMMAND,
                             stdout=True, stderr=True,
                             hosts=operation.get('OPT_SERVER_LIST', '').split())

        data = json.dumps(dict(id=ID, status=1, result=result),  ensure_ascii=False)

        response = requests.put(API_URL, data=data, headers={'content-type': 'application/json'})

        if response.status_code != requests.codes.ok:
            message = response.json.get('message', 'unknown errors')
            logger.error(u'UPDATE OPERATION FAILS| Operation ID is %s, Message is %s' % (ID, message))

    else:
        logger.error(u'CONFIG FAILS|Message is can\'t get API url from config')
def ping_connectivity_detecting(operation, config):
    """
    :Return:

        0: 执行中
        1: 已完成
        2: 内部错误

    """

    ID = operation.get('OPT_ID', 0)
    API_URL = '%s/operation/%s' % (config.get('SETTINGS_API_BASIC_URL',
                                              None), ID)

    COMMAND = 'ping -c%s -W%s {{ address }}' % \
              (config.get('SETTINGS_PING_COUNT', 4), config.get('SETTINGS_PING_TIMEOUT', 5))

    if API_URL is not None:
        with hide('everything'):
            result = execute(base_local_runner,
                             COMMAND,
                             hosts=operation.get('OPT_SERVER_LIST',
                                                 '').split())

        data = json.dumps(dict(id=ID, status=1, result=result),
                          ensure_ascii=False)

        response = requests.put(API_URL,
                                data=data,
                                headers={'content-type': 'application/json'})

        if response.status_code != requests.codes.ok:
            message = response.json.get('message', 'unknown errors')
            logger.error(
                u'UPDATE OPERATION FAILS|Operation ID is %s, Message is %s' %
                (ID, message))

    else:
        logger.error(u'CONFIG FAILS|Message is can\'t get API url from config')
Exemple #12
0
def get_histogram_bar_chart_data(data, distribution_function, shared_ids,
                                 data_name):
    intersection_data = data[data.matched_id.isin(shared_ids)]
    distribution, groups = distribution_function(data)
    distribution_intersection, _ = distribution_function(
        intersection_data, groups)
    bins = []
    logger.info(data_name)
    logger.info(distribution_intersection)
    logger.info(len(data.matched_id.unique()))
    for bin_index in range(len(distribution)):
        try:
            of_status = {
                "x":
                data_name,
                "y":
                int(distribution.iloc[bin_index]) /
                len(data.matched_id.unique()) * 100
            }
        except ZeroDivisionError:
            of_status = {"x": data_name, "y": 0}
        try:
            all_status = {
                "x":
                "Jail & Homeless",
                "y":
                int(distribution_intersection.iloc[bin_index]) /
                len(intersection_data.matched_id.unique()) * 100
            }
        except Exception as e:
            logger.error(
                'Error encountered while calculating intersection distribution: %s',
                e)
            all_status = {"x": "Jail & Homeless", "y": 0}
        bins.append((of_status, all_status))
    return [bins, list(distribution.index)]
Exemple #13
0
def get_validated_result(job_key):
    job = Job.fetch(job_key, connection=get_redis_connection())
    if job.is_failed:
        logger.error(job.exc_info)
        return jsonify(
            format_error_report(
                'System error. The error has been logged, please try again later'
            ))
    if not job.is_finished:
        return jsonify({
            'validation': {
                'jobKey': job_key,
                'status': 'validating',
                'message': 'Still validating data!'
            },
            'upload_result': {
                'status': 'not yet',
            }
        })

    result = job.result
    if 'validation' in result and 'upload_result' in result:
        return jsonify(result)
    validation_report = result['validation_report']
    event_type = result['event_type']
    filename_with_all_fields = result['filename_with_all_fields']
    if validation_report['valid']:
        upload_id = job.meta['upload_id']
        row_count = validation_report['tables'][0]['row-count'] - 1

        sample_rows, field_names = get_sample(filename_with_all_fields)
        db_session.commit()
        return jsonify({
            'validation': {
                'status': 'valid',
                'jobKey': job_key,
            },
            'upload_result': {
                'status': 'done',
                'rowCount': row_count,
                'exampleRows': sample_rows,
                'fieldOrder': field_names,
                'uploadId': upload_id
            }
        })
    else:
        db_session.commit()
        return jsonify({
            'validation': {
                'jobKey': job_key,
                'status': 'invalid'
            },
            'upload_result': {
                'status':
                'done',
                'rowCount':
                '',
                'fieldOrder': [],
                'errorReport':
                format_validation_report(validation_report, event_type),
                'upload_id':
                ''
            }
        })