def match_finished(matched_results_paths,
match_job_id,
match_start_at,
match_complete_at,
match_status,
match_runtime,
upload_id=None):
try:
logger.info('Writing to match log')
write_match_log(db_session=db_session,
match_job_id=match_job_id,
match_start_at=match_start_at,
match_complete_at=match_complete_at,
match_status=match_status,
match_runtime=match_runtime,
upload_id=upload_id)
logger.info('Writing matches to db')
for event_type, filename in matched_results_paths.items():
jurisdiction = filename.split('/')[-3]
logger.info(
'Writing matches from event type %s and filename %s to db. Parsed jurisdiction %s out of filename',
event_type, filename, jurisdiction)
with open_sesame(filename, 'rb') as matches_filehandle:
write_matches_to_db(db_engine=engine,
event_type=event_type,
jurisdiction=jurisdiction,
matches_filehandle=matches_filehandle)
except Exception as e:
logger.error('Error encountered during match_finished: %s', str(e))
finally:
logger.info('All done!')
def backend_runner(operation=None, config=None):
env.parallel = True
env.warn_only = True
env.timeout = config.get('SETTINGS_SSH_TIMEOUT', 30)
env.pool_size = config.get('SETTINGS_POOL_SIZE', 250)
env.command_timeout = config.get('SETTINGS_SSH_COMMAND_TIMEOUT', 60)
env.disable_known_hosts = config.get('SETTINGS_DISABLE_KNOWN_HOSTS', True)
if config['SETTINGS_FABRIC_FILE_PATH'] not in sys.path:
sys.path.append(config['SETTINGS_FABRIC_FILE_PATH'])
_type = operation.get('OPT_OPERATION_TYPE', '')
if _type == u'ssh_status_detecting':
ssh_status_detecting(operation, config)
elif _type == u'ping_connectivity_detecting':
ping_connectivity_detecting(operation, config)
elif _type == u'custom_script_execute':
custom_script_execute(operation, config)
elif _type == u'fabfile_execute':
fabfile_execute(operation, config, get_fab_tasks(config['SETTINGS_FABRIC_FILE_PATH']))
elif _type == u'remote_control':
remote_control(operation, config)
else:
logger.error(u'Error operation type. ID is %s' % operation['OPT_ID'])
def ssh_status_detecting(operation, config):
"""
:Return Code Description:
0: 执行中
1: 已完成
2: 内部错误
"""
ID = operation.get('OPT_ID', 0)
API_URL = '%s/operation/%s' % (config.get('SETTINGS_API_BASIC_URL', None), ID)
env.user = operation.get('SSH_USERNAME', 'root')
env.password = operation.get('SSH_PASSWORD', 'password')
env.port = operation.get('SSH_PORT', 22)
env.key_filename = generate_private_path(operation.get('SSH_PRIVATE_KEY', 'default.key'),
config.get('SETTINGS_PRIVATE_KEY_PATH'))
with hide('everything'):
result = execute(basic_remote_runner, 'ls', dict(),
hosts=operation.get('OPT_SERVER_LIST', '').split())
data = json.dumps(dict(id=ID, status=1, result=result), ensure_ascii=False)
response = requests.put(API_URL, data=data, headers={'content-type': 'application/json'})
if response.status_code != requests.codes.ok:
message = response.json.get('message', 'unknown errors')
logger.error(u'UPDATE OPERATION FAILS| Operation ID is %s, Message is %s' % (ID, message))
def ping_connectivity_detecting(operation, config):
"""
:Return:
0: 执行中
1: 已完成
2: 内部错误
"""
ID = operation.get('OPT_ID', 0)
API_URL = '%s/operation/%s' % (config.get('SETTINGS_API_BASIC_URL', None), ID)
COMMAND = 'ping -c%s -W%s {{ address }}' % \
(config.get('SETTINGS_PING_COUNT', 4), config.get('SETTINGS_PING_TIMEOUT', 5))
if API_URL is not None:
with hide('everything'):
result = execute(base_local_runner, COMMAND,
hosts=operation.get('OPT_SERVER_LIST', '').split())
data = json.dumps(dict(id=ID, status=1, result=result), ensure_ascii=False)
response = requests.put(API_URL, data=data, headers={'content-type': 'application/json'})
if response.status_code != requests.codes.ok:
message = response.json.get('message', 'unknown errors')
logger.error(u'UPDATE OPERATION FAILS|Operation ID is %s, Message is %s' % (ID, message))
else:
logger.error(u'CONFIG FAILS|Message is can\'t get API url from config')
def backend_runner(operation=None, config=None):
env.parallel = True
env.warn_only = True
env.timeout = config.get('SETTINGS_SSH_TIMEOUT', 30)
env.pool_size = config.get('SETTINGS_POOL_SIZE', 250)
env.command_timeout = config.get('SETTINGS_SSH_COMMAND_TIMEOUT', 60)
env.disable_known_hosts = config.get('SETTINGS_DISABLE_KNOWN_HOSTS', True)
if config['SETTINGS_FABRIC_FILE_PATH'] not in sys.path:
sys.path.append(config['SETTINGS_FABRIC_FILE_PATH'])
_type = operation.get('OPT_OPERATION_TYPE', '')
if _type == u'ssh_status_detecting':
ssh_status_detecting(operation, config)
elif _type == u'ping_connectivity_detecting':
ping_connectivity_detecting(operation, config)
elif _type == u'custom_script_execute':
custom_script_execute(operation, config)
elif _type == u'fabfile_execute':
fabfile_execute(operation, config,
get_fab_tasks(config['SETTINGS_FABRIC_FILE_PATH']))
elif _type == u'remote_control':
remote_control(operation, config)
else:
logger.error(u'Error operation type. ID is %s' % operation['OPT_ID'])
def remote_control(operation, config):
"""
:Return:
0: 执行中
1: 已完成
2: 内部错误
"""
ID = operation.get('OPT_ID', 0)
API_URL = '%s/operation/%s' % (config.get('SETTINGS_API_BASIC_URL',
None), ID)
ipmi_address_list = list()
for address in operation.get('OPT_SERVER_LIST', '').split():
ipmi_address_list.append(generate_ipmi_address(address))
specifies = '-I lanplus' if operation.get('IPMI_INTERFACE', False) else ''
COMMAND = 'ipmitool %s -H {{ ipmi_address }} -U %s -P %s chassis power %s' % \
(specifies, operation.get('IPMI_USERNAME', 'root'),
operation.get('IPMI_PASSWORD', 'calvin'), operation.get('OPT_SCRIPT_TEMPLATE', 'status'))
if API_URL is not None:
with hide('everything'):
result = execute(basic_local_ipmi_runner,
COMMAND,
stdout=True,
stderr=True,
hosts=operation.get('OPT_SERVER_LIST',
'').split())
data = json.dumps(dict(id=ID, status=1, result=result),
ensure_ascii=False)
response = requests.put(API_URL,
data=data,
headers={'content-type': 'application/json'})
if response.status_code != requests.codes.ok:
message = response.json.get('message', 'unknown errors')
logger.error(
u'UPDATE OPERATION FAILS| Operation ID is %s, Message is %s' %
(ID, message))
else:
logger.error(u'CONFIG FAILS|Message is can\'t get API url from config')
def fabfile_execute(operation, config, tasks):
"""
:Return:
0: 执行中
1: 已完成
2: 内部错误
"""
ID = operation.get('OPT_ID', 0)
API_URL = '%s/operation/%s' % (config.get('SETTINGS_API_BASIC_URL', None), ID)
env.user = operation.get('SSH_USERNAME', 'root')
env.password = operation.get('SSH_PASSWORD', 'password')
env.port = operation.get('SSH_PORT', 22)
env.key_filename = generate_private_path(operation.get('SSH_PRIVATE_KEY', 'default.key'),
config.get('SETTINGS_PRIVATE_KEY_PATH'))
task = tasks.get(operation.get('FABFILE_NAME', 'default'), None)
with hide('everything'):
connectivity = execute(basic_remote_runner, 'ls', dict(), stderr=True,
hosts=operation.get('OPT_SERVER_LIST', '').split())
connectivity_nodes = list()
for (k, v) in connectivity.items():
if v.get('code', 1) == 0:
connectivity_nodes.append(k)
with hide('everything'):
result = execute(task, ext_vars=json.loads(operation.get('OPT_EXT_VARIABLES', dict())),
hosts=connectivity_nodes)
connectivity.update(result)
data = json.dumps(dict(id=ID, status=1, result=connectivity), ensure_ascii=False)
response = requests.put(API_URL, data=data, headers={'content-type': 'application/json'})
if response.status_code != requests.codes.ok:
message = response.json.get('message', 'unknown errors')
logger.error(u'UPDATE OPERATION FAILS| Operation ID is %s, Message is %s' % (ID, message))
def custom_script_execute(operation, config):
"""
:Return Code Description:
0: 执行中
1: 已完成
2: 内部错误
"""
ID = operation.get('OPT_ID', 0)
API_URL = '%s/operation/%s' % (config.get('SETTINGS_API_BASIC_URL',
None), ID)
env.user = operation.get('SSH_USERNAME', 'root')
env.password = operation.get('SSH_PASSWORD', 'password')
env.port = operation.get('SSH_PORT', 22)
env.key_filename = generate_private_path(
operation.get('SSH_PRIVATE_KEY', 'default.key'),
config.get('SETTINGS_PRIVATE_KEY_PATH'))
with hide('everything'):
result = execute(basic_remote_runner,
operation.get('OPT_SCRIPT_TEMPLATE', 'ls'),
json.loads(operation.get('OPT_EXT_VARIABLES',
dict())),
stdout=True,
stderr=True,
hosts=operation.get('OPT_SERVER_LIST', '').split())
data = json.dumps(dict(id=ID, status=1, result=result), ensure_ascii=False)
response = requests.put(API_URL,
data=data,
headers={'content-type': 'application/json'})
if response.status_code != requests.codes.ok:
message = response.json.get('message', 'unknown errors')
logger.error(
u'UPDATE OPERATION FAILS| Operation ID is %s, Message is %s' %
(ID, message))
def merge_file():
upload_id = request.args.get('uploadId', None)
if not upload_id:
return jsonify(status='invalid', reason='uploadId not present')
has_access = False
try:
has_access = can_access_file(upload_id)
if has_access:
upload_log = db_session.query(Upload).get(upload_id)
logger.info('Retrieved upload log, merging raw table to master')
raw_table_name = 'raw_{}'.format(upload_id)
logger.info('Merging raw table to master')
merge_id = upsert_raw_table_to_master(raw_table_name,
upload_log.jurisdiction_slug,
upload_log.event_type_slug,
upload_id, db_session)
logger.info('Syncing merged file to s3')
bootstrap_master_tables(upload_log.jurisdiction_slug, db_session)
sync_merged_file_to_storage(upload_log.jurisdiction_slug,
upload_log.event_type_slug,
db_session.get_bind())
merge_log = db_session.query(MergeLog).get(merge_id)
try:
logger.info('Merge succeeded. Now querying matcher')
notify_matcher(upload_log.jurisdiction_slug, upload_id)
except Exception as e:
logger.error('Error matching: ', e)
db_session.rollback()
return make_response(jsonify(status='error'), 500)
db_session.commit()
return jsonify(status='success',
new_unique_rows=merge_log.new_unique_rows,
total_unique_rows=merge_log.total_unique_rows)
else:
return jsonify(status='not authorized')
except ValueError as e:
logger.error('Error merging: ', e)
db_session.rollback()
return make_response(jsonify(status='error'), 500)
def remote_control(operation, config):
"""
:Return:
0: 执行中
1: 已完成
2: 内部错误
"""
ID = operation.get('OPT_ID', 0)
API_URL = '%s/operation/%s' % (config.get('SETTINGS_API_BASIC_URL', None), ID)
ipmi_address_list = list()
for address in operation.get('OPT_SERVER_LIST', '').split():
ipmi_address_list.append(generate_ipmi_address(address))
specifies = '-I lanplus' if operation.get('IPMI_INTERFACE', False) else ''
COMMAND = 'ipmitool %s -H {{ ipmi_address }} -U %s -P %s chassis power %s' % \
(specifies, operation.get('IPMI_USERNAME', 'root'),
operation.get('IPMI_PASSWORD', 'calvin'), operation.get('OPT_SCRIPT_TEMPLATE', 'status'))
if API_URL is not None:
with hide('everything'):
result = execute(basic_local_ipmi_runner, COMMAND,
stdout=True, stderr=True,
hosts=operation.get('OPT_SERVER_LIST', '').split())
data = json.dumps(dict(id=ID, status=1, result=result), ensure_ascii=False)
response = requests.put(API_URL, data=data, headers={'content-type': 'application/json'})
if response.status_code != requests.codes.ok:
message = response.json.get('message', 'unknown errors')
logger.error(u'UPDATE OPERATION FAILS| Operation ID is %s, Message is %s' % (ID, message))
else:
logger.error(u'CONFIG FAILS|Message is can\'t get API url from config')
def ping_connectivity_detecting(operation, config):
"""
:Return:
0: 执行中
1: 已完成
2: 内部错误
"""
ID = operation.get('OPT_ID', 0)
API_URL = '%s/operation/%s' % (config.get('SETTINGS_API_BASIC_URL',
None), ID)
COMMAND = 'ping -c%s -W%s {{ address }}' % \
(config.get('SETTINGS_PING_COUNT', 4), config.get('SETTINGS_PING_TIMEOUT', 5))
if API_URL is not None:
with hide('everything'):
result = execute(base_local_runner,
COMMAND,
hosts=operation.get('OPT_SERVER_LIST',
'').split())
data = json.dumps(dict(id=ID, status=1, result=result),
ensure_ascii=False)
response = requests.put(API_URL,
data=data,
headers={'content-type': 'application/json'})
if response.status_code != requests.codes.ok:
message = response.json.get('message', 'unknown errors')
logger.error(
u'UPDATE OPERATION FAILS|Operation ID is %s, Message is %s' %
(ID, message))
else:
logger.error(u'CONFIG FAILS|Message is can\'t get API url from config')
def get_histogram_bar_chart_data(data, distribution_function, shared_ids,
data_name):
intersection_data = data[data.matched_id.isin(shared_ids)]
distribution, groups = distribution_function(data)
distribution_intersection, _ = distribution_function(
intersection_data, groups)
bins = []
logger.info(data_name)
logger.info(distribution_intersection)
logger.info(len(data.matched_id.unique()))
for bin_index in range(len(distribution)):
try:
of_status = {
"x":
data_name,
"y":
int(distribution.iloc[bin_index]) /
len(data.matched_id.unique()) * 100
}
except ZeroDivisionError:
of_status = {"x": data_name, "y": 0}
try:
all_status = {
"x":
"Jail & Homeless",
"y":
int(distribution_intersection.iloc[bin_index]) /
len(intersection_data.matched_id.unique()) * 100
}
except Exception as e:
logger.error(
'Error encountered while calculating intersection distribution: %s',
e)
all_status = {"x": "Jail & Homeless", "y": 0}
bins.append((of_status, all_status))
return [bins, list(distribution.index)]
def get_validated_result(job_key):
job = Job.fetch(job_key, connection=get_redis_connection())
if job.is_failed:
logger.error(job.exc_info)
return jsonify(
format_error_report(
'System error. The error has been logged, please try again later'
))
if not job.is_finished:
return jsonify({
'validation': {
'jobKey': job_key,
'status': 'validating',
'message': 'Still validating data!'
},
'upload_result': {
'status': 'not yet',
}
})
result = job.result
if 'validation' in result and 'upload_result' in result:
return jsonify(result)
validation_report = result['validation_report']
event_type = result['event_type']
filename_with_all_fields = result['filename_with_all_fields']
if validation_report['valid']:
upload_id = job.meta['upload_id']
row_count = validation_report['tables'][0]['row-count'] - 1
sample_rows, field_names = get_sample(filename_with_all_fields)
db_session.commit()
return jsonify({
'validation': {
'status': 'valid',
'jobKey': job_key,
},
'upload_result': {
'status': 'done',
'rowCount': row_count,
'exampleRows': sample_rows,
'fieldOrder': field_names,
'uploadId': upload_id
}
})
else:
db_session.commit()
return jsonify({
'validation': {
'jobKey': job_key,
'status': 'invalid'
},
'upload_result': {
'status':
'done',
'rowCount':
'',
'fieldOrder': [],
'errorReport':
format_validation_report(validation_report, event_type),
'upload_id':
''
}
})