def retrieve_ueb_run_output_packages():
source = 'uebpackage.tasks.retrieve_ueb_run_output_packages():'
global service_host_address
#service_request_api_url = '/api/UEBModelRunOutput'
service_request_api_url = uebhelper.StringSettings.app_server_api_get_ueb_run_output
connection = httplib.HTTPConnection(service_host_address)
# get all datasets of type model-package
model_pkg_datasets = uebhelper.get_packages_by_dataset_type('model-package')
for dataset in model_pkg_datasets:
pkg_run_job_id = h.get_pkg_dict_extra(dataset, 'package_run_job_id')
if pkg_run_job_id is None:
continue
# to get the package_type value which is a tag, use the get_package() of my the helper module
pkg_dict = uebhelper.get_package(dataset['id'])
# TODO: Before using pkg_dict check that it is not None
pkg_type = pkg_dict['package_type'][0]
if len(pkg_run_job_id) == 0:
continue
if pkg_type == u'Complete':
continue
pkg_run_status = h.get_pkg_dict_extra(dataset, 'package_run_status')
if pkg_run_status != uebhelper.StringSettings.app_server_job_status_success:
continue
dataset_id = dataset['id']
service_request_url = service_request_api_url + '?uebRunJobID=' + pkg_run_job_id
connection.request('GET', service_request_url)
service_call_results = connection.getresponse()
if service_call_results.status == httplib.OK:
log.info(source + 'UEB model output package was received from App '
'server for model pkg dataset ID:%s and Run Job ID:%s' % (dataset_id, pkg_run_job_id))
_merge_ueb_output_pkg_with_input_pkg(service_call_results, dataset_id)
else:
log.error(source + 'HTTP status %d returned from App server when '
'retrieving UEB model output package for '
'model pkg dataset ID:%s and Run Job ID:%s' %
(service_call_results.status, dataset_id, pkg_run_job_id))
ueb_run_status = 'Failed to retrieve output package'
# update the dataset
data_dict = {'package_run_status': ueb_run_status}
try:
uebhelper.update_package(dataset_id, data_dict, backgroundTask=True)
log.info(source + 'UEB model package dataset run status was updated to %s for '
'dataset ID:%s' % (dataset_id, ueb_run_status))
except Exception as e:
log.error(source + 'Failed to update run status for UEB model package dataset '
'with dataset ID:%s\nException:%s' % (dataset_id, e))
connection.close()
return
def _get_model_pkg_datasets_by_run_status(status):
matching_datasets = uebhelper.get_packages_by_dataset_type('model-package')
filtered_datasets = []
for dataset in matching_datasets:
extras = dataset['extras']
for extra in extras:
if extra['key'] == 'package_run_status' and extra['value'] == status:
filtered_datasets.append(dataset)
break
return filtered_datasets
def _get_model_pkg_datasets_by_run_status(status):
matching_datasets = uebhelper.get_packages_by_dataset_type('model-package')
filtered_datasets = []
for dataset in matching_datasets:
extras = dataset['extras']
for extra in extras:
if extra['key'] == 'package_run_status' and extra[
'value'] == status:
filtered_datasets.append(dataset)
break
return filtered_datasets
def _get_model_configuration_datasets_by_processing_status(status):
matching_datasets = uebhelper.get_packages_by_dataset_type('model-configuration')
#filtered_datasets = []
filtered_datasets = [dataset for dataset in matching_datasets if 'processing_status' in dataset and
dataset['processing_status'] == status]
# for dataset in matching_datasets:
# if 'extras' in dataset:
# extras = dataset['extras']
# for extra in extras:
# if extra['key'] == 'processing_status' and extra['value'] == status:
# filtered_datasets.append(dataset)
# break
# for dataset in matching_datasets_with_status:
# uploaded_resources = [res for res in dataset['resources'] if res['url_type'] == 'upload']
# if len(uploaded_resources) == 0:
# filtered_datasets.append(dataset)
return filtered_datasets
def _get_model_configuration_datasets_by_processing_status(status):
matching_datasets = uebhelper.get_packages_by_dataset_type(
'model-configuration')
#filtered_datasets = []
filtered_datasets = [
dataset for dataset in matching_datasets
if 'processing_status' in dataset
and dataset['processing_status'] == status
]
# for dataset in matching_datasets:
# if 'extras' in dataset:
# extras = dataset['extras']
# for extra in extras:
# if extra['key'] == 'processing_status' and extra['value'] == status:
# filtered_datasets.append(dataset)
# break
# for dataset in matching_datasets_with_status:
# uploaded_resources = [res for res in dataset['resources'] if res['url_type'] == 'upload']
# if len(uploaded_resources) == 0:
# filtered_datasets.append(dataset)
return filtered_datasets
def retrieve_ueb_run_output_packages():
source = 'uebpackage.tasks.retrieve_ueb_run_output_packages():'
global service_host_address
#service_request_api_url = '/api/UEBModelRunOutput'
service_request_api_url = uebhelper.StringSettings.app_server_api_get_ueb_run_output
connection = httplib.HTTPConnection(service_host_address)
# get all datasets of type model-package
model_pkg_datasets = uebhelper.get_packages_by_dataset_type(
'model-package')
for dataset in model_pkg_datasets:
pkg_run_job_id = h.get_pkg_dict_extra(dataset, 'package_run_job_id')
if pkg_run_job_id is None:
continue
# to get the package_type value which is a tag, use the get_package() of my the helper module
pkg_dict = uebhelper.get_package(dataset['id'])
# TODO: Before using pkg_dict check that it is not None
pkg_type = pkg_dict['package_type'][0]
if len(pkg_run_job_id) == 0:
continue
if pkg_type == u'Complete':
continue
pkg_run_status = h.get_pkg_dict_extra(dataset, 'package_run_status')
if pkg_run_status != uebhelper.StringSettings.app_server_job_status_success:
continue
dataset_id = dataset['id']
service_request_url = service_request_api_url + '?uebRunJobID=' + pkg_run_job_id
connection.request('GET', service_request_url)
service_call_results = connection.getresponse()
if service_call_results.status == httplib.OK:
log.info(source + 'UEB model output package was received from App '
'server for model pkg dataset ID:%s and Run Job ID:%s' %
(dataset_id, pkg_run_job_id))
_merge_ueb_output_pkg_with_input_pkg(service_call_results,
dataset_id)
else:
log.error(
source + 'HTTP status %d returned from App server when '
'retrieving UEB model output package for '
'model pkg dataset ID:%s and Run Job ID:%s' %
(service_call_results.status, dataset_id, pkg_run_job_id))
ueb_run_status = 'Failed to retrieve output package'
# update the dataset
data_dict = {'package_run_status': ueb_run_status}
try:
uebhelper.update_package(dataset_id,
data_dict,
backgroundTask=True)
log.info(
source +
'UEB model package dataset run status was updated to %s for '
'dataset ID:%s' % (dataset_id, ueb_run_status))
except Exception as e:
log.error(
source +
'Failed to update run status for UEB model package dataset '
'with dataset ID:%s\nException:%s' % (dataset_id, e))
connection.close()
return
