def retrieve_ueb_run_output_packages(): source = 'uebpackage.tasks.retrieve_ueb_run_output_packages():' global service_host_address #service_request_api_url = '/api/UEBModelRunOutput' service_request_api_url = uebhelper.StringSettings.app_server_api_get_ueb_run_output connection = httplib.HTTPConnection(service_host_address) # get all datasets of type model-package model_pkg_datasets = uebhelper.get_packages_by_dataset_type('model-package') for dataset in model_pkg_datasets: pkg_run_job_id = h.get_pkg_dict_extra(dataset, 'package_run_job_id') if pkg_run_job_id is None: continue # to get the package_type value which is a tag, use the get_package() of my the helper module pkg_dict = uebhelper.get_package(dataset['id']) # TODO: Before using pkg_dict check that it is not None pkg_type = pkg_dict['package_type'][0] if len(pkg_run_job_id) == 0: continue if pkg_type == u'Complete': continue pkg_run_status = h.get_pkg_dict_extra(dataset, 'package_run_status') if pkg_run_status != uebhelper.StringSettings.app_server_job_status_success: continue dataset_id = dataset['id'] service_request_url = service_request_api_url + '?uebRunJobID=' + pkg_run_job_id connection.request('GET', service_request_url) service_call_results = connection.getresponse() if service_call_results.status == httplib.OK: log.info(source + 'UEB model output package was received from App ' 'server for model pkg dataset ID:%s and Run Job ID:%s' % (dataset_id, pkg_run_job_id)) _merge_ueb_output_pkg_with_input_pkg(service_call_results, dataset_id) else: log.error(source + 'HTTP status %d returned from App server when ' 'retrieving UEB model output package for ' 'model pkg dataset ID:%s and Run Job ID:%s' % (service_call_results.status, dataset_id, pkg_run_job_id)) ueb_run_status = 'Failed to retrieve output package' # update the dataset data_dict = {'package_run_status': ueb_run_status} try: uebhelper.update_package(dataset_id, data_dict, backgroundTask=True) log.info(source + 'UEB model package dataset run status was updated to %s for ' 'dataset ID:%s' % (dataset_id, ueb_run_status)) except Exception as e: log.error(source + 'Failed to update run status for UEB model package dataset ' 'with dataset ID:%s\nException:%s' % (dataset_id, e)) connection.close() return
def _get_model_pkg_datasets_by_run_status(status): matching_datasets = uebhelper.get_packages_by_dataset_type('model-package') filtered_datasets = [] for dataset in matching_datasets: extras = dataset['extras'] for extra in extras: if extra['key'] == 'package_run_status' and extra['value'] == status: filtered_datasets.append(dataset) break return filtered_datasets
def _get_model_pkg_datasets_by_run_status(status): matching_datasets = uebhelper.get_packages_by_dataset_type('model-package') filtered_datasets = [] for dataset in matching_datasets: extras = dataset['extras'] for extra in extras: if extra['key'] == 'package_run_status' and extra[ 'value'] == status: filtered_datasets.append(dataset) break return filtered_datasets
def _get_model_configuration_datasets_by_processing_status(status): matching_datasets = uebhelper.get_packages_by_dataset_type('model-configuration') #filtered_datasets = [] filtered_datasets = [dataset for dataset in matching_datasets if 'processing_status' in dataset and dataset['processing_status'] == status] # for dataset in matching_datasets: # if 'extras' in dataset: # extras = dataset['extras'] # for extra in extras: # if extra['key'] == 'processing_status' and extra['value'] == status: # filtered_datasets.append(dataset) # break # for dataset in matching_datasets_with_status: # uploaded_resources = [res for res in dataset['resources'] if res['url_type'] == 'upload'] # if len(uploaded_resources) == 0: # filtered_datasets.append(dataset) return filtered_datasets
def _get_model_configuration_datasets_by_processing_status(status): matching_datasets = uebhelper.get_packages_by_dataset_type( 'model-configuration') #filtered_datasets = [] filtered_datasets = [ dataset for dataset in matching_datasets if 'processing_status' in dataset and dataset['processing_status'] == status ] # for dataset in matching_datasets: # if 'extras' in dataset: # extras = dataset['extras'] # for extra in extras: # if extra['key'] == 'processing_status' and extra['value'] == status: # filtered_datasets.append(dataset) # break # for dataset in matching_datasets_with_status: # uploaded_resources = [res for res in dataset['resources'] if res['url_type'] == 'upload'] # if len(uploaded_resources) == 0: # filtered_datasets.append(dataset) return filtered_datasets
def retrieve_ueb_run_output_packages(): source = 'uebpackage.tasks.retrieve_ueb_run_output_packages():' global service_host_address #service_request_api_url = '/api/UEBModelRunOutput' service_request_api_url = uebhelper.StringSettings.app_server_api_get_ueb_run_output connection = httplib.HTTPConnection(service_host_address) # get all datasets of type model-package model_pkg_datasets = uebhelper.get_packages_by_dataset_type( 'model-package') for dataset in model_pkg_datasets: pkg_run_job_id = h.get_pkg_dict_extra(dataset, 'package_run_job_id') if pkg_run_job_id is None: continue # to get the package_type value which is a tag, use the get_package() of my the helper module pkg_dict = uebhelper.get_package(dataset['id']) # TODO: Before using pkg_dict check that it is not None pkg_type = pkg_dict['package_type'][0] if len(pkg_run_job_id) == 0: continue if pkg_type == u'Complete': continue pkg_run_status = h.get_pkg_dict_extra(dataset, 'package_run_status') if pkg_run_status != uebhelper.StringSettings.app_server_job_status_success: continue dataset_id = dataset['id'] service_request_url = service_request_api_url + '?uebRunJobID=' + pkg_run_job_id connection.request('GET', service_request_url) service_call_results = connection.getresponse() if service_call_results.status == httplib.OK: log.info(source + 'UEB model output package was received from App ' 'server for model pkg dataset ID:%s and Run Job ID:%s' % (dataset_id, pkg_run_job_id)) _merge_ueb_output_pkg_with_input_pkg(service_call_results, dataset_id) else: log.error( source + 'HTTP status %d returned from App server when ' 'retrieving UEB model output package for ' 'model pkg dataset ID:%s and Run Job ID:%s' % (service_call_results.status, dataset_id, pkg_run_job_id)) ueb_run_status = 'Failed to retrieve output package' # update the dataset data_dict = {'package_run_status': ueb_run_status} try: uebhelper.update_package(dataset_id, data_dict, backgroundTask=True) log.info( source + 'UEB model package dataset run status was updated to %s for ' 'dataset ID:%s' % (dataset_id, ueb_run_status)) except Exception as e: log.error( source + 'Failed to update run status for UEB model package dataset ' 'with dataset ID:%s\nException:%s' % (dataset_id, e)) connection.close() return