def retrieve_ueb_run_output_packages():
source = 'uebpackage.tasks.retrieve_ueb_run_output_packages():'
global service_host_address
#service_request_api_url = '/api/UEBModelRunOutput'
service_request_api_url = uebhelper.StringSettings.app_server_api_get_ueb_run_output
connection = httplib.HTTPConnection(service_host_address)
# get all datasets of type model-package
model_pkg_datasets = uebhelper.get_packages_by_dataset_type('model-package')
for dataset in model_pkg_datasets:
pkg_run_job_id = h.get_pkg_dict_extra(dataset, 'package_run_job_id')
if pkg_run_job_id is None:
continue
# to get the package_type value which is a tag, use the get_package() of my the helper module
pkg_dict = uebhelper.get_package(dataset['id'])
# TODO: Before using pkg_dict check that it is not None
pkg_type = pkg_dict['package_type'][0]
if len(pkg_run_job_id) == 0:
continue
if pkg_type == u'Complete':
continue
pkg_run_status = h.get_pkg_dict_extra(dataset, 'package_run_status')
if pkg_run_status != uebhelper.StringSettings.app_server_job_status_success:
continue
dataset_id = dataset['id']
service_request_url = service_request_api_url + '?uebRunJobID=' + pkg_run_job_id
connection.request('GET', service_request_url)
service_call_results = connection.getresponse()
if service_call_results.status == httplib.OK:
log.info(source + 'UEB model output package was received from App '
'server for model pkg dataset ID:%s and Run Job ID:%s' % (dataset_id, pkg_run_job_id))
_merge_ueb_output_pkg_with_input_pkg(service_call_results, dataset_id)
else:
log.error(source + 'HTTP status %d returned from App server when '
'retrieving UEB model output package for '
'model pkg dataset ID:%s and Run Job ID:%s' %
(service_call_results.status, dataset_id, pkg_run_job_id))
ueb_run_status = 'Failed to retrieve output package'
# update the dataset
data_dict = {'package_run_status': ueb_run_status}
try:
uebhelper.update_package(dataset_id, data_dict, backgroundTask=True)
log.info(source + 'UEB model package dataset run status was updated to %s for '
'dataset ID:%s' % (dataset_id, ueb_run_status))
except Exception as e:
log.error(source + 'Failed to update run status for UEB model package dataset '
'with dataset ID:%s\nException:%s' % (dataset_id, e))
connection.close()
return
def check_ueb_run_status():
source = 'uebpackage.tasks.check_ueb_run_status():'
global service_host_address
service_request_api_url = uebhelper.StringSettings.app_server_api_check_ueb_run_status_url
connection = httplib.HTTPConnection(service_host_address)
job_status_processing = uebhelper.StringSettings.app_server_job_status_processing
job_status_in_queue = uebhelper.StringSettings.app_server_job_status_in_queue
model_pkg_datasets_with_run_status_processing = _get_model_pkg_datasets_by_run_status(job_status_processing)
model_pkg_datasets_with_run_status_in_queue = _get_model_pkg_datasets_by_run_status(job_status_in_queue)
model_pkg_datasets_need_run_status_update = model_pkg_datasets_with_run_status_processing + \
model_pkg_datasets_with_run_status_in_queue
if len(model_pkg_datasets_need_run_status_update) == 0:
log.info(source + "No UEB model package dataset has a run status of %s at this time" % job_status_processing)
else:
log.info(source + "Number of UEB model package datatsets with run status of %s or %s at this time is:%s"
% (job_status_processing, job_status_in_queue, len(model_pkg_datasets_need_run_status_update)))
for dataset in model_pkg_datasets_need_run_status_update:
pkg_run_job_id = h.get_pkg_dict_extra(dataset, 'package_run_job_id')
if pkg_run_job_id is None:
continue
dataset_id = dataset['id']
service_request_url = service_request_api_url + '?uebRunJobID=' + pkg_run_job_id
connection.request('GET', service_request_url)
service_call_results = connection.getresponse()
if service_call_results.status == httplib.OK:
request_processing_status = service_call_results.read()
log.info(source + 'UEB model package run status as returned from App '
'server for dataset ID: %s and Run Job ID:%s is %s' %
(dataset_id, pkg_run_job_id, request_processing_status))
else:
request_processing_status = uebhelper.StringSettings.app_server_job_status_error
log.error(source + 'HTTP status %d returned from App server when checking '
'run status for Run Job ID:%s and model pkg dataset ID:%s' %
(service_call_results.status, pkg_run_job_id, dataset_id))
connection.close()
# update the dataset
data_dict = {'package_run_status': request_processing_status}
try:
uebhelper.update_package(dataset_id, data_dict, backgroundTask=True)
log.info(source + 'UEB model package dataset run status was updated to %s for '
'dataset ID:%s' % (dataset_id, request_processing_status))
except Exception as e:
log.error(source + 'Failed to update run status for UEB model package dataset '
'with dataset ID:%s\nException:%s' % (dataset_id, e))
def _merge_ueb_output_pkg_with_input_pkg(service_call_results,
model_pkg_dataset_id):
source = 'uebpackage.tasks._merge_ueb_output_pkg_with_input_pkg():'
# save the output model pkg to temp directory
#ckan_default_dir = '/tmp/ckan'
ckan_default_dir = uebhelper.StringSettings.ckan_user_session_temp_dir
# create a directory for saving the file
# this will be a dir in the form of: /tmp/ckan/{random_id}
random_id = base.model.types.make_uuid()
destination_dir = os.path.join(ckan_default_dir, random_id)
os.makedirs(destination_dir)
ueb_output_pkg_filename = uebhelper.StringSettings.ueb_output_model_package_default_filename
ueb_output_pkg_file = os.path.join(destination_dir,
ueb_output_pkg_filename)
bytes_to_read = 16 * 1024
try:
with open(ueb_output_pkg_file, 'wb') as file_obj:
while True:
data = service_call_results.read(bytes_to_read)
if not data:
break
file_obj.write(data)
except Exception as e:
log.error(source +
'Failed to save ueb model output package zip file to '
'temporary location for model package dataset ID: %s \n '
'Exception: %s' % (model_pkg_dataset_id, e))
pass
return # no need to show error to the user as this a background operation
log.info(
source +
'ueb model output package zip file was saved to temporary location '
'for model package dataset ID: %s' % model_pkg_dataset_id)
# access the input model pkg zip file
model_pkg_dataset = uebhelper.get_package(model_pkg_dataset_id)
# TODO: Before using model_pk_dataset check that it is not None
model_pkg_resource_zip_file = model_pkg_dataset['resources'][0]
# replace the '%3A' in the file url with ':' to get the correct folder name in the file system
pkg_zip_file_url = model_pkg_resource_zip_file['url']
pkg_zip_file_url = pkg_zip_file_url.replace('%3A', ':')
sting_to_search_in_file_url = 'storage/f/'
search_string_index = pkg_zip_file_url.find(sting_to_search_in_file_url)
file_path_start_index = search_string_index + len(
sting_to_search_in_file_url)
original_zip_file_path = pkg_zip_file_url[file_path_start_index:]
original_zip_file_path = os.path.join('/', 'var', 'lib', 'ckan', 'default',
'pairtree_root', 'de', 'fa', 'ul',
't', 'obj', original_zip_file_path)
'''
open the original input zip file in the append mode and then
read the output pkg zip file and append it to the original zip file
'''
is_merge_successful = False
try:
with zip.ZipFile(original_zip_file_path, 'a') as orig_file_obj:
zip_file_to_merge = zip.ZipFile(ueb_output_pkg_file, 'r')
for fname in zip_file_to_merge.namelist():
orig_file_obj.writestr(fname,
zip_file_to_merge.open(fname).read())
is_merge_successful = True
except Exception as e:
log.error(
source +
'Failed to merge output model pkg zip file with the input model pkg zip file '
'for model package dataset ID: %s \n '
'Exception: %s' % (model_pkg_dataset_id, e))
pass
# update the model package dataset package_type to complete
if is_merge_successful:
data_dict = {
'package_run_status': 'Output package available',
'package_type': u'Complete'
}
else:
data_dict = {
'package_run_status': 'Success'
} # TODO: set to 'Success' for testing. Finally needs to be set "Merge failed"
update_msg = 'system auto updated ueb package dataset'
background_task = True
try:
updated_package = uebhelper.update_package(model_pkg_dataset_id,
data_dict, update_msg,
background_task)
log.info(source +
'UEB model package dataset was updated as a result of '
'receiving model output package for dataset:%s' %
updated_package['name'])
except Exception as e:
log.error(source + 'Failed to update UEB model package dataset after '
'receiving model input package for dataset ID:%s \n'
'Exception: %s' % (model_pkg_dataset_id, e))
pass
def _save_ueb_package_as_dataset_obsolete(service_call_results,
model_config_dataset_id):
source = 'uebpackage.tasks._save_ueb_package_as_dataset():'
ckan_default_dir = uebhelper.StringSettings.ckan_user_session_temp_dir # '/tmp/ckan'
# get the matching model configuration dataset object
model_config_dataset_obj = base.model.Package.get(model_config_dataset_id)
model_config_dataset_title = model_config_dataset_obj.title
model_config_dataset_owner_org = model_config_dataset_obj.owner_org
model_config_dataset_author = model_config_dataset_obj.author
# create a directory for saving the file
# this will be a dir in the form of: /tmp/ckan/{random_id}
random_id = base.model.types.make_uuid()
destination_dir = os.path.join(ckan_default_dir, random_id)
os.makedirs(destination_dir)
model_pkg_filename = uebhelper.StringSettings.ueb_input_model_package_default_filename # 'ueb_model_pkg.zip'
model_pkg_file = os.path.join(destination_dir, model_pkg_filename)
bytes_to_read = 16 * 1024
try:
with open(model_pkg_file, 'wb') as file_obj:
while True:
data = service_call_results.read(bytes_to_read)
if not data:
break
file_obj.write(data)
except Exception as e:
log.error(source +
'Failed to save the ueb_package zip file to temporary '
'location for UEB model configuration dataset ID: %s \n '
'Exception: %s' % (model_config_dataset_id, e))
raise e
log.info(source +
'ueb_package zip file was saved to temporary location for '
'UEB model configuration dataset ID: %s' %
model_config_dataset_id)
# upload the file to CKAN file store
resource_metadata = _upload_file(model_pkg_file)
if resource_metadata:
log.info(
source +
'UEB model package zip file was uploaded for model configuration dataset ID:%s'
% model_config_dataset_id)
else:
log.error(source + 'Failed to upload UEB model package zip file '
'for model configuration dataset ID: %s' %
model_config_dataset_id)
return
# retrieve some of the file meta data
resource_url = resource_metadata.get(
'_label') # this will return datetime stamp/filename
resource_url = '/storage/f/' + resource_url
if resource_url.startswith('/'):
resource_url = base.config.get('ckan.site_url',
'').rstrip('/') + resource_url
else:
resource_url = base.config.get('ckan.site_url', '') + resource_url
resource_created_date = resource_metadata.get('_creation_date')
resource_name = resource_metadata.get('filename_original')
resource_size = resource_metadata.get('_content_length')
# add the uploaded ueb model pkg data file as a resource to the dataset
resource_create_action = tk.get_action('resource_create')
context = {
'model': base.model,
'session': base.model.Session,
'save': 'save'
}
user = uebhelper.get_site_user()
context['user'] = user.get('name')
context['ignore_auth'] = True
context['validate'] = False
# create a package
package_create_action = tk.get_action('package_create')
# create unique package name using the current time stamp as a postfix to any package name
unique_postfix = datetime.now().isoformat().replace(':', '-').replace(
'.', '-').lower()
pkg_title = model_config_dataset_title
data_dict = {
'name': 'model_package_' +
unique_postfix, # this needs to be unique as required by DB
'type':
'model-package', # dataset type as defined in custom dataset plugin
'title': pkg_title,
'owner_org': model_config_dataset_owner_org,
'author': model_config_dataset_author,
'notes': 'UEB model package',
'pkg_model_name': 'UEB',
'model_version': '1.0',
'north_extent': '',
'south_extent': '',
'east_extent': '',
'west_extent': '',
'simulation_start_day': '',
'simulation_end_day': '',
'time_step': '',
'package_type': u'Input',
'package_run_status': 'Not yet submitted',
'package_run_job_id': '',
'dataset_type': 'model-package'
}
context = {
'model': base.model,
'session': base.model.Session,
'ignore_auth': True,
'user': user.get('name'),
'save': 'save'
}
try:
uebhelper.register_translator(
) # this is needed since we are creating a package in a background operation
pkg_dict = package_create_action(context, data_dict)
log.info(
source +
'A new dataset was created for UEB input model package with name: %s'
% data_dict['title'])
except Exception as e:
log.error(
source +
'Failed to create a new dataset for ueb input model package for'
' the related model configuration dataset title: %s \n Exception: %s'
% (pkg_title, e))
return
pkg_id = pkg_dict['id']
#A value for key 'message" in the context must be set to
# avoid a bug in line#192 of the update.py under lib/action
# without this, a TypeError will occur (TypeError:No object(name:translator) has been registered for this thread)
context['message'] = 'Auto uploaded model package '
data_dict = {
"package_id":
pkg_id, # id of the package/dataset to which the resource needs to be added
"url": resource_url,
"name": resource_name,
"created": resource_created_date,
"format": 'zip',
"size": resource_size,
"description": 'UEB model package'
}
is_resource_add_success = False
try:
resource_create_action(context, data_dict)
is_resource_add_success = True
log.info(source + 'UEB model package was added as a resource to the '
'newly created model package dataset with ID:%s' % pkg_id)
except Exception as e:
log.error(
source + 'Failed to add UEB model package as a resource to '
'the newly created model package dataset with ID:%s \nException: %s'
% (pkg_id, e))
pass
# link this newly created model package dataset to the model configuration dataset
package_relationship_create_action = tk.get_action(
'package_relationship_create')
data_dict = {
'subject': pkg_id,
'object': model_config_dataset_id,
'type': 'links_to'
}
package_relationship_create_action(context, data_dict)
# update the related model configuration dataset to show that the package is available
if is_resource_add_success:
# update the related dataset
data_dict = {'package_availability': 'Available'}
update_msg = 'system auto updated ueb package dataset'
background_task = True
try:
updated_package = uebhelper.update_package(model_config_dataset_id,
data_dict, update_msg,
background_task)
log.info(
source +
'UEB model configuration dataset was updated as a result of '
'receiving model input package for dataset:%s' %
updated_package['name'])
except Exception as e:
log.error(source +
'Failed to update UEB model configuration dataset after '
'receiving model input package for dataset ID:%s \n'
'Exception: %s' % (model_config_dataset_id, e))
pass
return
def _save_ueb_package_as_dataset(service_call_results,
model_config_dataset_id):
source = 'uebpackage.tasks._save_ueb_package_as_dataset():'
ckan_default_dir = uebhelper.StringSettings.ckan_user_session_temp_dir # '/tmp/ckan'
# get the matching model configuration dataset object
model_config_dataset_obj = base.model.Package.get(model_config_dataset_id)
model_config_dataset_title = model_config_dataset_obj.title
model_config_dataset_owner_org = model_config_dataset_obj.owner_org
model_config_dataset_author = model_config_dataset_obj.author
# create a directory for saving the file
# this will be a dir in the form of: /tmp/ckan/{random_id}
random_id = base.model.types.make_uuid()
destination_dir = os.path.join(ckan_default_dir, random_id)
os.makedirs(destination_dir)
model_pkg_filename = uebhelper.StringSettings.ueb_input_model_package_default_filename # 'ueb_model_pkg.zip'
model_pkg_file = os.path.join(destination_dir, model_pkg_filename)
bytes_to_read = 16 * 1024
try:
with open(model_pkg_file, 'wb') as file_obj:
while True:
data = service_call_results.read(bytes_to_read)
if not data:
break
file_obj.write(data)
except Exception as e:
log.error(source +
'Failed to save the ueb_package zip file to temporary '
'location for UEB model configuration dataset ID: %s \n '
'Exception: %s' % (model_config_dataset_id, e))
raise e
log.info(source +
'ueb_package zip file was saved to temporary location for '
'UEB model configuration dataset ID: %s' %
model_config_dataset_id)
# upload the file to CKAN file store
# resource_metadata = _upload_file(model_pkg_file)
# if resource_metadata:
# log.info(source + 'UEB model package zip file was uploaded for model configuration dataset ID:%s' % model_config_dataset_id)
# else:
# log.error(source + 'Failed to upload UEB model package zip file '
# 'for model configuration dataset ID: %s' % model_config_dataset_id)
# return
#
# # retrieve some of the file meta data
# resource_url = resource_metadata.get('_label') # this will return datetime stamp/filename
#
# resource_url = '/storage/f/' + resource_url
# if resource_url.startswith('/'):
# resource_url = base.config.get('ckan.site_url', '').rstrip('/') + resource_url
# else:
# resource_url = base.config.get('ckan.site_url', '') + resource_url
#
# resource_created_date = resource_metadata.get('_creation_date')
# resource_name = resource_metadata.get('filename_original')
# resource_size = resource_metadata.get('_content_length')
#
# # add the uploaded ueb model pkg data file as a resource to the dataset
# resource_create_action = tk.get_action('resource_create')
# context = {'model': base.model, 'session': base.model.Session, 'save': 'save'}
# user = uebhelper.get_site_user()
# context['user'] = user.get('name')
# context['ignore_auth'] = True
# context['validate'] = False
user = uebhelper.get_site_user()
# create a package
package_create_action = tk.get_action('package_create')
# create unique package name using the current time stamp as a postfix to any package name
unique_postfix = datetime.now().isoformat().replace(':', '-').replace(
'.', '-').lower()
pkg_title = model_config_dataset_title
data_dict = {
'name': 'model_package_' +
unique_postfix, # this needs to be unique as required by DB
'type':
'model-package', # dataset type as defined in custom dataset plugin
'title': pkg_title,
'owner_org': model_config_dataset_owner_org,
'author': model_config_dataset_author,
'notes': 'UEB model package',
'pkg_model_name': 'UEB',
'model_version': '1.0',
'north_extent': '',
'south_extent': '',
'east_extent': '',
'west_extent': '',
'simulation_start_day': '',
'simulation_end_day': '',
'time_step': '',
'package_type': u'Input',
'package_run_status': 'Not yet submitted',
'package_run_job_id': '',
'dataset_type': 'model-package'
}
context = {
'model': base.model,
'session': base.model.Session,
'ignore_auth': True,
'user': user.get('name'),
'save': 'save'
}
try:
uebhelper.register_translator(
) # this is needed since we are creating a package in a background operation
pkg_dict = package_create_action(context, data_dict)
log.info(
source +
'A new dataset was created for UEB input model package with name: %s'
% data_dict['title'])
except Exception as e:
log.error(
source +
'Failed to create a new dataset for ueb input model package for'
' the related model configuration dataset title: %s \n Exception: %s'
% (pkg_title, e))
raise e
pkg_id = pkg_dict['id']
if not 'resources' in pkg_dict:
pkg_dict['resources'] = []
file_name = munge.munge_filename(model_pkg_filename)
resource = {'url': file_name, 'url_type': 'upload'}
upload = uploader.ResourceUpload(resource)
upload.filename = file_name
upload.upload_file = open(model_pkg_file, 'r')
data_dict = {
'format': 'zip',
'name': file_name,
'url': file_name,
'url_type': 'upload'
}
pkg_dict['resources'].append(data_dict)
try:
context['defer_commit'] = True
context['use_cache'] = False
# update the package
package_update_action = tk.get_action('package_update')
package_update_action(context, pkg_dict)
context.pop('defer_commit')
except Exception as e:
log.error(
source +
' Failed to update the new dataset for adding the input model package zip file as'
' a resource.\n Exception: %s' % e)
raise e
# link this newly created model package dataset to the model configuration dataset
package_relationship_create_action = tk.get_action(
'package_relationship_create')
data_dict = {
'subject': pkg_id,
'object': model_config_dataset_id,
'type': 'links_to'
}
package_relationship_create_action(context, data_dict)
# Get out resource_id resource from model as it will not appear in
# package_show until after commit
upload.upload(context['package'].resources[-1].id,
uploader.get_max_resource_size())
base.model.repo.commit()
# update the related model configuration dataset to show that the package is available
data_dict = {'package_availability': 'Available'}
update_msg = 'system auto updated ueb package dataset'
background_task = True
try:
updated_package = uebhelper.update_package(model_config_dataset_id,
data_dict, update_msg,
background_task)
log.info(source +
'UEB model configuration dataset was updated as a result of '
'receiving model input package for dataset:%s' %
updated_package['name'])
except Exception as e:
log.error(source +
'Failed to update UEB model configuration dataset after '
'receiving model input package for dataset ID:%s \n'
'Exception: %s' % (model_config_dataset_id, e))
raise e
def check_ueb_request_process_status():
source = 'uebpackage.tasks.check_ueb_request_process_status():'
global service_host_address
service_request_api_url = uebhelper.StringSettings.app_server_api_check_ueb_package_build_status
connection = httplib.HTTPConnection(service_host_address)
job_status_processing = uebhelper.StringSettings.app_server_job_status_processing
model_config_datasets_with_status_processing = _get_model_configuration_datasets_by_processing_status(
job_status_processing)
job_status_in_queue = uebhelper.StringSettings.app_server_job_status_in_queue
model_config_datasets_with_status_in_queue = _get_model_configuration_datasets_by_processing_status(
job_status_in_queue)
# merge the 2 lists
model_config_datasets_with_status_processing_or_in_queue = model_config_datasets_with_status_processing + \
model_config_datasets_with_status_in_queue
if len(model_config_datasets_with_status_processing_or_in_queue) == 0:
log.info(
source +
"No UEB model configuration dataset has a status of %s or %s at this time"
% (job_status_processing, job_status_in_queue))
else:
log.info(
source +
"Number of UEB model configuration datasets build status of %s or %s at this time is:%s"
% (job_status_processing, job_status_in_queue,
len(model_config_datasets_with_status_processing_or_in_queue)))
for dataset in model_config_datasets_with_status_processing_or_in_queue:
#pkg_process_job_id = h.get_pkg_dict_extra(dataset, 'package_build_request_job_id')
# in CKAN 2.2 additional metatadata elements added to the dataset of specific type
# seems not part of the 'extra' key of the dataset dict. These additional metadata elements
# are now part of the dataset main dictionary
pkg_process_job_id = dataset.get('package_build_request_job_id', None)
#pkg_current_processing_status = h.get_pkg_dict_extra(dataset, 'processing_status')
pkg_current_processing_status = dataset.get('processing_status', None)
if pkg_process_job_id and pkg_current_processing_status:
dataset_id = dataset['id']
service_request_url = service_request_api_url + '?packageID=' + pkg_process_job_id
connection.request('GET', service_request_url)
service_call_results = connection.getresponse()
if service_call_results.status == httplib.OK:
request_processing_status = service_call_results.read()
log.info(
source +
'UEB model package build status as returned from App server '
'for PackageJobID:%s is %s' %
(pkg_process_job_id, request_processing_status))
else:
request_processing_status = uebhelper.StringSettings.app_server_job_status_error
log.error(
source +
'HTTP status %d returned from App server when checking '
'status for PackageJobID:%s' %
(service_call_results.status, pkg_process_job_id))
connection.close()
# update the dataset if the status has changed
if pkg_current_processing_status != request_processing_status:
data_dict = {'processing_status': request_processing_status}
uebhelper.update_package(dataset_id,
data_dict,
backgroundTask=True)
else:
log.error(
source +
"Either the metadata element 'package_build_request_job_id' "
"or the 'processing_status' was not found "
"for dataset ID:%s" % dataset['id'])
def retrieve_ueb_run_output_packages():
source = 'uebpackage.tasks.retrieve_ueb_run_output_packages():'
global service_host_address
#service_request_api_url = '/api/UEBModelRunOutput'
service_request_api_url = uebhelper.StringSettings.app_server_api_get_ueb_run_output
connection = httplib.HTTPConnection(service_host_address)
# get all datasets of type model-package
model_pkg_datasets = uebhelper.get_packages_by_dataset_type(
'model-package')
for dataset in model_pkg_datasets:
pkg_run_job_id = h.get_pkg_dict_extra(dataset, 'package_run_job_id')
if pkg_run_job_id is None:
continue
# to get the package_type value which is a tag, use the get_package() of my the helper module
pkg_dict = uebhelper.get_package(dataset['id'])
# TODO: Before using pkg_dict check that it is not None
pkg_type = pkg_dict['package_type'][0]
if len(pkg_run_job_id) == 0:
continue
if pkg_type == u'Complete':
continue
pkg_run_status = h.get_pkg_dict_extra(dataset, 'package_run_status')
if pkg_run_status != uebhelper.StringSettings.app_server_job_status_success:
continue
dataset_id = dataset['id']
service_request_url = service_request_api_url + '?uebRunJobID=' + pkg_run_job_id
connection.request('GET', service_request_url)
service_call_results = connection.getresponse()
if service_call_results.status == httplib.OK:
log.info(source + 'UEB model output package was received from App '
'server for model pkg dataset ID:%s and Run Job ID:%s' %
(dataset_id, pkg_run_job_id))
_merge_ueb_output_pkg_with_input_pkg(service_call_results,
dataset_id)
else:
log.error(
source + 'HTTP status %d returned from App server when '
'retrieving UEB model output package for '
'model pkg dataset ID:%s and Run Job ID:%s' %
(service_call_results.status, dataset_id, pkg_run_job_id))
ueb_run_status = 'Failed to retrieve output package'
# update the dataset
data_dict = {'package_run_status': ueb_run_status}
try:
uebhelper.update_package(dataset_id,
data_dict,
backgroundTask=True)
log.info(
source +
'UEB model package dataset run status was updated to %s for '
'dataset ID:%s' % (dataset_id, ueb_run_status))
except Exception as e:
log.error(
source +
'Failed to update run status for UEB model package dataset '
'with dataset ID:%s\nException:%s' % (dataset_id, e))
connection.close()
return
def retrieve_ueb_packages():
source = 'uebpackage.tasks.retrieve_ueb_packages():'
global service_host_address
service_request_api_url = uebhelper.StringSettings.app_server_api_get_ueb_package_url
connection = httplib.HTTPConnection(service_host_address)
job_status_complete = uebhelper.StringSettings.app_server_job_status_success
model_config_datasets_with_status_complete = _get_model_configuration_datasets_by_processing_status(
job_status_complete)
if len(model_config_datasets_with_status_complete) == 0:
log.info(
source +
"No UEB model configuration dataset has a status of %s at this time"
% job_status_complete)
else:
log.info(
source +
"Number of UEB model configuration datasets with build status of %s at this time is:%s"
% (job_status_complete,
len(model_config_datasets_with_status_complete)))
for dataset in model_config_datasets_with_status_complete:
pkg_availability_status = h.get_pkg_dict_extra(dataset,
'package_availability')
if pkg_availability_status == uebhelper.StringSettings.app_server_job_status_package_available:
continue
pkg_process_job_id = h.get_pkg_dict_extra(
dataset, 'package_build_request_job_id')
dataset_id = dataset['id']
package_availability_status = h.get_pkg_dict_extra(
dataset, 'package_availability')
# if package is already available or error has been logged for package retrieval then skip this dataset
if package_availability_status == uebhelper.StringSettings.app_server_job_status_package_available or \
package_availability_status == uebhelper.StringSettings.app_server_job_status_error:
continue
service_request_url = service_request_api_url + '?packageID=' + pkg_process_job_id
connection.request('GET', service_request_url)
service_call_results = connection.getresponse()
if service_call_results.status == httplib.OK:
log.info(
source +
'UEB model package was received from App server for PackageJobID:%s'
% pkg_process_job_id)
try:
_save_ueb_package_as_dataset(service_call_results, dataset_id)
pkg_availability_status = uebhelper.StringSettings.app_server_job_status_package_available
except Exception as e:
log.error(
source +
'Failed to save ueb model package as a new dataset '
'for model configuration dataset ID:%s\nException:%s' %
(dataset_id, e))
pkg_availability_status = uebhelper.StringSettings.app_server_job_status_error
else:
log.error(
source +
'HTTP status %d returned from App server when retrieving '
'UEB model package for PackageJobID:'
'%s' % (service_call_results.status, pkg_process_job_id))
pkg_availability_status = uebhelper.StringSettings.app_server_job_status_error
connection.close()
# update the resource processing status
# update the related dataset
data_dict = {'package_availability': pkg_availability_status}
update_msg = 'system auto updated ueb package dataset'
background_task = True
try:
updated_package = uebhelper.update_package(dataset_id, data_dict,
update_msg,
background_task)
log.info(
source +
'UEB model configuration dataset was updated as a result of '
'receiving model input package for dataset:%s' %
updated_package['name'])
except Exception as e:
log.error(source +
'Failed to update UEB model configuration dataset after '
'receiving model input package for dataset ID:%s \n'
'Exception: %s' % (dataset_id, e))
pass
return
def check_ueb_run_status():
source = 'uebpackage.tasks.check_ueb_run_status():'
global service_host_address
service_request_api_url = uebhelper.StringSettings.app_server_api_check_ueb_run_status_url
connection = httplib.HTTPConnection(service_host_address)
job_status_processing = uebhelper.StringSettings.app_server_job_status_processing
job_status_in_queue = uebhelper.StringSettings.app_server_job_status_in_queue
model_pkg_datasets_with_run_status_processing = _get_model_pkg_datasets_by_run_status(
job_status_processing)
model_pkg_datasets_with_run_status_in_queue = _get_model_pkg_datasets_by_run_status(
job_status_in_queue)
model_pkg_datasets_need_run_status_update = model_pkg_datasets_with_run_status_processing + \
model_pkg_datasets_with_run_status_in_queue
if len(model_pkg_datasets_need_run_status_update) == 0:
log.info(
source +
"No UEB model package dataset has a run status of %s at this time"
% job_status_processing)
else:
log.info(
source +
"Number of UEB model package datatsets with run status of %s or %s at this time is:%s"
% (job_status_processing, job_status_in_queue,
len(model_pkg_datasets_need_run_status_update)))
for dataset in model_pkg_datasets_need_run_status_update:
pkg_run_job_id = h.get_pkg_dict_extra(dataset, 'package_run_job_id')
if pkg_run_job_id is None:
continue
dataset_id = dataset['id']
service_request_url = service_request_api_url + '?uebRunJobID=' + pkg_run_job_id
connection.request('GET', service_request_url)
service_call_results = connection.getresponse()
if service_call_results.status == httplib.OK:
request_processing_status = service_call_results.read()
log.info(source +
'UEB model package run status as returned from App '
'server for dataset ID: %s and Run Job ID:%s is %s' %
(dataset_id, pkg_run_job_id, request_processing_status))
else:
request_processing_status = uebhelper.StringSettings.app_server_job_status_error
log.error(
source +
'HTTP status %d returned from App server when checking '
'run status for Run Job ID:%s and model pkg dataset ID:%s' %
(service_call_results.status, pkg_run_job_id, dataset_id))
connection.close()
# update the dataset
data_dict = {'package_run_status': request_processing_status}
try:
uebhelper.update_package(dataset_id,
data_dict,
backgroundTask=True)
log.info(
source +
'UEB model package dataset run status was updated to %s for '
'dataset ID:%s' % (dataset_id, request_processing_status))
except Exception as e:
log.error(
source +
'Failed to update run status for UEB model package dataset '
'with dataset ID:%s\nException:%s' % (dataset_id, e))
def _merge_ueb_output_pkg_with_input_pkg(service_call_results, model_pkg_dataset_id):
source = 'uebpackage.tasks._merge_ueb_output_pkg_with_input_pkg():'
# save the output model pkg to temp directory
#ckan_default_dir = '/tmp/ckan'
ckan_default_dir = uebhelper.StringSettings.ckan_user_session_temp_dir
# create a directory for saving the file
# this will be a dir in the form of: /tmp/ckan/{random_id}
random_id = base.model.types.make_uuid()
destination_dir = os.path.join(ckan_default_dir, random_id)
os.makedirs(destination_dir)
ueb_output_pkg_filename = uebhelper.StringSettings.ueb_output_model_package_default_filename
ueb_output_pkg_file = os.path.join(destination_dir, ueb_output_pkg_filename)
bytes_to_read = 16 * 1024
try:
with open(ueb_output_pkg_file, 'wb') as file_obj:
while True:
data = service_call_results.read(bytes_to_read)
if not data:
break
file_obj.write(data)
except Exception as e:
log.error(source + 'Failed to save ueb model output package zip file to '
'temporary location for model package dataset ID: %s \n '
'Exception: %s' % (model_pkg_dataset_id, e))
pass
return # no need to show error to the user as this a background operation
log.info(source + 'ueb model output package zip file was saved to temporary location '
'for model package dataset ID: %s' % model_pkg_dataset_id)
# access the input model pkg zip file
model_pkg_dataset = uebhelper.get_package(model_pkg_dataset_id)
# TODO: Before using model_pk_dataset check that it is not None
model_pkg_resource_zip_file = model_pkg_dataset['resources'][0]
# replace the '%3A' in the file url with ':' to get the correct folder name in the file system
pkg_zip_file_url = model_pkg_resource_zip_file['url']
pkg_zip_file_url = pkg_zip_file_url.replace('%3A', ':')
sting_to_search_in_file_url = 'storage/f/'
search_string_index = pkg_zip_file_url.find(sting_to_search_in_file_url)
file_path_start_index = search_string_index + len(sting_to_search_in_file_url)
original_zip_file_path = pkg_zip_file_url[file_path_start_index:]
original_zip_file_path = os.path.join('/', 'var', 'lib', 'ckan', 'default', 'pairtree_root', 'de', 'fa', 'ul', 't', 'obj', original_zip_file_path)
'''
open the original input zip file in the append mode and then
read the output pkg zip file and append it to the original zip file
'''
is_merge_successful = False
try:
with zip.ZipFile(original_zip_file_path, 'a') as orig_file_obj:
zip_file_to_merge = zip.ZipFile(ueb_output_pkg_file, 'r')
for fname in zip_file_to_merge.namelist():
orig_file_obj.writestr(fname, zip_file_to_merge.open(fname).read())
is_merge_successful = True
except Exception as e:
log.error(source + 'Failed to merge output model pkg zip file with the input model pkg zip file '
'for model package dataset ID: %s \n '
'Exception: %s' % (model_pkg_dataset_id, e))
pass
# update the model package dataset package_type to complete
if is_merge_successful:
data_dict = {'package_run_status': 'Output package available', 'package_type': u'Complete'}
else:
data_dict = {'package_run_status': 'Success'} # TODO: set to 'Success' for testing. Finally needs to be set "Merge failed"
update_msg = 'system auto updated ueb package dataset'
background_task = True
try:
updated_package = uebhelper.update_package(model_pkg_dataset_id, data_dict, update_msg, background_task)
log.info(source + 'UEB model package dataset was updated as a result of '
'receiving model output package for dataset:%s' % updated_package['name'])
except Exception as e:
log.error(source + 'Failed to update UEB model package dataset after '
'receiving model input package for dataset ID:%s \n'
'Exception: %s' % (model_pkg_dataset_id, e))
pass
def _save_ueb_package_as_dataset_obsolete(service_call_results, model_config_dataset_id):
source = 'uebpackage.tasks._save_ueb_package_as_dataset():'
ckan_default_dir = uebhelper.StringSettings.ckan_user_session_temp_dir # '/tmp/ckan'
# get the matching model configuration dataset object
model_config_dataset_obj = base.model.Package.get(model_config_dataset_id)
model_config_dataset_title = model_config_dataset_obj.title
model_config_dataset_owner_org = model_config_dataset_obj.owner_org
model_config_dataset_author = model_config_dataset_obj.author
# create a directory for saving the file
# this will be a dir in the form of: /tmp/ckan/{random_id}
random_id = base.model.types.make_uuid()
destination_dir = os.path.join(ckan_default_dir, random_id)
os.makedirs(destination_dir)
model_pkg_filename = uebhelper.StringSettings.ueb_input_model_package_default_filename # 'ueb_model_pkg.zip'
model_pkg_file = os.path.join(destination_dir, model_pkg_filename)
bytes_to_read = 16 * 1024
try:
with open(model_pkg_file, 'wb') as file_obj:
while True:
data = service_call_results.read(bytes_to_read)
if not data:
break
file_obj.write(data)
except Exception as e:
log.error(source + 'Failed to save the ueb_package zip file to temporary '
'location for UEB model configuration dataset ID: %s \n '
'Exception: %s' % (model_config_dataset_id, e))
raise e
log.info(source + 'ueb_package zip file was saved to temporary location for '
'UEB model configuration dataset ID: %s' % model_config_dataset_id)
# upload the file to CKAN file store
resource_metadata = _upload_file(model_pkg_file)
if resource_metadata:
log.info(source + 'UEB model package zip file was uploaded for model configuration dataset ID:%s' % model_config_dataset_id)
else:
log.error(source + 'Failed to upload UEB model package zip file '
'for model configuration dataset ID: %s' % model_config_dataset_id)
return
# retrieve some of the file meta data
resource_url = resource_metadata.get('_label') # this will return datetime stamp/filename
resource_url = '/storage/f/' + resource_url
if resource_url.startswith('/'):
resource_url = base.config.get('ckan.site_url', '').rstrip('/') + resource_url
else:
resource_url = base.config.get('ckan.site_url', '') + resource_url
resource_created_date = resource_metadata.get('_creation_date')
resource_name = resource_metadata.get('filename_original')
resource_size = resource_metadata.get('_content_length')
# add the uploaded ueb model pkg data file as a resource to the dataset
resource_create_action = tk.get_action('resource_create')
context = {'model': base.model, 'session': base.model.Session, 'save': 'save'}
user = uebhelper.get_site_user()
context['user'] = user.get('name')
context['ignore_auth'] = True
context['validate'] = False
# create a package
package_create_action = tk.get_action('package_create')
# create unique package name using the current time stamp as a postfix to any package name
unique_postfix = datetime.now().isoformat().replace(':', '-').replace('.', '-').lower()
pkg_title = model_config_dataset_title
data_dict = {
'name': 'model_package_' + unique_postfix, # this needs to be unique as required by DB
'type': 'model-package', # dataset type as defined in custom dataset plugin
'title': pkg_title,
'owner_org': model_config_dataset_owner_org,
'author': model_config_dataset_author,
'notes': 'UEB model package',
'pkg_model_name': 'UEB',
'model_version': '1.0',
'north_extent': '',
'south_extent': '',
'east_extent': '',
'west_extent': '',
'simulation_start_day': '',
'simulation_end_day': '',
'time_step': '',
'package_type': u'Input',
'package_run_status': 'Not yet submitted',
'package_run_job_id': '',
'dataset_type': 'model-package'
}
context = {'model': base.model, 'session': base.model.Session, 'ignore_auth': True, 'user': user.get('name'), 'save': 'save'}
try:
uebhelper.register_translator() # this is needed since we are creating a package in a background operation
pkg_dict = package_create_action(context, data_dict)
log.info(source + 'A new dataset was created for UEB input model package with name: %s' % data_dict['title'])
except Exception as e:
log.error(source + 'Failed to create a new dataset for ueb input model package for'
' the related model configuration dataset title: %s \n Exception: %s' % (pkg_title, e))
return
pkg_id = pkg_dict['id']
#A value for key 'message" in the context must be set to
# avoid a bug in line#192 of the update.py under lib/action
# without this, a TypeError will occur (TypeError:No object(name:translator) has been registered for this thread)
context['message'] = 'Auto uploaded model package '
data_dict = {
"package_id": pkg_id, # id of the package/dataset to which the resource needs to be added
"url": resource_url,
"name": resource_name,
"created": resource_created_date,
"format": 'zip',
"size": resource_size,
"description": 'UEB model package'
}
is_resource_add_success = False
try:
resource_create_action(context, data_dict)
is_resource_add_success = True
log.info(source + 'UEB model package was added as a resource to the '
'newly created model package dataset with ID:%s' % pkg_id)
except Exception as e:
log.error(source + 'Failed to add UEB model package as a resource to '
'the newly created model package dataset with ID:%s \nException: %s' % (pkg_id, e))
pass
# link this newly created model package dataset to the model configuration dataset
package_relationship_create_action = tk.get_action('package_relationship_create')
data_dict = {'subject': pkg_id, 'object': model_config_dataset_id, 'type': 'links_to'}
package_relationship_create_action(context, data_dict)
# update the related model configuration dataset to show that the package is available
if is_resource_add_success:
# update the related dataset
data_dict = {'package_availability': 'Available'}
update_msg = 'system auto updated ueb package dataset'
background_task = True
try:
updated_package = uebhelper.update_package(model_config_dataset_id, data_dict, update_msg, background_task)
log.info(source + 'UEB model configuration dataset was updated as a result of '
'receiving model input package for dataset:%s' % updated_package['name'])
except Exception as e:
log.error(source + 'Failed to update UEB model configuration dataset after '
'receiving model input package for dataset ID:%s \n'
'Exception: %s' % (model_config_dataset_id, e))
pass
return
def _save_ueb_package_as_dataset(service_call_results, model_config_dataset_id):
source = 'uebpackage.tasks._save_ueb_package_as_dataset():'
ckan_default_dir = uebhelper.StringSettings.ckan_user_session_temp_dir # '/tmp/ckan'
# get the matching model configuration dataset object
model_config_dataset_obj = base.model.Package.get(model_config_dataset_id)
model_config_dataset_title = model_config_dataset_obj.title
model_config_dataset_owner_org = model_config_dataset_obj.owner_org
model_config_dataset_author = model_config_dataset_obj.author
# create a directory for saving the file
# this will be a dir in the form of: /tmp/ckan/{random_id}
random_id = base.model.types.make_uuid()
destination_dir = os.path.join(ckan_default_dir, random_id)
os.makedirs(destination_dir)
model_pkg_filename = uebhelper.StringSettings.ueb_input_model_package_default_filename # 'ueb_model_pkg.zip'
model_pkg_file = os.path.join(destination_dir, model_pkg_filename)
bytes_to_read = 16 * 1024
try:
with open(model_pkg_file, 'wb') as file_obj:
while True:
data = service_call_results.read(bytes_to_read)
if not data:
break
file_obj.write(data)
except Exception as e:
log.error(source + 'Failed to save the ueb_package zip file to temporary '
'location for UEB model configuration dataset ID: %s \n '
'Exception: %s' % (model_config_dataset_id, e))
raise e
log.info(source + 'ueb_package zip file was saved to temporary location for '
'UEB model configuration dataset ID: %s' % model_config_dataset_id)
# upload the file to CKAN file store
# resource_metadata = _upload_file(model_pkg_file)
# if resource_metadata:
# log.info(source + 'UEB model package zip file was uploaded for model configuration dataset ID:%s' % model_config_dataset_id)
# else:
# log.error(source + 'Failed to upload UEB model package zip file '
# 'for model configuration dataset ID: %s' % model_config_dataset_id)
# return
#
# # retrieve some of the file meta data
# resource_url = resource_metadata.get('_label') # this will return datetime stamp/filename
#
# resource_url = '/storage/f/' + resource_url
# if resource_url.startswith('/'):
# resource_url = base.config.get('ckan.site_url', '').rstrip('/') + resource_url
# else:
# resource_url = base.config.get('ckan.site_url', '') + resource_url
#
# resource_created_date = resource_metadata.get('_creation_date')
# resource_name = resource_metadata.get('filename_original')
# resource_size = resource_metadata.get('_content_length')
#
# # add the uploaded ueb model pkg data file as a resource to the dataset
# resource_create_action = tk.get_action('resource_create')
# context = {'model': base.model, 'session': base.model.Session, 'save': 'save'}
# user = uebhelper.get_site_user()
# context['user'] = user.get('name')
# context['ignore_auth'] = True
# context['validate'] = False
user = uebhelper.get_site_user()
# create a package
package_create_action = tk.get_action('package_create')
# create unique package name using the current time stamp as a postfix to any package name
unique_postfix = datetime.now().isoformat().replace(':', '-').replace('.', '-').lower()
pkg_title = model_config_dataset_title
data_dict = {
'name': 'model_package_' + unique_postfix, # this needs to be unique as required by DB
'type': 'model-package', # dataset type as defined in custom dataset plugin
'title': pkg_title,
'owner_org': model_config_dataset_owner_org,
'author': model_config_dataset_author,
'notes': 'UEB model package',
'pkg_model_name': 'UEB',
'model_version': '1.0',
'north_extent': '',
'south_extent': '',
'east_extent': '',
'west_extent': '',
'simulation_start_day': '',
'simulation_end_day': '',
'time_step': '',
'package_type': u'Input',
'package_run_status': 'Not yet submitted',
'package_run_job_id': '',
'dataset_type': 'model-package'
}
context = {'model': base.model, 'session': base.model.Session, 'ignore_auth': True, 'user': user.get('name'), 'save': 'save'}
try:
uebhelper.register_translator() # this is needed since we are creating a package in a background operation
pkg_dict = package_create_action(context, data_dict)
log.info(source + 'A new dataset was created for UEB input model package with name: %s' % data_dict['title'])
except Exception as e:
log.error(source + 'Failed to create a new dataset for ueb input model package for'
' the related model configuration dataset title: %s \n Exception: %s' % (pkg_title, e))
raise e
pkg_id = pkg_dict['id']
if not 'resources' in pkg_dict:
pkg_dict['resources'] = []
file_name = munge.munge_filename(model_pkg_filename)
resource = {'url': file_name, 'url_type': 'upload'}
upload = uploader.ResourceUpload(resource)
upload.filename = file_name
upload.upload_file = open(model_pkg_file, 'r')
data_dict = {'format': 'zip', 'name': file_name, 'url': file_name, 'url_type': 'upload'}
pkg_dict['resources'].append(data_dict)
try:
context['defer_commit'] = True
context['use_cache'] = False
# update the package
package_update_action = tk.get_action('package_update')
package_update_action(context, pkg_dict)
context.pop('defer_commit')
except Exception as e:
log.error(source + ' Failed to update the new dataset for adding the input model package zip file as'
' a resource.\n Exception: %s' % e)
raise e
# link this newly created model package dataset to the model configuration dataset
package_relationship_create_action = tk.get_action('package_relationship_create')
data_dict = {'subject': pkg_id, 'object': model_config_dataset_id, 'type': 'links_to'}
package_relationship_create_action(context, data_dict)
# Get out resource_id resource from model as it will not appear in
# package_show until after commit
upload.upload(context['package'].resources[-1].id, uploader.get_max_resource_size())
base.model.repo.commit()
# update the related model configuration dataset to show that the package is available
data_dict = {'package_availability': 'Available'}
update_msg = 'system auto updated ueb package dataset'
background_task = True
try:
updated_package = uebhelper.update_package(model_config_dataset_id, data_dict, update_msg, background_task)
log.info(source + 'UEB model configuration dataset was updated as a result of '
'receiving model input package for dataset:%s' % updated_package['name'])
except Exception as e:
log.error(source + 'Failed to update UEB model configuration dataset after '
'receiving model input package for dataset ID:%s \n'
'Exception: %s' % (model_config_dataset_id, e))
raise e
def check_ueb_request_process_status():
source = 'uebpackage.tasks.check_ueb_request_process_status():'
global service_host_address
service_request_api_url = uebhelper.StringSettings.app_server_api_check_ueb_package_build_status
connection = httplib.HTTPConnection(service_host_address)
job_status_processing = uebhelper.StringSettings.app_server_job_status_processing
model_config_datasets_with_status_processing = _get_model_configuration_datasets_by_processing_status(
job_status_processing)
job_status_in_queue = uebhelper.StringSettings.app_server_job_status_in_queue
model_config_datasets_with_status_in_queue = _get_model_configuration_datasets_by_processing_status(
job_status_in_queue)
# merge the 2 lists
model_config_datasets_with_status_processing_or_in_queue = model_config_datasets_with_status_processing + \
model_config_datasets_with_status_in_queue
if len(model_config_datasets_with_status_processing_or_in_queue) == 0:
log.info(source + "No UEB model configuration dataset has a status of %s or %s at this time"
% (job_status_processing, job_status_in_queue))
else:
log.info(source + "Number of UEB model configuration datasets build status of %s or %s at this time is:%s"
% (job_status_processing, job_status_in_queue,
len(model_config_datasets_with_status_processing_or_in_queue)))
for dataset in model_config_datasets_with_status_processing_or_in_queue:
#pkg_process_job_id = h.get_pkg_dict_extra(dataset, 'package_build_request_job_id')
# in CKAN 2.2 additional metatadata elements added to the dataset of specific type
# seems not part of the 'extra' key of the dataset dict. These additional metadata elements
# are now part of the dataset main dictionary
pkg_process_job_id = dataset.get('package_build_request_job_id', None)
#pkg_current_processing_status = h.get_pkg_dict_extra(dataset, 'processing_status')
pkg_current_processing_status = dataset.get('processing_status', None)
if pkg_process_job_id and pkg_current_processing_status:
dataset_id = dataset['id']
service_request_url = service_request_api_url + '?packageID=' + pkg_process_job_id
connection.request('GET', service_request_url)
service_call_results = connection.getresponse()
if service_call_results.status == httplib.OK:
request_processing_status = service_call_results.read()
log.info(source + 'UEB model package build status as returned from App server '
'for PackageJobID:%s is %s' % (pkg_process_job_id, request_processing_status))
else:
request_processing_status = uebhelper.StringSettings.app_server_job_status_error
log.error(source + 'HTTP status %d returned from App server when checking '
'status for PackageJobID:%s' % (service_call_results.status, pkg_process_job_id))
connection.close()
# update the dataset if the status has changed
if pkg_current_processing_status != request_processing_status:
data_dict = {'processing_status': request_processing_status}
uebhelper.update_package(dataset_id, data_dict, backgroundTask=True)
else:
log.error(source + "Either the metadata element 'package_build_request_job_id' "
"or the 'processing_status' was not found "
"for dataset ID:%s" % dataset['id'])
def retrieve_ueb_packages():
source = 'uebpackage.tasks.retrieve_ueb_packages():'
global service_host_address
service_request_api_url = uebhelper.StringSettings.app_server_api_get_ueb_package_url
connection = httplib.HTTPConnection(service_host_address)
job_status_complete = uebhelper.StringSettings.app_server_job_status_success
model_config_datasets_with_status_complete = _get_model_configuration_datasets_by_processing_status(
job_status_complete)
if len(model_config_datasets_with_status_complete) == 0:
log.info(source + "No UEB model configuration dataset has a status of %s at this time" % job_status_complete)
else:
log.info(source + "Number of UEB model configuration datasets with build status of %s at this time is:%s"
% (job_status_complete, len(model_config_datasets_with_status_complete)))
for dataset in model_config_datasets_with_status_complete:
pkg_availability_status = h.get_pkg_dict_extra(dataset, 'package_availability')
if pkg_availability_status == uebhelper.StringSettings.app_server_job_status_package_available:
continue
pkg_process_job_id = h.get_pkg_dict_extra(dataset, 'package_build_request_job_id')
dataset_id = dataset['id']
package_availability_status = h.get_pkg_dict_extra(dataset, 'package_availability')
# if package is already available or error has been logged for package retrieval then skip this dataset
if package_availability_status == uebhelper.StringSettings.app_server_job_status_package_available or \
package_availability_status == uebhelper.StringSettings.app_server_job_status_error:
continue
service_request_url = service_request_api_url + '?packageID=' + pkg_process_job_id
connection.request('GET', service_request_url)
service_call_results = connection.getresponse()
if service_call_results.status == httplib.OK:
log.info(source + 'UEB model package was received from App server for PackageJobID:%s' % pkg_process_job_id)
try:
_save_ueb_package_as_dataset(service_call_results, dataset_id)
pkg_availability_status = uebhelper.StringSettings.app_server_job_status_package_available
except Exception as e:
log.error(source + 'Failed to save ueb model package as a new dataset '
'for model configuration dataset ID:%s\nException:%s' % (dataset_id, e))
pkg_availability_status = uebhelper.StringSettings.app_server_job_status_error
else:
log.error(source + 'HTTP status %d returned from App server when retrieving '
'UEB model package for PackageJobID:'
'%s' % (service_call_results.status, pkg_process_job_id))
pkg_availability_status = uebhelper.StringSettings.app_server_job_status_error
connection.close()
# update the resource processing status
# update the related dataset
data_dict = {'package_availability': pkg_availability_status}
update_msg = 'system auto updated ueb package dataset'
background_task = True
try:
updated_package = uebhelper.update_package(dataset_id, data_dict, update_msg, background_task)
log.info(source + 'UEB model configuration dataset was updated as a result of '
'receiving model input package for dataset:%s' % updated_package['name'])
except Exception as e:
log.error(source + 'Failed to update UEB model configuration dataset after '
'receiving model input package for dataset ID:%s \n'
'Exception: %s' % (dataset_id, e))
pass
return
