def _save_ueb_package_as_dataset_obsolete(service_call_results,
model_config_dataset_id):
source = 'uebpackage.tasks._save_ueb_package_as_dataset():'
ckan_default_dir = uebhelper.StringSettings.ckan_user_session_temp_dir # '/tmp/ckan'
# get the matching model configuration dataset object
model_config_dataset_obj = base.model.Package.get(model_config_dataset_id)
model_config_dataset_title = model_config_dataset_obj.title
model_config_dataset_owner_org = model_config_dataset_obj.owner_org
model_config_dataset_author = model_config_dataset_obj.author
# create a directory for saving the file
# this will be a dir in the form of: /tmp/ckan/{random_id}
random_id = base.model.types.make_uuid()
destination_dir = os.path.join(ckan_default_dir, random_id)
os.makedirs(destination_dir)
model_pkg_filename = uebhelper.StringSettings.ueb_input_model_package_default_filename # 'ueb_model_pkg.zip'
model_pkg_file = os.path.join(destination_dir, model_pkg_filename)
bytes_to_read = 16 * 1024
try:
with open(model_pkg_file, 'wb') as file_obj:
while True:
data = service_call_results.read(bytes_to_read)
if not data:
break
file_obj.write(data)
except Exception as e:
log.error(source +
'Failed to save the ueb_package zip file to temporary '
'location for UEB model configuration dataset ID: %s \n '
'Exception: %s' % (model_config_dataset_id, e))
raise e
log.info(source +
'ueb_package zip file was saved to temporary location for '
'UEB model configuration dataset ID: %s' %
model_config_dataset_id)
# upload the file to CKAN file store
resource_metadata = _upload_file(model_pkg_file)
if resource_metadata:
log.info(
source +
'UEB model package zip file was uploaded for model configuration dataset ID:%s'
% model_config_dataset_id)
else:
log.error(source + 'Failed to upload UEB model package zip file '
'for model configuration dataset ID: %s' %
model_config_dataset_id)
return
# retrieve some of the file meta data
resource_url = resource_metadata.get(
'_label') # this will return datetime stamp/filename
resource_url = '/storage/f/' + resource_url
if resource_url.startswith('/'):
resource_url = base.config.get('ckan.site_url',
'').rstrip('/') + resource_url
else:
resource_url = base.config.get('ckan.site_url', '') + resource_url
resource_created_date = resource_metadata.get('_creation_date')
resource_name = resource_metadata.get('filename_original')
resource_size = resource_metadata.get('_content_length')
# add the uploaded ueb model pkg data file as a resource to the dataset
resource_create_action = tk.get_action('resource_create')
context = {
'model': base.model,
'session': base.model.Session,
'save': 'save'
}
user = uebhelper.get_site_user()
context['user'] = user.get('name')
context['ignore_auth'] = True
context['validate'] = False
# create a package
package_create_action = tk.get_action('package_create')
# create unique package name using the current time stamp as a postfix to any package name
unique_postfix = datetime.now().isoformat().replace(':', '-').replace(
'.', '-').lower()
pkg_title = model_config_dataset_title
data_dict = {
'name': 'model_package_' +
unique_postfix, # this needs to be unique as required by DB
'type':
'model-package', # dataset type as defined in custom dataset plugin
'title': pkg_title,
'owner_org': model_config_dataset_owner_org,
'author': model_config_dataset_author,
'notes': 'UEB model package',
'pkg_model_name': 'UEB',
'model_version': '1.0',
'north_extent': '',
'south_extent': '',
'east_extent': '',
'west_extent': '',
'simulation_start_day': '',
'simulation_end_day': '',
'time_step': '',
'package_type': u'Input',
'package_run_status': 'Not yet submitted',
'package_run_job_id': '',
'dataset_type': 'model-package'
}
context = {
'model': base.model,
'session': base.model.Session,
'ignore_auth': True,
'user': user.get('name'),
'save': 'save'
}
try:
uebhelper.register_translator(
) # this is needed since we are creating a package in a background operation
pkg_dict = package_create_action(context, data_dict)
log.info(
source +
'A new dataset was created for UEB input model package with name: %s'
% data_dict['title'])
except Exception as e:
log.error(
source +
'Failed to create a new dataset for ueb input model package for'
' the related model configuration dataset title: %s \n Exception: %s'
% (pkg_title, e))
return
pkg_id = pkg_dict['id']
#A value for key 'message" in the context must be set to
# avoid a bug in line#192 of the update.py under lib/action
# without this, a TypeError will occur (TypeError:No object(name:translator) has been registered for this thread)
context['message'] = 'Auto uploaded model package '
data_dict = {
"package_id":
pkg_id, # id of the package/dataset to which the resource needs to be added
"url": resource_url,
"name": resource_name,
"created": resource_created_date,
"format": 'zip',
"size": resource_size,
"description": 'UEB model package'
}
is_resource_add_success = False
try:
resource_create_action(context, data_dict)
is_resource_add_success = True
log.info(source + 'UEB model package was added as a resource to the '
'newly created model package dataset with ID:%s' % pkg_id)
except Exception as e:
log.error(
source + 'Failed to add UEB model package as a resource to '
'the newly created model package dataset with ID:%s \nException: %s'
% (pkg_id, e))
pass
# link this newly created model package dataset to the model configuration dataset
package_relationship_create_action = tk.get_action(
'package_relationship_create')
data_dict = {
'subject': pkg_id,
'object': model_config_dataset_id,
'type': 'links_to'
}
package_relationship_create_action(context, data_dict)
# update the related model configuration dataset to show that the package is available
if is_resource_add_success:
# update the related dataset
data_dict = {'package_availability': 'Available'}
update_msg = 'system auto updated ueb package dataset'
background_task = True
try:
updated_package = uebhelper.update_package(model_config_dataset_id,
data_dict, update_msg,
background_task)
log.info(
source +
'UEB model configuration dataset was updated as a result of '
'receiving model input package for dataset:%s' %
updated_package['name'])
except Exception as e:
log.error(source +
'Failed to update UEB model configuration dataset after '
'receiving model input package for dataset ID:%s \n'
'Exception: %s' % (model_config_dataset_id, e))
pass
return
def _save_ueb_package_as_dataset_obsolete(service_call_results, model_config_dataset_id):
source = 'uebpackage.tasks._save_ueb_package_as_dataset():'
ckan_default_dir = uebhelper.StringSettings.ckan_user_session_temp_dir # '/tmp/ckan'
# get the matching model configuration dataset object
model_config_dataset_obj = base.model.Package.get(model_config_dataset_id)
model_config_dataset_title = model_config_dataset_obj.title
model_config_dataset_owner_org = model_config_dataset_obj.owner_org
model_config_dataset_author = model_config_dataset_obj.author
# create a directory for saving the file
# this will be a dir in the form of: /tmp/ckan/{random_id}
random_id = base.model.types.make_uuid()
destination_dir = os.path.join(ckan_default_dir, random_id)
os.makedirs(destination_dir)
model_pkg_filename = uebhelper.StringSettings.ueb_input_model_package_default_filename # 'ueb_model_pkg.zip'
model_pkg_file = os.path.join(destination_dir, model_pkg_filename)
bytes_to_read = 16 * 1024
try:
with open(model_pkg_file, 'wb') as file_obj:
while True:
data = service_call_results.read(bytes_to_read)
if not data:
break
file_obj.write(data)
except Exception as e:
log.error(source + 'Failed to save the ueb_package zip file to temporary '
'location for UEB model configuration dataset ID: %s \n '
'Exception: %s' % (model_config_dataset_id, e))
raise e
log.info(source + 'ueb_package zip file was saved to temporary location for '
'UEB model configuration dataset ID: %s' % model_config_dataset_id)
# upload the file to CKAN file store
resource_metadata = _upload_file(model_pkg_file)
if resource_metadata:
log.info(source + 'UEB model package zip file was uploaded for model configuration dataset ID:%s' % model_config_dataset_id)
else:
log.error(source + 'Failed to upload UEB model package zip file '
'for model configuration dataset ID: %s' % model_config_dataset_id)
return
# retrieve some of the file meta data
resource_url = resource_metadata.get('_label') # this will return datetime stamp/filename
resource_url = '/storage/f/' + resource_url
if resource_url.startswith('/'):
resource_url = base.config.get('ckan.site_url', '').rstrip('/') + resource_url
else:
resource_url = base.config.get('ckan.site_url', '') + resource_url
resource_created_date = resource_metadata.get('_creation_date')
resource_name = resource_metadata.get('filename_original')
resource_size = resource_metadata.get('_content_length')
# add the uploaded ueb model pkg data file as a resource to the dataset
resource_create_action = tk.get_action('resource_create')
context = {'model': base.model, 'session': base.model.Session, 'save': 'save'}
user = uebhelper.get_site_user()
context['user'] = user.get('name')
context['ignore_auth'] = True
context['validate'] = False
# create a package
package_create_action = tk.get_action('package_create')
# create unique package name using the current time stamp as a postfix to any package name
unique_postfix = datetime.now().isoformat().replace(':', '-').replace('.', '-').lower()
pkg_title = model_config_dataset_title
data_dict = {
'name': 'model_package_' + unique_postfix, # this needs to be unique as required by DB
'type': 'model-package', # dataset type as defined in custom dataset plugin
'title': pkg_title,
'owner_org': model_config_dataset_owner_org,
'author': model_config_dataset_author,
'notes': 'UEB model package',
'pkg_model_name': 'UEB',
'model_version': '1.0',
'north_extent': '',
'south_extent': '',
'east_extent': '',
'west_extent': '',
'simulation_start_day': '',
'simulation_end_day': '',
'time_step': '',
'package_type': u'Input',
'package_run_status': 'Not yet submitted',
'package_run_job_id': '',
'dataset_type': 'model-package'
}
context = {'model': base.model, 'session': base.model.Session, 'ignore_auth': True, 'user': user.get('name'), 'save': 'save'}
try:
uebhelper.register_translator() # this is needed since we are creating a package in a background operation
pkg_dict = package_create_action(context, data_dict)
log.info(source + 'A new dataset was created for UEB input model package with name: %s' % data_dict['title'])
except Exception as e:
log.error(source + 'Failed to create a new dataset for ueb input model package for'
' the related model configuration dataset title: %s \n Exception: %s' % (pkg_title, e))
return
pkg_id = pkg_dict['id']
#A value for key 'message" in the context must be set to
# avoid a bug in line#192 of the update.py under lib/action
# without this, a TypeError will occur (TypeError:No object(name:translator) has been registered for this thread)
context['message'] = 'Auto uploaded model package '
data_dict = {
"package_id": pkg_id, # id of the package/dataset to which the resource needs to be added
"url": resource_url,
"name": resource_name,
"created": resource_created_date,
"format": 'zip',
"size": resource_size,
"description": 'UEB model package'
}
is_resource_add_success = False
try:
resource_create_action(context, data_dict)
is_resource_add_success = True
log.info(source + 'UEB model package was added as a resource to the '
'newly created model package dataset with ID:%s' % pkg_id)
except Exception as e:
log.error(source + 'Failed to add UEB model package as a resource to '
'the newly created model package dataset with ID:%s \nException: %s' % (pkg_id, e))
pass
# link this newly created model package dataset to the model configuration dataset
package_relationship_create_action = tk.get_action('package_relationship_create')
data_dict = {'subject': pkg_id, 'object': model_config_dataset_id, 'type': 'links_to'}
package_relationship_create_action(context, data_dict)
# update the related model configuration dataset to show that the package is available
if is_resource_add_success:
# update the related dataset
data_dict = {'package_availability': 'Available'}
update_msg = 'system auto updated ueb package dataset'
background_task = True
try:
updated_package = uebhelper.update_package(model_config_dataset_id, data_dict, update_msg, background_task)
log.info(source + 'UEB model configuration dataset was updated as a result of '
'receiving model input package for dataset:%s' % updated_package['name'])
except Exception as e:
log.error(source + 'Failed to update UEB model configuration dataset after '
'receiving model input package for dataset ID:%s \n'
'Exception: %s' % (model_config_dataset_id, e))
pass
return
def _save_ueb_package_as_dataset(service_call_results,
model_config_dataset_id):
source = 'uebpackage.tasks._save_ueb_package_as_dataset():'
ckan_default_dir = uebhelper.StringSettings.ckan_user_session_temp_dir # '/tmp/ckan'
# get the matching model configuration dataset object
model_config_dataset_obj = base.model.Package.get(model_config_dataset_id)
model_config_dataset_title = model_config_dataset_obj.title
model_config_dataset_owner_org = model_config_dataset_obj.owner_org
model_config_dataset_author = model_config_dataset_obj.author
# create a directory for saving the file
# this will be a dir in the form of: /tmp/ckan/{random_id}
random_id = base.model.types.make_uuid()
destination_dir = os.path.join(ckan_default_dir, random_id)
os.makedirs(destination_dir)
model_pkg_filename = uebhelper.StringSettings.ueb_input_model_package_default_filename # 'ueb_model_pkg.zip'
model_pkg_file = os.path.join(destination_dir, model_pkg_filename)
bytes_to_read = 16 * 1024
try:
with open(model_pkg_file, 'wb') as file_obj:
while True:
data = service_call_results.read(bytes_to_read)
if not data:
break
file_obj.write(data)
except Exception as e:
log.error(source +
'Failed to save the ueb_package zip file to temporary '
'location for UEB model configuration dataset ID: %s \n '
'Exception: %s' % (model_config_dataset_id, e))
raise e
log.info(source +
'ueb_package zip file was saved to temporary location for '
'UEB model configuration dataset ID: %s' %
model_config_dataset_id)
# upload the file to CKAN file store
# resource_metadata = _upload_file(model_pkg_file)
# if resource_metadata:
# log.info(source + 'UEB model package zip file was uploaded for model configuration dataset ID:%s' % model_config_dataset_id)
# else:
# log.error(source + 'Failed to upload UEB model package zip file '
# 'for model configuration dataset ID: %s' % model_config_dataset_id)
# return
#
# # retrieve some of the file meta data
# resource_url = resource_metadata.get('_label') # this will return datetime stamp/filename
#
# resource_url = '/storage/f/' + resource_url
# if resource_url.startswith('/'):
# resource_url = base.config.get('ckan.site_url', '').rstrip('/') + resource_url
# else:
# resource_url = base.config.get('ckan.site_url', '') + resource_url
#
# resource_created_date = resource_metadata.get('_creation_date')
# resource_name = resource_metadata.get('filename_original')
# resource_size = resource_metadata.get('_content_length')
#
# # add the uploaded ueb model pkg data file as a resource to the dataset
# resource_create_action = tk.get_action('resource_create')
# context = {'model': base.model, 'session': base.model.Session, 'save': 'save'}
# user = uebhelper.get_site_user()
# context['user'] = user.get('name')
# context['ignore_auth'] = True
# context['validate'] = False
user = uebhelper.get_site_user()
# create a package
package_create_action = tk.get_action('package_create')
# create unique package name using the current time stamp as a postfix to any package name
unique_postfix = datetime.now().isoformat().replace(':', '-').replace(
'.', '-').lower()
pkg_title = model_config_dataset_title
data_dict = {
'name': 'model_package_' +
unique_postfix, # this needs to be unique as required by DB
'type':
'model-package', # dataset type as defined in custom dataset plugin
'title': pkg_title,
'owner_org': model_config_dataset_owner_org,
'author': model_config_dataset_author,
'notes': 'UEB model package',
'pkg_model_name': 'UEB',
'model_version': '1.0',
'north_extent': '',
'south_extent': '',
'east_extent': '',
'west_extent': '',
'simulation_start_day': '',
'simulation_end_day': '',
'time_step': '',
'package_type': u'Input',
'package_run_status': 'Not yet submitted',
'package_run_job_id': '',
'dataset_type': 'model-package'
}
context = {
'model': base.model,
'session': base.model.Session,
'ignore_auth': True,
'user': user.get('name'),
'save': 'save'
}
try:
uebhelper.register_translator(
) # this is needed since we are creating a package in a background operation
pkg_dict = package_create_action(context, data_dict)
log.info(
source +
'A new dataset was created for UEB input model package with name: %s'
% data_dict['title'])
except Exception as e:
log.error(
source +
'Failed to create a new dataset for ueb input model package for'
' the related model configuration dataset title: %s \n Exception: %s'
% (pkg_title, e))
raise e
pkg_id = pkg_dict['id']
if not 'resources' in pkg_dict:
pkg_dict['resources'] = []
file_name = munge.munge_filename(model_pkg_filename)
resource = {'url': file_name, 'url_type': 'upload'}
upload = uploader.ResourceUpload(resource)
upload.filename = file_name
upload.upload_file = open(model_pkg_file, 'r')
data_dict = {
'format': 'zip',
'name': file_name,
'url': file_name,
'url_type': 'upload'
}
pkg_dict['resources'].append(data_dict)
try:
context['defer_commit'] = True
context['use_cache'] = False
# update the package
package_update_action = tk.get_action('package_update')
package_update_action(context, pkg_dict)
context.pop('defer_commit')
except Exception as e:
log.error(
source +
' Failed to update the new dataset for adding the input model package zip file as'
' a resource.\n Exception: %s' % e)
raise e
# link this newly created model package dataset to the model configuration dataset
package_relationship_create_action = tk.get_action(
'package_relationship_create')
data_dict = {
'subject': pkg_id,
'object': model_config_dataset_id,
'type': 'links_to'
}
package_relationship_create_action(context, data_dict)
# Get out resource_id resource from model as it will not appear in
# package_show until after commit
upload.upload(context['package'].resources[-1].id,
uploader.get_max_resource_size())
base.model.repo.commit()
# update the related model configuration dataset to show that the package is available
data_dict = {'package_availability': 'Available'}
update_msg = 'system auto updated ueb package dataset'
background_task = True
try:
updated_package = uebhelper.update_package(model_config_dataset_id,
data_dict, update_msg,
background_task)
log.info(source +
'UEB model configuration dataset was updated as a result of '
'receiving model input package for dataset:%s' %
updated_package['name'])
except Exception as e:
log.error(source +
'Failed to update UEB model configuration dataset after '
'receiving model input package for dataset ID:%s \n'
'Exception: %s' % (model_config_dataset_id, e))
raise e
def _save_ueb_package_as_dataset(service_call_results, model_config_dataset_id):
source = 'uebpackage.tasks._save_ueb_package_as_dataset():'
ckan_default_dir = uebhelper.StringSettings.ckan_user_session_temp_dir # '/tmp/ckan'
# get the matching model configuration dataset object
model_config_dataset_obj = base.model.Package.get(model_config_dataset_id)
model_config_dataset_title = model_config_dataset_obj.title
model_config_dataset_owner_org = model_config_dataset_obj.owner_org
model_config_dataset_author = model_config_dataset_obj.author
# create a directory for saving the file
# this will be a dir in the form of: /tmp/ckan/{random_id}
random_id = base.model.types.make_uuid()
destination_dir = os.path.join(ckan_default_dir, random_id)
os.makedirs(destination_dir)
model_pkg_filename = uebhelper.StringSettings.ueb_input_model_package_default_filename # 'ueb_model_pkg.zip'
model_pkg_file = os.path.join(destination_dir, model_pkg_filename)
bytes_to_read = 16 * 1024
try:
with open(model_pkg_file, 'wb') as file_obj:
while True:
data = service_call_results.read(bytes_to_read)
if not data:
break
file_obj.write(data)
except Exception as e:
log.error(source + 'Failed to save the ueb_package zip file to temporary '
'location for UEB model configuration dataset ID: %s \n '
'Exception: %s' % (model_config_dataset_id, e))
raise e
log.info(source + 'ueb_package zip file was saved to temporary location for '
'UEB model configuration dataset ID: %s' % model_config_dataset_id)
# upload the file to CKAN file store
# resource_metadata = _upload_file(model_pkg_file)
# if resource_metadata:
# log.info(source + 'UEB model package zip file was uploaded for model configuration dataset ID:%s' % model_config_dataset_id)
# else:
# log.error(source + 'Failed to upload UEB model package zip file '
# 'for model configuration dataset ID: %s' % model_config_dataset_id)
# return
#
# # retrieve some of the file meta data
# resource_url = resource_metadata.get('_label') # this will return datetime stamp/filename
#
# resource_url = '/storage/f/' + resource_url
# if resource_url.startswith('/'):
# resource_url = base.config.get('ckan.site_url', '').rstrip('/') + resource_url
# else:
# resource_url = base.config.get('ckan.site_url', '') + resource_url
#
# resource_created_date = resource_metadata.get('_creation_date')
# resource_name = resource_metadata.get('filename_original')
# resource_size = resource_metadata.get('_content_length')
#
# # add the uploaded ueb model pkg data file as a resource to the dataset
# resource_create_action = tk.get_action('resource_create')
# context = {'model': base.model, 'session': base.model.Session, 'save': 'save'}
# user = uebhelper.get_site_user()
# context['user'] = user.get('name')
# context['ignore_auth'] = True
# context['validate'] = False
user = uebhelper.get_site_user()
# create a package
package_create_action = tk.get_action('package_create')
# create unique package name using the current time stamp as a postfix to any package name
unique_postfix = datetime.now().isoformat().replace(':', '-').replace('.', '-').lower()
pkg_title = model_config_dataset_title
data_dict = {
'name': 'model_package_' + unique_postfix, # this needs to be unique as required by DB
'type': 'model-package', # dataset type as defined in custom dataset plugin
'title': pkg_title,
'owner_org': model_config_dataset_owner_org,
'author': model_config_dataset_author,
'notes': 'UEB model package',
'pkg_model_name': 'UEB',
'model_version': '1.0',
'north_extent': '',
'south_extent': '',
'east_extent': '',
'west_extent': '',
'simulation_start_day': '',
'simulation_end_day': '',
'time_step': '',
'package_type': u'Input',
'package_run_status': 'Not yet submitted',
'package_run_job_id': '',
'dataset_type': 'model-package'
}
context = {'model': base.model, 'session': base.model.Session, 'ignore_auth': True, 'user': user.get('name'), 'save': 'save'}
try:
uebhelper.register_translator() # this is needed since we are creating a package in a background operation
pkg_dict = package_create_action(context, data_dict)
log.info(source + 'A new dataset was created for UEB input model package with name: %s' % data_dict['title'])
except Exception as e:
log.error(source + 'Failed to create a new dataset for ueb input model package for'
' the related model configuration dataset title: %s \n Exception: %s' % (pkg_title, e))
raise e
pkg_id = pkg_dict['id']
if not 'resources' in pkg_dict:
pkg_dict['resources'] = []
file_name = munge.munge_filename(model_pkg_filename)
resource = {'url': file_name, 'url_type': 'upload'}
upload = uploader.ResourceUpload(resource)
upload.filename = file_name
upload.upload_file = open(model_pkg_file, 'r')
data_dict = {'format': 'zip', 'name': file_name, 'url': file_name, 'url_type': 'upload'}
pkg_dict['resources'].append(data_dict)
try:
context['defer_commit'] = True
context['use_cache'] = False
# update the package
package_update_action = tk.get_action('package_update')
package_update_action(context, pkg_dict)
context.pop('defer_commit')
except Exception as e:
log.error(source + ' Failed to update the new dataset for adding the input model package zip file as'
' a resource.\n Exception: %s' % e)
raise e
# link this newly created model package dataset to the model configuration dataset
package_relationship_create_action = tk.get_action('package_relationship_create')
data_dict = {'subject': pkg_id, 'object': model_config_dataset_id, 'type': 'links_to'}
package_relationship_create_action(context, data_dict)
# Get out resource_id resource from model as it will not appear in
# package_show until after commit
upload.upload(context['package'].resources[-1].id, uploader.get_max_resource_size())
base.model.repo.commit()
# update the related model configuration dataset to show that the package is available
data_dict = {'package_availability': 'Available'}
update_msg = 'system auto updated ueb package dataset'
background_task = True
try:
updated_package = uebhelper.update_package(model_config_dataset_id, data_dict, update_msg, background_task)
log.info(source + 'UEB model configuration dataset was updated as a result of '
'receiving model input package for dataset:%s' % updated_package['name'])
except Exception as e:
log.error(source + 'Failed to update UEB model configuration dataset after '
'receiving model input package for dataset ID:%s \n'
'Exception: %s' % (model_config_dataset_id, e))
raise e