def _save_ueb_package_as_dataset_obsolete(service_call_results, model_config_dataset_id): source = 'uebpackage.tasks._save_ueb_package_as_dataset():' ckan_default_dir = uebhelper.StringSettings.ckan_user_session_temp_dir # '/tmp/ckan' # get the matching model configuration dataset object model_config_dataset_obj = base.model.Package.get(model_config_dataset_id) model_config_dataset_title = model_config_dataset_obj.title model_config_dataset_owner_org = model_config_dataset_obj.owner_org model_config_dataset_author = model_config_dataset_obj.author # create a directory for saving the file # this will be a dir in the form of: /tmp/ckan/{random_id} random_id = base.model.types.make_uuid() destination_dir = os.path.join(ckan_default_dir, random_id) os.makedirs(destination_dir) model_pkg_filename = uebhelper.StringSettings.ueb_input_model_package_default_filename # 'ueb_model_pkg.zip' model_pkg_file = os.path.join(destination_dir, model_pkg_filename) bytes_to_read = 16 * 1024 try: with open(model_pkg_file, 'wb') as file_obj: while True: data = service_call_results.read(bytes_to_read) if not data: break file_obj.write(data) except Exception as e: log.error(source + 'Failed to save the ueb_package zip file to temporary ' 'location for UEB model configuration dataset ID: %s \n ' 'Exception: %s' % (model_config_dataset_id, e)) raise e log.info(source + 'ueb_package zip file was saved to temporary location for ' 'UEB model configuration dataset ID: %s' % model_config_dataset_id) # upload the file to CKAN file store resource_metadata = _upload_file(model_pkg_file) if resource_metadata: log.info( source + 'UEB model package zip file was uploaded for model configuration dataset ID:%s' % model_config_dataset_id) else: log.error(source + 'Failed to upload UEB model package zip file ' 'for model configuration dataset ID: %s' % model_config_dataset_id) return # retrieve some of the file meta data resource_url = resource_metadata.get( '_label') # this will return datetime stamp/filename resource_url = '/storage/f/' + resource_url if resource_url.startswith('/'): resource_url = base.config.get('ckan.site_url', '').rstrip('/') + resource_url else: resource_url = base.config.get('ckan.site_url', '') + resource_url resource_created_date = resource_metadata.get('_creation_date') resource_name = resource_metadata.get('filename_original') resource_size = resource_metadata.get('_content_length') # add the uploaded ueb model pkg data file as a resource to the dataset resource_create_action = tk.get_action('resource_create') context = { 'model': base.model, 'session': base.model.Session, 'save': 'save' } user = uebhelper.get_site_user() context['user'] = user.get('name') context['ignore_auth'] = True context['validate'] = False # create a package package_create_action = tk.get_action('package_create') # create unique package name using the current time stamp as a postfix to any package name unique_postfix = datetime.now().isoformat().replace(':', '-').replace( '.', '-').lower() pkg_title = model_config_dataset_title data_dict = { 'name': 'model_package_' + unique_postfix, # this needs to be unique as required by DB 'type': 'model-package', # dataset type as defined in custom dataset plugin 'title': pkg_title, 'owner_org': model_config_dataset_owner_org, 'author': model_config_dataset_author, 'notes': 'UEB model package', 'pkg_model_name': 'UEB', 'model_version': '1.0', 'north_extent': '', 'south_extent': '', 'east_extent': '', 'west_extent': '', 'simulation_start_day': '', 'simulation_end_day': '', 'time_step': '', 'package_type': u'Input', 'package_run_status': 'Not yet submitted', 'package_run_job_id': '', 'dataset_type': 'model-package' } context = { 'model': base.model, 'session': base.model.Session, 'ignore_auth': True, 'user': user.get('name'), 'save': 'save' } try: uebhelper.register_translator( ) # this is needed since we are creating a package in a background operation pkg_dict = package_create_action(context, data_dict) log.info( source + 'A new dataset was created for UEB input model package with name: %s' % data_dict['title']) except Exception as e: log.error( source + 'Failed to create a new dataset for ueb input model package for' ' the related model configuration dataset title: %s \n Exception: %s' % (pkg_title, e)) return pkg_id = pkg_dict['id'] #A value for key 'message" in the context must be set to # avoid a bug in line#192 of the update.py under lib/action # without this, a TypeError will occur (TypeError:No object(name:translator) has been registered for this thread) context['message'] = 'Auto uploaded model package ' data_dict = { "package_id": pkg_id, # id of the package/dataset to which the resource needs to be added "url": resource_url, "name": resource_name, "created": resource_created_date, "format": 'zip', "size": resource_size, "description": 'UEB model package' } is_resource_add_success = False try: resource_create_action(context, data_dict) is_resource_add_success = True log.info(source + 'UEB model package was added as a resource to the ' 'newly created model package dataset with ID:%s' % pkg_id) except Exception as e: log.error( source + 'Failed to add UEB model package as a resource to ' 'the newly created model package dataset with ID:%s \nException: %s' % (pkg_id, e)) pass # link this newly created model package dataset to the model configuration dataset package_relationship_create_action = tk.get_action( 'package_relationship_create') data_dict = { 'subject': pkg_id, 'object': model_config_dataset_id, 'type': 'links_to' } package_relationship_create_action(context, data_dict) # update the related model configuration dataset to show that the package is available if is_resource_add_success: # update the related dataset data_dict = {'package_availability': 'Available'} update_msg = 'system auto updated ueb package dataset' background_task = True try: updated_package = uebhelper.update_package(model_config_dataset_id, data_dict, update_msg, background_task) log.info( source + 'UEB model configuration dataset was updated as a result of ' 'receiving model input package for dataset:%s' % updated_package['name']) except Exception as e: log.error(source + 'Failed to update UEB model configuration dataset after ' 'receiving model input package for dataset ID:%s \n' 'Exception: %s' % (model_config_dataset_id, e)) pass return
def _save_ueb_package_as_dataset_obsolete(service_call_results, model_config_dataset_id): source = 'uebpackage.tasks._save_ueb_package_as_dataset():' ckan_default_dir = uebhelper.StringSettings.ckan_user_session_temp_dir # '/tmp/ckan' # get the matching model configuration dataset object model_config_dataset_obj = base.model.Package.get(model_config_dataset_id) model_config_dataset_title = model_config_dataset_obj.title model_config_dataset_owner_org = model_config_dataset_obj.owner_org model_config_dataset_author = model_config_dataset_obj.author # create a directory for saving the file # this will be a dir in the form of: /tmp/ckan/{random_id} random_id = base.model.types.make_uuid() destination_dir = os.path.join(ckan_default_dir, random_id) os.makedirs(destination_dir) model_pkg_filename = uebhelper.StringSettings.ueb_input_model_package_default_filename # 'ueb_model_pkg.zip' model_pkg_file = os.path.join(destination_dir, model_pkg_filename) bytes_to_read = 16 * 1024 try: with open(model_pkg_file, 'wb') as file_obj: while True: data = service_call_results.read(bytes_to_read) if not data: break file_obj.write(data) except Exception as e: log.error(source + 'Failed to save the ueb_package zip file to temporary ' 'location for UEB model configuration dataset ID: %s \n ' 'Exception: %s' % (model_config_dataset_id, e)) raise e log.info(source + 'ueb_package zip file was saved to temporary location for ' 'UEB model configuration dataset ID: %s' % model_config_dataset_id) # upload the file to CKAN file store resource_metadata = _upload_file(model_pkg_file) if resource_metadata: log.info(source + 'UEB model package zip file was uploaded for model configuration dataset ID:%s' % model_config_dataset_id) else: log.error(source + 'Failed to upload UEB model package zip file ' 'for model configuration dataset ID: %s' % model_config_dataset_id) return # retrieve some of the file meta data resource_url = resource_metadata.get('_label') # this will return datetime stamp/filename resource_url = '/storage/f/' + resource_url if resource_url.startswith('/'): resource_url = base.config.get('ckan.site_url', '').rstrip('/') + resource_url else: resource_url = base.config.get('ckan.site_url', '') + resource_url resource_created_date = resource_metadata.get('_creation_date') resource_name = resource_metadata.get('filename_original') resource_size = resource_metadata.get('_content_length') # add the uploaded ueb model pkg data file as a resource to the dataset resource_create_action = tk.get_action('resource_create') context = {'model': base.model, 'session': base.model.Session, 'save': 'save'} user = uebhelper.get_site_user() context['user'] = user.get('name') context['ignore_auth'] = True context['validate'] = False # create a package package_create_action = tk.get_action('package_create') # create unique package name using the current time stamp as a postfix to any package name unique_postfix = datetime.now().isoformat().replace(':', '-').replace('.', '-').lower() pkg_title = model_config_dataset_title data_dict = { 'name': 'model_package_' + unique_postfix, # this needs to be unique as required by DB 'type': 'model-package', # dataset type as defined in custom dataset plugin 'title': pkg_title, 'owner_org': model_config_dataset_owner_org, 'author': model_config_dataset_author, 'notes': 'UEB model package', 'pkg_model_name': 'UEB', 'model_version': '1.0', 'north_extent': '', 'south_extent': '', 'east_extent': '', 'west_extent': '', 'simulation_start_day': '', 'simulation_end_day': '', 'time_step': '', 'package_type': u'Input', 'package_run_status': 'Not yet submitted', 'package_run_job_id': '', 'dataset_type': 'model-package' } context = {'model': base.model, 'session': base.model.Session, 'ignore_auth': True, 'user': user.get('name'), 'save': 'save'} try: uebhelper.register_translator() # this is needed since we are creating a package in a background operation pkg_dict = package_create_action(context, data_dict) log.info(source + 'A new dataset was created for UEB input model package with name: %s' % data_dict['title']) except Exception as e: log.error(source + 'Failed to create a new dataset for ueb input model package for' ' the related model configuration dataset title: %s \n Exception: %s' % (pkg_title, e)) return pkg_id = pkg_dict['id'] #A value for key 'message" in the context must be set to # avoid a bug in line#192 of the update.py under lib/action # without this, a TypeError will occur (TypeError:No object(name:translator) has been registered for this thread) context['message'] = 'Auto uploaded model package ' data_dict = { "package_id": pkg_id, # id of the package/dataset to which the resource needs to be added "url": resource_url, "name": resource_name, "created": resource_created_date, "format": 'zip', "size": resource_size, "description": 'UEB model package' } is_resource_add_success = False try: resource_create_action(context, data_dict) is_resource_add_success = True log.info(source + 'UEB model package was added as a resource to the ' 'newly created model package dataset with ID:%s' % pkg_id) except Exception as e: log.error(source + 'Failed to add UEB model package as a resource to ' 'the newly created model package dataset with ID:%s \nException: %s' % (pkg_id, e)) pass # link this newly created model package dataset to the model configuration dataset package_relationship_create_action = tk.get_action('package_relationship_create') data_dict = {'subject': pkg_id, 'object': model_config_dataset_id, 'type': 'links_to'} package_relationship_create_action(context, data_dict) # update the related model configuration dataset to show that the package is available if is_resource_add_success: # update the related dataset data_dict = {'package_availability': 'Available'} update_msg = 'system auto updated ueb package dataset' background_task = True try: updated_package = uebhelper.update_package(model_config_dataset_id, data_dict, update_msg, background_task) log.info(source + 'UEB model configuration dataset was updated as a result of ' 'receiving model input package for dataset:%s' % updated_package['name']) except Exception as e: log.error(source + 'Failed to update UEB model configuration dataset after ' 'receiving model input package for dataset ID:%s \n' 'Exception: %s' % (model_config_dataset_id, e)) pass return
def _save_ueb_package_as_dataset(service_call_results, model_config_dataset_id): source = 'uebpackage.tasks._save_ueb_package_as_dataset():' ckan_default_dir = uebhelper.StringSettings.ckan_user_session_temp_dir # '/tmp/ckan' # get the matching model configuration dataset object model_config_dataset_obj = base.model.Package.get(model_config_dataset_id) model_config_dataset_title = model_config_dataset_obj.title model_config_dataset_owner_org = model_config_dataset_obj.owner_org model_config_dataset_author = model_config_dataset_obj.author # create a directory for saving the file # this will be a dir in the form of: /tmp/ckan/{random_id} random_id = base.model.types.make_uuid() destination_dir = os.path.join(ckan_default_dir, random_id) os.makedirs(destination_dir) model_pkg_filename = uebhelper.StringSettings.ueb_input_model_package_default_filename # 'ueb_model_pkg.zip' model_pkg_file = os.path.join(destination_dir, model_pkg_filename) bytes_to_read = 16 * 1024 try: with open(model_pkg_file, 'wb') as file_obj: while True: data = service_call_results.read(bytes_to_read) if not data: break file_obj.write(data) except Exception as e: log.error(source + 'Failed to save the ueb_package zip file to temporary ' 'location for UEB model configuration dataset ID: %s \n ' 'Exception: %s' % (model_config_dataset_id, e)) raise e log.info(source + 'ueb_package zip file was saved to temporary location for ' 'UEB model configuration dataset ID: %s' % model_config_dataset_id) # upload the file to CKAN file store # resource_metadata = _upload_file(model_pkg_file) # if resource_metadata: # log.info(source + 'UEB model package zip file was uploaded for model configuration dataset ID:%s' % model_config_dataset_id) # else: # log.error(source + 'Failed to upload UEB model package zip file ' # 'for model configuration dataset ID: %s' % model_config_dataset_id) # return # # # retrieve some of the file meta data # resource_url = resource_metadata.get('_label') # this will return datetime stamp/filename # # resource_url = '/storage/f/' + resource_url # if resource_url.startswith('/'): # resource_url = base.config.get('ckan.site_url', '').rstrip('/') + resource_url # else: # resource_url = base.config.get('ckan.site_url', '') + resource_url # # resource_created_date = resource_metadata.get('_creation_date') # resource_name = resource_metadata.get('filename_original') # resource_size = resource_metadata.get('_content_length') # # # add the uploaded ueb model pkg data file as a resource to the dataset # resource_create_action = tk.get_action('resource_create') # context = {'model': base.model, 'session': base.model.Session, 'save': 'save'} # user = uebhelper.get_site_user() # context['user'] = user.get('name') # context['ignore_auth'] = True # context['validate'] = False user = uebhelper.get_site_user() # create a package package_create_action = tk.get_action('package_create') # create unique package name using the current time stamp as a postfix to any package name unique_postfix = datetime.now().isoformat().replace(':', '-').replace( '.', '-').lower() pkg_title = model_config_dataset_title data_dict = { 'name': 'model_package_' + unique_postfix, # this needs to be unique as required by DB 'type': 'model-package', # dataset type as defined in custom dataset plugin 'title': pkg_title, 'owner_org': model_config_dataset_owner_org, 'author': model_config_dataset_author, 'notes': 'UEB model package', 'pkg_model_name': 'UEB', 'model_version': '1.0', 'north_extent': '', 'south_extent': '', 'east_extent': '', 'west_extent': '', 'simulation_start_day': '', 'simulation_end_day': '', 'time_step': '', 'package_type': u'Input', 'package_run_status': 'Not yet submitted', 'package_run_job_id': '', 'dataset_type': 'model-package' } context = { 'model': base.model, 'session': base.model.Session, 'ignore_auth': True, 'user': user.get('name'), 'save': 'save' } try: uebhelper.register_translator( ) # this is needed since we are creating a package in a background operation pkg_dict = package_create_action(context, data_dict) log.info( source + 'A new dataset was created for UEB input model package with name: %s' % data_dict['title']) except Exception as e: log.error( source + 'Failed to create a new dataset for ueb input model package for' ' the related model configuration dataset title: %s \n Exception: %s' % (pkg_title, e)) raise e pkg_id = pkg_dict['id'] if not 'resources' in pkg_dict: pkg_dict['resources'] = [] file_name = munge.munge_filename(model_pkg_filename) resource = {'url': file_name, 'url_type': 'upload'} upload = uploader.ResourceUpload(resource) upload.filename = file_name upload.upload_file = open(model_pkg_file, 'r') data_dict = { 'format': 'zip', 'name': file_name, 'url': file_name, 'url_type': 'upload' } pkg_dict['resources'].append(data_dict) try: context['defer_commit'] = True context['use_cache'] = False # update the package package_update_action = tk.get_action('package_update') package_update_action(context, pkg_dict) context.pop('defer_commit') except Exception as e: log.error( source + ' Failed to update the new dataset for adding the input model package zip file as' ' a resource.\n Exception: %s' % e) raise e # link this newly created model package dataset to the model configuration dataset package_relationship_create_action = tk.get_action( 'package_relationship_create') data_dict = { 'subject': pkg_id, 'object': model_config_dataset_id, 'type': 'links_to' } package_relationship_create_action(context, data_dict) # Get out resource_id resource from model as it will not appear in # package_show until after commit upload.upload(context['package'].resources[-1].id, uploader.get_max_resource_size()) base.model.repo.commit() # update the related model configuration dataset to show that the package is available data_dict = {'package_availability': 'Available'} update_msg = 'system auto updated ueb package dataset' background_task = True try: updated_package = uebhelper.update_package(model_config_dataset_id, data_dict, update_msg, background_task) log.info(source + 'UEB model configuration dataset was updated as a result of ' 'receiving model input package for dataset:%s' % updated_package['name']) except Exception as e: log.error(source + 'Failed to update UEB model configuration dataset after ' 'receiving model input package for dataset ID:%s \n' 'Exception: %s' % (model_config_dataset_id, e)) raise e
def _save_ueb_package_as_dataset(service_call_results, model_config_dataset_id): source = 'uebpackage.tasks._save_ueb_package_as_dataset():' ckan_default_dir = uebhelper.StringSettings.ckan_user_session_temp_dir # '/tmp/ckan' # get the matching model configuration dataset object model_config_dataset_obj = base.model.Package.get(model_config_dataset_id) model_config_dataset_title = model_config_dataset_obj.title model_config_dataset_owner_org = model_config_dataset_obj.owner_org model_config_dataset_author = model_config_dataset_obj.author # create a directory for saving the file # this will be a dir in the form of: /tmp/ckan/{random_id} random_id = base.model.types.make_uuid() destination_dir = os.path.join(ckan_default_dir, random_id) os.makedirs(destination_dir) model_pkg_filename = uebhelper.StringSettings.ueb_input_model_package_default_filename # 'ueb_model_pkg.zip' model_pkg_file = os.path.join(destination_dir, model_pkg_filename) bytes_to_read = 16 * 1024 try: with open(model_pkg_file, 'wb') as file_obj: while True: data = service_call_results.read(bytes_to_read) if not data: break file_obj.write(data) except Exception as e: log.error(source + 'Failed to save the ueb_package zip file to temporary ' 'location for UEB model configuration dataset ID: %s \n ' 'Exception: %s' % (model_config_dataset_id, e)) raise e log.info(source + 'ueb_package zip file was saved to temporary location for ' 'UEB model configuration dataset ID: %s' % model_config_dataset_id) # upload the file to CKAN file store # resource_metadata = _upload_file(model_pkg_file) # if resource_metadata: # log.info(source + 'UEB model package zip file was uploaded for model configuration dataset ID:%s' % model_config_dataset_id) # else: # log.error(source + 'Failed to upload UEB model package zip file ' # 'for model configuration dataset ID: %s' % model_config_dataset_id) # return # # # retrieve some of the file meta data # resource_url = resource_metadata.get('_label') # this will return datetime stamp/filename # # resource_url = '/storage/f/' + resource_url # if resource_url.startswith('/'): # resource_url = base.config.get('ckan.site_url', '').rstrip('/') + resource_url # else: # resource_url = base.config.get('ckan.site_url', '') + resource_url # # resource_created_date = resource_metadata.get('_creation_date') # resource_name = resource_metadata.get('filename_original') # resource_size = resource_metadata.get('_content_length') # # # add the uploaded ueb model pkg data file as a resource to the dataset # resource_create_action = tk.get_action('resource_create') # context = {'model': base.model, 'session': base.model.Session, 'save': 'save'} # user = uebhelper.get_site_user() # context['user'] = user.get('name') # context['ignore_auth'] = True # context['validate'] = False user = uebhelper.get_site_user() # create a package package_create_action = tk.get_action('package_create') # create unique package name using the current time stamp as a postfix to any package name unique_postfix = datetime.now().isoformat().replace(':', '-').replace('.', '-').lower() pkg_title = model_config_dataset_title data_dict = { 'name': 'model_package_' + unique_postfix, # this needs to be unique as required by DB 'type': 'model-package', # dataset type as defined in custom dataset plugin 'title': pkg_title, 'owner_org': model_config_dataset_owner_org, 'author': model_config_dataset_author, 'notes': 'UEB model package', 'pkg_model_name': 'UEB', 'model_version': '1.0', 'north_extent': '', 'south_extent': '', 'east_extent': '', 'west_extent': '', 'simulation_start_day': '', 'simulation_end_day': '', 'time_step': '', 'package_type': u'Input', 'package_run_status': 'Not yet submitted', 'package_run_job_id': '', 'dataset_type': 'model-package' } context = {'model': base.model, 'session': base.model.Session, 'ignore_auth': True, 'user': user.get('name'), 'save': 'save'} try: uebhelper.register_translator() # this is needed since we are creating a package in a background operation pkg_dict = package_create_action(context, data_dict) log.info(source + 'A new dataset was created for UEB input model package with name: %s' % data_dict['title']) except Exception as e: log.error(source + 'Failed to create a new dataset for ueb input model package for' ' the related model configuration dataset title: %s \n Exception: %s' % (pkg_title, e)) raise e pkg_id = pkg_dict['id'] if not 'resources' in pkg_dict: pkg_dict['resources'] = [] file_name = munge.munge_filename(model_pkg_filename) resource = {'url': file_name, 'url_type': 'upload'} upload = uploader.ResourceUpload(resource) upload.filename = file_name upload.upload_file = open(model_pkg_file, 'r') data_dict = {'format': 'zip', 'name': file_name, 'url': file_name, 'url_type': 'upload'} pkg_dict['resources'].append(data_dict) try: context['defer_commit'] = True context['use_cache'] = False # update the package package_update_action = tk.get_action('package_update') package_update_action(context, pkg_dict) context.pop('defer_commit') except Exception as e: log.error(source + ' Failed to update the new dataset for adding the input model package zip file as' ' a resource.\n Exception: %s' % e) raise e # link this newly created model package dataset to the model configuration dataset package_relationship_create_action = tk.get_action('package_relationship_create') data_dict = {'subject': pkg_id, 'object': model_config_dataset_id, 'type': 'links_to'} package_relationship_create_action(context, data_dict) # Get out resource_id resource from model as it will not appear in # package_show until after commit upload.upload(context['package'].resources[-1].id, uploader.get_max_resource_size()) base.model.repo.commit() # update the related model configuration dataset to show that the package is available data_dict = {'package_availability': 'Available'} update_msg = 'system auto updated ueb package dataset' background_task = True try: updated_package = uebhelper.update_package(model_config_dataset_id, data_dict, update_msg, background_task) log.info(source + 'UEB model configuration dataset was updated as a result of ' 'receiving model input package for dataset:%s' % updated_package['name']) except Exception as e: log.error(source + 'Failed to update UEB model configuration dataset after ' 'receiving model input package for dataset ID:%s \n' 'Exception: %s' % (model_config_dataset_id, e)) raise e