def run_cmd(self, data): """Runs the given command. """ from app.mod_cmd.commands.projects.load_project import load_project from app import app from app.mod_cmd.client_instruction import ClientInstruction command = data['method'] params = data['params'] package = get_package(command) project = None if 'active_project' in app.session: project, _ = load_project(app.session['active_user'], app.session['active_project']) if os.path.exists(os.path.join('app', 'mod_cmd', 'commands', package, command+'.py')): if package != '': package += '.' cmd = importlib.import_module("app.mod_cmd.commands.%s%s" % (package, command)) [project, instruction] = cmd.run(project, params, connection=self) else: instruction = ClientInstruction({'message': "Command %s not found. Run 'help' to view all available commands." % command}) project, instruction = self.pass_other_stuff(project, instruction) return (project, instruction.to_json())
def retrieve_ueb_run_output_packages(): source = 'uebpackage.tasks.retrieve_ueb_run_output_packages():' global service_host_address #service_request_api_url = '/api/UEBModelRunOutput' service_request_api_url = uebhelper.StringSettings.app_server_api_get_ueb_run_output connection = httplib.HTTPConnection(service_host_address) # get all datasets of type model-package model_pkg_datasets = uebhelper.get_packages_by_dataset_type('model-package') for dataset in model_pkg_datasets: pkg_run_job_id = h.get_pkg_dict_extra(dataset, 'package_run_job_id') if pkg_run_job_id is None: continue # to get the package_type value which is a tag, use the get_package() of my the helper module pkg_dict = uebhelper.get_package(dataset['id']) # TODO: Before using pkg_dict check that it is not None pkg_type = pkg_dict['package_type'][0] if len(pkg_run_job_id) == 0: continue if pkg_type == u'Complete': continue pkg_run_status = h.get_pkg_dict_extra(dataset, 'package_run_status') if pkg_run_status != uebhelper.StringSettings.app_server_job_status_success: continue dataset_id = dataset['id'] service_request_url = service_request_api_url + '?uebRunJobID=' + pkg_run_job_id connection.request('GET', service_request_url) service_call_results = connection.getresponse() if service_call_results.status == httplib.OK: log.info(source + 'UEB model output package was received from App ' 'server for model pkg dataset ID:%s and Run Job ID:%s' % (dataset_id, pkg_run_job_id)) _merge_ueb_output_pkg_with_input_pkg(service_call_results, dataset_id) else: log.error(source + 'HTTP status %d returned from App server when ' 'retrieving UEB model output package for ' 'model pkg dataset ID:%s and Run Job ID:%s' % (service_call_results.status, dataset_id, pkg_run_job_id)) ueb_run_status = 'Failed to retrieve output package' # update the dataset data_dict = {'package_run_status': ueb_run_status} try: uebhelper.update_package(dataset_id, data_dict, backgroundTask=True) log.info(source + 'UEB model package dataset run status was updated to %s for ' 'dataset ID:%s' % (dataset_id, ueb_run_status)) except Exception as e: log.error(source + 'Failed to update run status for UEB model package dataset ' 'with dataset ID:%s\nException:%s' % (dataset_id, e)) connection.close() return
def _merge_ueb_output_pkg_with_input_pkg(service_call_results, model_pkg_dataset_id): source = 'uebpackage.tasks._merge_ueb_output_pkg_with_input_pkg():' # save the output model pkg to temp directory #ckan_default_dir = '/tmp/ckan' ckan_default_dir = uebhelper.StringSettings.ckan_user_session_temp_dir # create a directory for saving the file # this will be a dir in the form of: /tmp/ckan/{random_id} random_id = base.model.types.make_uuid() destination_dir = os.path.join(ckan_default_dir, random_id) os.makedirs(destination_dir) ueb_output_pkg_filename = uebhelper.StringSettings.ueb_output_model_package_default_filename ueb_output_pkg_file = os.path.join(destination_dir, ueb_output_pkg_filename) bytes_to_read = 16 * 1024 try: with open(ueb_output_pkg_file, 'wb') as file_obj: while True: data = service_call_results.read(bytes_to_read) if not data: break file_obj.write(data) except Exception as e: log.error(source + 'Failed to save ueb model output package zip file to ' 'temporary location for model package dataset ID: %s \n ' 'Exception: %s' % (model_pkg_dataset_id, e)) pass return # no need to show error to the user as this a background operation log.info( source + 'ueb model output package zip file was saved to temporary location ' 'for model package dataset ID: %s' % model_pkg_dataset_id) # access the input model pkg zip file model_pkg_dataset = uebhelper.get_package(model_pkg_dataset_id) # TODO: Before using model_pk_dataset check that it is not None model_pkg_resource_zip_file = model_pkg_dataset['resources'][0] # replace the '%3A' in the file url with ':' to get the correct folder name in the file system pkg_zip_file_url = model_pkg_resource_zip_file['url'] pkg_zip_file_url = pkg_zip_file_url.replace('%3A', ':') sting_to_search_in_file_url = 'storage/f/' search_string_index = pkg_zip_file_url.find(sting_to_search_in_file_url) file_path_start_index = search_string_index + len( sting_to_search_in_file_url) original_zip_file_path = pkg_zip_file_url[file_path_start_index:] original_zip_file_path = os.path.join('/', 'var', 'lib', 'ckan', 'default', 'pairtree_root', 'de', 'fa', 'ul', 't', 'obj', original_zip_file_path) ''' open the original input zip file in the append mode and then read the output pkg zip file and append it to the original zip file ''' is_merge_successful = False try: with zip.ZipFile(original_zip_file_path, 'a') as orig_file_obj: zip_file_to_merge = zip.ZipFile(ueb_output_pkg_file, 'r') for fname in zip_file_to_merge.namelist(): orig_file_obj.writestr(fname, zip_file_to_merge.open(fname).read()) is_merge_successful = True except Exception as e: log.error( source + 'Failed to merge output model pkg zip file with the input model pkg zip file ' 'for model package dataset ID: %s \n ' 'Exception: %s' % (model_pkg_dataset_id, e)) pass # update the model package dataset package_type to complete if is_merge_successful: data_dict = { 'package_run_status': 'Output package available', 'package_type': u'Complete' } else: data_dict = { 'package_run_status': 'Success' } # TODO: set to 'Success' for testing. Finally needs to be set "Merge failed" update_msg = 'system auto updated ueb package dataset' background_task = True try: updated_package = uebhelper.update_package(model_pkg_dataset_id, data_dict, update_msg, background_task) log.info(source + 'UEB model package dataset was updated as a result of ' 'receiving model output package for dataset:%s' % updated_package['name']) except Exception as e: log.error(source + 'Failed to update UEB model package dataset after ' 'receiving model input package for dataset ID:%s \n' 'Exception: %s' % (model_pkg_dataset_id, e)) pass
def retrieve_ueb_run_output_packages(): source = 'uebpackage.tasks.retrieve_ueb_run_output_packages():' global service_host_address #service_request_api_url = '/api/UEBModelRunOutput' service_request_api_url = uebhelper.StringSettings.app_server_api_get_ueb_run_output connection = httplib.HTTPConnection(service_host_address) # get all datasets of type model-package model_pkg_datasets = uebhelper.get_packages_by_dataset_type( 'model-package') for dataset in model_pkg_datasets: pkg_run_job_id = h.get_pkg_dict_extra(dataset, 'package_run_job_id') if pkg_run_job_id is None: continue # to get the package_type value which is a tag, use the get_package() of my the helper module pkg_dict = uebhelper.get_package(dataset['id']) # TODO: Before using pkg_dict check that it is not None pkg_type = pkg_dict['package_type'][0] if len(pkg_run_job_id) == 0: continue if pkg_type == u'Complete': continue pkg_run_status = h.get_pkg_dict_extra(dataset, 'package_run_status') if pkg_run_status != uebhelper.StringSettings.app_server_job_status_success: continue dataset_id = dataset['id'] service_request_url = service_request_api_url + '?uebRunJobID=' + pkg_run_job_id connection.request('GET', service_request_url) service_call_results = connection.getresponse() if service_call_results.status == httplib.OK: log.info(source + 'UEB model output package was received from App ' 'server for model pkg dataset ID:%s and Run Job ID:%s' % (dataset_id, pkg_run_job_id)) _merge_ueb_output_pkg_with_input_pkg(service_call_results, dataset_id) else: log.error( source + 'HTTP status %d returned from App server when ' 'retrieving UEB model output package for ' 'model pkg dataset ID:%s and Run Job ID:%s' % (service_call_results.status, dataset_id, pkg_run_job_id)) ueb_run_status = 'Failed to retrieve output package' # update the dataset data_dict = {'package_run_status': ueb_run_status} try: uebhelper.update_package(dataset_id, data_dict, backgroundTask=True) log.info( source + 'UEB model package dataset run status was updated to %s for ' 'dataset ID:%s' % (dataset_id, ueb_run_status)) except Exception as e: log.error( source + 'Failed to update run status for UEB model package dataset ' 'with dataset ID:%s\nException:%s' % (dataset_id, e)) connection.close() return
def _merge_ueb_output_pkg_with_input_pkg(service_call_results, model_pkg_dataset_id): source = 'uebpackage.tasks._merge_ueb_output_pkg_with_input_pkg():' # save the output model pkg to temp directory #ckan_default_dir = '/tmp/ckan' ckan_default_dir = uebhelper.StringSettings.ckan_user_session_temp_dir # create a directory for saving the file # this will be a dir in the form of: /tmp/ckan/{random_id} random_id = base.model.types.make_uuid() destination_dir = os.path.join(ckan_default_dir, random_id) os.makedirs(destination_dir) ueb_output_pkg_filename = uebhelper.StringSettings.ueb_output_model_package_default_filename ueb_output_pkg_file = os.path.join(destination_dir, ueb_output_pkg_filename) bytes_to_read = 16 * 1024 try: with open(ueb_output_pkg_file, 'wb') as file_obj: while True: data = service_call_results.read(bytes_to_read) if not data: break file_obj.write(data) except Exception as e: log.error(source + 'Failed to save ueb model output package zip file to ' 'temporary location for model package dataset ID: %s \n ' 'Exception: %s' % (model_pkg_dataset_id, e)) pass return # no need to show error to the user as this a background operation log.info(source + 'ueb model output package zip file was saved to temporary location ' 'for model package dataset ID: %s' % model_pkg_dataset_id) # access the input model pkg zip file model_pkg_dataset = uebhelper.get_package(model_pkg_dataset_id) # TODO: Before using model_pk_dataset check that it is not None model_pkg_resource_zip_file = model_pkg_dataset['resources'][0] # replace the '%3A' in the file url with ':' to get the correct folder name in the file system pkg_zip_file_url = model_pkg_resource_zip_file['url'] pkg_zip_file_url = pkg_zip_file_url.replace('%3A', ':') sting_to_search_in_file_url = 'storage/f/' search_string_index = pkg_zip_file_url.find(sting_to_search_in_file_url) file_path_start_index = search_string_index + len(sting_to_search_in_file_url) original_zip_file_path = pkg_zip_file_url[file_path_start_index:] original_zip_file_path = os.path.join('/', 'var', 'lib', 'ckan', 'default', 'pairtree_root', 'de', 'fa', 'ul', 't', 'obj', original_zip_file_path) ''' open the original input zip file in the append mode and then read the output pkg zip file and append it to the original zip file ''' is_merge_successful = False try: with zip.ZipFile(original_zip_file_path, 'a') as orig_file_obj: zip_file_to_merge = zip.ZipFile(ueb_output_pkg_file, 'r') for fname in zip_file_to_merge.namelist(): orig_file_obj.writestr(fname, zip_file_to_merge.open(fname).read()) is_merge_successful = True except Exception as e: log.error(source + 'Failed to merge output model pkg zip file with the input model pkg zip file ' 'for model package dataset ID: %s \n ' 'Exception: %s' % (model_pkg_dataset_id, e)) pass # update the model package dataset package_type to complete if is_merge_successful: data_dict = {'package_run_status': 'Output package available', 'package_type': u'Complete'} else: data_dict = {'package_run_status': 'Success'} # TODO: set to 'Success' for testing. Finally needs to be set "Merge failed" update_msg = 'system auto updated ueb package dataset' background_task = True try: updated_package = uebhelper.update_package(model_pkg_dataset_id, data_dict, update_msg, background_task) log.info(source + 'UEB model package dataset was updated as a result of ' 'receiving model output package for dataset:%s' % updated_package['name']) except Exception as e: log.error(source + 'Failed to update UEB model package dataset after ' 'receiving model input package for dataset ID:%s \n' 'Exception: %s' % (model_pkg_dataset_id, e)) pass