def run_cmd(self, data):
"""Runs the given command.
"""
from app.mod_cmd.commands.projects.load_project import load_project
from app import app
from app.mod_cmd.client_instruction import ClientInstruction
command = data['method']
params = data['params']
package = get_package(command)
project = None
if 'active_project' in app.session:
project, _ = load_project(app.session['active_user'], app.session['active_project'])
if os.path.exists(os.path.join('app', 'mod_cmd', 'commands', package, command+'.py')):
if package != '':
package += '.'
cmd = importlib.import_module("app.mod_cmd.commands.%s%s" % (package, command))
[project, instruction] = cmd.run(project, params, connection=self)
else:
instruction = ClientInstruction({'message': "Command %s not found. Run 'help' to view all available commands." % command})
project, instruction = self.pass_other_stuff(project, instruction)
return (project, instruction.to_json())
def retrieve_ueb_run_output_packages():
source = 'uebpackage.tasks.retrieve_ueb_run_output_packages():'
global service_host_address
#service_request_api_url = '/api/UEBModelRunOutput'
service_request_api_url = uebhelper.StringSettings.app_server_api_get_ueb_run_output
connection = httplib.HTTPConnection(service_host_address)
# get all datasets of type model-package
model_pkg_datasets = uebhelper.get_packages_by_dataset_type('model-package')
for dataset in model_pkg_datasets:
pkg_run_job_id = h.get_pkg_dict_extra(dataset, 'package_run_job_id')
if pkg_run_job_id is None:
continue
# to get the package_type value which is a tag, use the get_package() of my the helper module
pkg_dict = uebhelper.get_package(dataset['id'])
# TODO: Before using pkg_dict check that it is not None
pkg_type = pkg_dict['package_type'][0]
if len(pkg_run_job_id) == 0:
continue
if pkg_type == u'Complete':
continue
pkg_run_status = h.get_pkg_dict_extra(dataset, 'package_run_status')
if pkg_run_status != uebhelper.StringSettings.app_server_job_status_success:
continue
dataset_id = dataset['id']
service_request_url = service_request_api_url + '?uebRunJobID=' + pkg_run_job_id
connection.request('GET', service_request_url)
service_call_results = connection.getresponse()
if service_call_results.status == httplib.OK:
log.info(source + 'UEB model output package was received from App '
'server for model pkg dataset ID:%s and Run Job ID:%s' % (dataset_id, pkg_run_job_id))
_merge_ueb_output_pkg_with_input_pkg(service_call_results, dataset_id)
else:
log.error(source + 'HTTP status %d returned from App server when '
'retrieving UEB model output package for '
'model pkg dataset ID:%s and Run Job ID:%s' %
(service_call_results.status, dataset_id, pkg_run_job_id))
ueb_run_status = 'Failed to retrieve output package'
# update the dataset
data_dict = {'package_run_status': ueb_run_status}
try:
uebhelper.update_package(dataset_id, data_dict, backgroundTask=True)
log.info(source + 'UEB model package dataset run status was updated to %s for '
'dataset ID:%s' % (dataset_id, ueb_run_status))
except Exception as e:
log.error(source + 'Failed to update run status for UEB model package dataset '
'with dataset ID:%s\nException:%s' % (dataset_id, e))
connection.close()
return
def _merge_ueb_output_pkg_with_input_pkg(service_call_results,
model_pkg_dataset_id):
source = 'uebpackage.tasks._merge_ueb_output_pkg_with_input_pkg():'
# save the output model pkg to temp directory
#ckan_default_dir = '/tmp/ckan'
ckan_default_dir = uebhelper.StringSettings.ckan_user_session_temp_dir
# create a directory for saving the file
# this will be a dir in the form of: /tmp/ckan/{random_id}
random_id = base.model.types.make_uuid()
destination_dir = os.path.join(ckan_default_dir, random_id)
os.makedirs(destination_dir)
ueb_output_pkg_filename = uebhelper.StringSettings.ueb_output_model_package_default_filename
ueb_output_pkg_file = os.path.join(destination_dir,
ueb_output_pkg_filename)
bytes_to_read = 16 * 1024
try:
with open(ueb_output_pkg_file, 'wb') as file_obj:
while True:
data = service_call_results.read(bytes_to_read)
if not data:
break
file_obj.write(data)
except Exception as e:
log.error(source +
'Failed to save ueb model output package zip file to '
'temporary location for model package dataset ID: %s \n '
'Exception: %s' % (model_pkg_dataset_id, e))
pass
return # no need to show error to the user as this a background operation
log.info(
source +
'ueb model output package zip file was saved to temporary location '
'for model package dataset ID: %s' % model_pkg_dataset_id)
# access the input model pkg zip file
model_pkg_dataset = uebhelper.get_package(model_pkg_dataset_id)
# TODO: Before using model_pk_dataset check that it is not None
model_pkg_resource_zip_file = model_pkg_dataset['resources'][0]
# replace the '%3A' in the file url with ':' to get the correct folder name in the file system
pkg_zip_file_url = model_pkg_resource_zip_file['url']
pkg_zip_file_url = pkg_zip_file_url.replace('%3A', ':')
sting_to_search_in_file_url = 'storage/f/'
search_string_index = pkg_zip_file_url.find(sting_to_search_in_file_url)
file_path_start_index = search_string_index + len(
sting_to_search_in_file_url)
original_zip_file_path = pkg_zip_file_url[file_path_start_index:]
original_zip_file_path = os.path.join('/', 'var', 'lib', 'ckan', 'default',
'pairtree_root', 'de', 'fa', 'ul',
't', 'obj', original_zip_file_path)
'''
open the original input zip file in the append mode and then
read the output pkg zip file and append it to the original zip file
'''
is_merge_successful = False
try:
with zip.ZipFile(original_zip_file_path, 'a') as orig_file_obj:
zip_file_to_merge = zip.ZipFile(ueb_output_pkg_file, 'r')
for fname in zip_file_to_merge.namelist():
orig_file_obj.writestr(fname,
zip_file_to_merge.open(fname).read())
is_merge_successful = True
except Exception as e:
log.error(
source +
'Failed to merge output model pkg zip file with the input model pkg zip file '
'for model package dataset ID: %s \n '
'Exception: %s' % (model_pkg_dataset_id, e))
pass
# update the model package dataset package_type to complete
if is_merge_successful:
data_dict = {
'package_run_status': 'Output package available',
'package_type': u'Complete'
}
else:
data_dict = {
'package_run_status': 'Success'
} # TODO: set to 'Success' for testing. Finally needs to be set "Merge failed"
update_msg = 'system auto updated ueb package dataset'
background_task = True
try:
updated_package = uebhelper.update_package(model_pkg_dataset_id,
data_dict, update_msg,
background_task)
log.info(source +
'UEB model package dataset was updated as a result of '
'receiving model output package for dataset:%s' %
updated_package['name'])
except Exception as e:
log.error(source + 'Failed to update UEB model package dataset after '
'receiving model input package for dataset ID:%s \n'
'Exception: %s' % (model_pkg_dataset_id, e))
pass
def retrieve_ueb_run_output_packages():
source = 'uebpackage.tasks.retrieve_ueb_run_output_packages():'
global service_host_address
#service_request_api_url = '/api/UEBModelRunOutput'
service_request_api_url = uebhelper.StringSettings.app_server_api_get_ueb_run_output
connection = httplib.HTTPConnection(service_host_address)
# get all datasets of type model-package
model_pkg_datasets = uebhelper.get_packages_by_dataset_type(
'model-package')
for dataset in model_pkg_datasets:
pkg_run_job_id = h.get_pkg_dict_extra(dataset, 'package_run_job_id')
if pkg_run_job_id is None:
continue
# to get the package_type value which is a tag, use the get_package() of my the helper module
pkg_dict = uebhelper.get_package(dataset['id'])
# TODO: Before using pkg_dict check that it is not None
pkg_type = pkg_dict['package_type'][0]
if len(pkg_run_job_id) == 0:
continue
if pkg_type == u'Complete':
continue
pkg_run_status = h.get_pkg_dict_extra(dataset, 'package_run_status')
if pkg_run_status != uebhelper.StringSettings.app_server_job_status_success:
continue
dataset_id = dataset['id']
service_request_url = service_request_api_url + '?uebRunJobID=' + pkg_run_job_id
connection.request('GET', service_request_url)
service_call_results = connection.getresponse()
if service_call_results.status == httplib.OK:
log.info(source + 'UEB model output package was received from App '
'server for model pkg dataset ID:%s and Run Job ID:%s' %
(dataset_id, pkg_run_job_id))
_merge_ueb_output_pkg_with_input_pkg(service_call_results,
dataset_id)
else:
log.error(
source + 'HTTP status %d returned from App server when '
'retrieving UEB model output package for '
'model pkg dataset ID:%s and Run Job ID:%s' %
(service_call_results.status, dataset_id, pkg_run_job_id))
ueb_run_status = 'Failed to retrieve output package'
# update the dataset
data_dict = {'package_run_status': ueb_run_status}
try:
uebhelper.update_package(dataset_id,
data_dict,
backgroundTask=True)
log.info(
source +
'UEB model package dataset run status was updated to %s for '
'dataset ID:%s' % (dataset_id, ueb_run_status))
except Exception as e:
log.error(
source +
'Failed to update run status for UEB model package dataset '
'with dataset ID:%s\nException:%s' % (dataset_id, e))
connection.close()
return
def _merge_ueb_output_pkg_with_input_pkg(service_call_results, model_pkg_dataset_id):
source = 'uebpackage.tasks._merge_ueb_output_pkg_with_input_pkg():'
# save the output model pkg to temp directory
#ckan_default_dir = '/tmp/ckan'
ckan_default_dir = uebhelper.StringSettings.ckan_user_session_temp_dir
# create a directory for saving the file
# this will be a dir in the form of: /tmp/ckan/{random_id}
random_id = base.model.types.make_uuid()
destination_dir = os.path.join(ckan_default_dir, random_id)
os.makedirs(destination_dir)
ueb_output_pkg_filename = uebhelper.StringSettings.ueb_output_model_package_default_filename
ueb_output_pkg_file = os.path.join(destination_dir, ueb_output_pkg_filename)
bytes_to_read = 16 * 1024
try:
with open(ueb_output_pkg_file, 'wb') as file_obj:
while True:
data = service_call_results.read(bytes_to_read)
if not data:
break
file_obj.write(data)
except Exception as e:
log.error(source + 'Failed to save ueb model output package zip file to '
'temporary location for model package dataset ID: %s \n '
'Exception: %s' % (model_pkg_dataset_id, e))
pass
return # no need to show error to the user as this a background operation
log.info(source + 'ueb model output package zip file was saved to temporary location '
'for model package dataset ID: %s' % model_pkg_dataset_id)
# access the input model pkg zip file
model_pkg_dataset = uebhelper.get_package(model_pkg_dataset_id)
# TODO: Before using model_pk_dataset check that it is not None
model_pkg_resource_zip_file = model_pkg_dataset['resources'][0]
# replace the '%3A' in the file url with ':' to get the correct folder name in the file system
pkg_zip_file_url = model_pkg_resource_zip_file['url']
pkg_zip_file_url = pkg_zip_file_url.replace('%3A', ':')
sting_to_search_in_file_url = 'storage/f/'
search_string_index = pkg_zip_file_url.find(sting_to_search_in_file_url)
file_path_start_index = search_string_index + len(sting_to_search_in_file_url)
original_zip_file_path = pkg_zip_file_url[file_path_start_index:]
original_zip_file_path = os.path.join('/', 'var', 'lib', 'ckan', 'default', 'pairtree_root', 'de', 'fa', 'ul', 't', 'obj', original_zip_file_path)
'''
open the original input zip file in the append mode and then
read the output pkg zip file and append it to the original zip file
'''
is_merge_successful = False
try:
with zip.ZipFile(original_zip_file_path, 'a') as orig_file_obj:
zip_file_to_merge = zip.ZipFile(ueb_output_pkg_file, 'r')
for fname in zip_file_to_merge.namelist():
orig_file_obj.writestr(fname, zip_file_to_merge.open(fname).read())
is_merge_successful = True
except Exception as e:
log.error(source + 'Failed to merge output model pkg zip file with the input model pkg zip file '
'for model package dataset ID: %s \n '
'Exception: %s' % (model_pkg_dataset_id, e))
pass
# update the model package dataset package_type to complete
if is_merge_successful:
data_dict = {'package_run_status': 'Output package available', 'package_type': u'Complete'}
else:
data_dict = {'package_run_status': 'Success'} # TODO: set to 'Success' for testing. Finally needs to be set "Merge failed"
update_msg = 'system auto updated ueb package dataset'
background_task = True
try:
updated_package = uebhelper.update_package(model_pkg_dataset_id, data_dict, update_msg, background_task)
log.info(source + 'UEB model package dataset was updated as a result of '
'receiving model output package for dataset:%s' % updated_package['name'])
except Exception as e:
log.error(source + 'Failed to update UEB model package dataset after '
'receiving model input package for dataset ID:%s \n'
'Exception: %s' % (model_pkg_dataset_id, e))
pass
