def register_module(module_name, zip_search_path, cache_path):
#Find and open the input zip file
zip_input_filename = module_name + ".zip"
zip_input_filepath = cga_util.find_file_in_path(zip_input_filename,
zip_search_path)
zip_in = zipfile.ZipFile(zip_input_filepath)
#Get the LSID from the input zip file
manifest_contents = zip_in.read("manifest")
lsid = common.get_lsid_from_manifest(manifest_contents)
#Prepare output dir
cached_module_dir = common.get_cached_module_dir(module_name, lsid,
cache_path)
# TODO - create filesystem-based semaphore
if os.path.exists(cached_module_dir):
raise Exception('Already registered module %s lsid %s' %
(module_name, lsid))
cga_util.safe_make_dirs(cached_module_dir)
#files are stored in flat directory structure within zip.
#unzip them into the output dir
zip_filelist = zip_in.namelist()
for component_filename in zip_filelist:
file_contents = zip_in.read(component_filename)
out_filepath = os.path.join(cached_module_dir, component_filename)
cga_util.write_string_to_file(out_filepath, file_contents)
zip_in.close()
#All done.
print('Successfully registered module %s lsid %s' % (module_name, lsid))
def execute_command_like_gp(exe_str, cwd, logparams, error_mode):
if os.path.exists(cwd):
#raise Exception ('Output directory already exists: %s'%cwd)
pass
stdout_path = os.path.join(cwd, 'stdout.txt')
stderr_path = os.path.join(cwd, 'stderr.txt')
gp_execution_log_path = os.path.join(cwd, 'gp_execution_log.txt')
cga_util.safe_make_dirs(cwd)
if logparams is not None:
gp_execution_log_str = generate_gp_execution_log(logparams, exe_str)
else:
gp_execution_log_str = exe_str + '\n'
cga_util.write_string_to_file(gp_execution_log_path, gp_execution_log_str)
stdout_fid = open(stdout_path, 'w')
stderr_fid = open(stderr_path, 'w')
exit_code = subprocess.call(exe_str,
shell=True,
stdout=stdout_fid,
stderr=stderr_fid,
cwd=cwd)
stdout_fid.close()
stderr_fid.close()
if not os.path.exists(stderr_path) or os.path.getsize(stderr_path) == 0:
stderr_empty = True
else:
stderr_empty = False
if error_mode == 'exit_code':
passing = (exit_code == 0)
elif error_mode == 'stderr':
passing = stderr_empty
else:
raise Exception('unrecognized value for error_mode')
return passing
def verstore(src, dest, tag):
#TODO allow a glob for a pattern match?
src_fullpath = os.path.realpath(os.path.abspath(os.path.expanduser(src)))
link_data_filepath = os.path.abspath(os.path.expanduser(dest))
link_md5_filepath = link_data_filepath + '.md5'
store_dir = link_data_filepath + '.store'
log_filepath = os.path.join(store_dir, 'logfile.txt')
link_dir = os.path.dirname(link_data_filepath)
dest_filename = os.path.basename(link_data_filepath)
if not os.path.exists(src_fullpath):
raise Exception('src file not found: ' + src_fullpath)
if os.path.exists(link_data_filepath) or os.path.exists(
link_md5_filepath) or os.path.exists(store_dir):
if os.path.exists(store_dir) and \
(not os.path.exists(link_data_filepath) or not os.path.exists(link_md5_filepath)):
#incomplete setup last time... nuke the directory
shutil.rmtree(store_dir)
if os.path.exists(link_data_filepath):
os.remove(link_data_filepath)
if os.path.exists(link_md5_filepath):
os.remove(link_md5_filepath)
# check that a valid existing store exists
elif not os.path.islink(link_data_filepath) or not os.path.islink(link_md5_filepath) or \
not os.path.isdir(store_dir) or not os.path.exists(log_filepath):
raise Exception('existing data store corrupted at destination ' +
link_data_filepath)
storage_request_num = get_new_storage_request_num(log_filepath)
timestamp = cga_util.get_timestamp()
username = getpass.getuser()
filesize = os.path.getsize(src_fullpath)
md5_str = cga_util.compute_md5(src_fullpath)
(file_status, file_version) = get_new_file_version(log_filepath, md5_str)
if file_status == 'New':
operation = 'Copy'
# TODO check that there is enough space to do the copy
elif file_status == 'Preexisting':
operation = 'Link'
start_line_fields = [
'action', 'storage_request_num', 'operation', 'file_version', 'md5',
'timestamp', 'size', 'user', 'original_path', 'tag'
]
start_line_dict = {
'action': 'Starting',
'storage_request_num': storage_request_num,
'operation': operation,
'file_version': file_version,
'md5': md5_str,
'timestamp': timestamp,
'size': str(filesize),
'user': username,
'original_path': src_fullpath,
'tag': tag,
}
cga_util.safe_make_dirs(store_dir, 0775)
write_log_line(log_filepath, start_line_fields, start_line_dict)
store_data_filepath = os.path.join(
store_dir, '.'.join(['verstore', file_version, dest_filename]))
store_md5_filepath = store_data_filepath + '.md5'
if operation == 'Copy':
# This file contents has never been stored here
# write the md5 and data files
cga_util.write_string_to_file(store_md5_filepath, md5_str + '\n')
shutil.copy(src_fullpath, store_data_filepath)
os.chmod(store_md5_filepath, 0555)
os.chmod(store_data_filepath, 0555)
# TODO do setuid on these files
elif operation == 'Link':
# verify that the existing stored file is intact.
old_md5_stored = cga_util.read_string_from_file(
store_md5_filepath, 'rstrip')
if old_md5_stored != md5_str:
raise Exception(
'Corrupted store: md5 in file does not match the expected value'
)
old_md5_computed = cga_util.compute_md5(store_data_filepath)
if old_md5_computed != md5_str:
raise Exception(
'Corrupted store: computed md5 of old data file does not match the expected value'
)
else:
raise Exception('unexpected value for operation: ' + operation)
# make symlinks point to the current version
if os.path.exists(link_data_filepath):
os.remove(link_data_filepath)
os.symlink(store_data_filepath, link_data_filepath)
if os.path.exists(link_md5_filepath):
os.remove(link_md5_filepath)
os.symlink(store_md5_filepath, link_md5_filepath)
end_line_fields = ['action', 'storage_request_num']
end_line_dict = {
'action': 'Complete',
'storage_request_num': storage_request_num,
}
write_log_line(log_filepath, end_line_fields, end_line_dict)