def run(self):
if self.verbose:
print('Loading environment modules...', file=stderr)
if self.modules is not None:
self.module_cmd(['load'])
if self.verbose:
print('Gathering input files...', file=stderr)
self.get_files()
if self.verbose:
print('Removing exclusions...', file=stderr)
if self.verbose:
print("Making output directories...", file=stderr)
mkdir_p(self.output_root)
if self.exclusions_paths:
self.exclude_files_below(self.exclusions_paths)
self.exclude_files_below(self.output_root)
if self.exclusions:
self.remove_regex_from_input(self.exclusions)
if self.verbose:
print('Formatting commands...', file=stderr)
self.format_commands()
def run(self):
"""
Run the Parallel Command from start to finish
1) Load Environment Modules
2) Gather input files
3) Remove exclusions
4) Make Directories
5) Format Commands
6) Dispatch Scripts to Cluster Scheduler
7) Unload the modules
:return: list<str>: a list of job IDs returned by cluster scheduler
"""
if self.verbose:
print('Loading environment modules...', file=stderr)
if self.modules is not None:
self.module_cmd(['load'])
if self.verbose:
print('Gathering input files...', file=stderr)
self.get_files()
if self.verbose:
print('Removing exclusions...', file=stderr)
if self.verbose:
print("Making output directories...", file=stderr)
mkdir_p(self.output_root)
if self.exclusions_paths:
self.exclude_files_below(self.exclusions_paths)
self.exclude_files_below(self.output_root)
if self.exclusions:
self.remove_regex_from_input(self.exclusions)
self.remove_regex_from_input(r".combine.{}".format(self.extension))
self.remove_regex_from_input(r"{}".format(self.all_reads_name))
if self.verbose:
print('Formatting commands...', file=stderr)
self.format_commands()
if self.verbose:
print('Dispatching to cluster...', file=stderr)
jobs = self.dispatch() # Return the job IDs from the dispatched cmds
return (jobs)
def make_directories(self):
"""
Make the relative output directories that are necessary to preserve
output directory structure at the specified output root. All directories
below input_root will be created below output root
:return:
"""
directories = [x[0] for x in walk(self.input_root)] # all dirs
output_directories = [self.rebase_directory(x, self.input_root,
self.output_root)
for x in directories] # rebase_directory each dir
for directory in output_directories:
if self.verbose:
print("Attempting to make: {}".format(directory),
file=stderr)
if not self.dry_run:
mkdir_p(directory) # Attempt safe creation of each dir
def main(directory):
filenames = [x for x in listdir(directory) if isfile(join(directory, x))]
for exclusion in exclusions:
for filename in list(filenames):
if exclusion in filename:
filenames.remove(filename)
libraries = list(set([basename(x).split("_")[0] for x in filenames]))
[mkdir_p(join(directory, f)) for f in libraries]
for lib in libraries:
for filename in filenames:
if "{}_".format(lib) in filename:
initial = join(directory, filename)
library = join(directory, lib)
final = join(library, filename)
try:
rename(initial, final)
except Exception as err:
print(err)