def __init__(self, folder, name=None, parameters=None, passes=None): """ initializes a project with all necessary information """ self.logger = logging.getLogger(self.__class__.__name__) self.folder = folder self.name = name if passes is None: passes = self.default_passes if not hasattr(passes, '__iter__'): self.passes = range(1, passes + 1) else: self.passes = passes # save tracking parameters self.parameters = DictXpath(PARAMETERS_DEFAULT) if parameters is not None: self.parameters.from_dict(parameters)
def create_parameter_summary(outfile): """ creates a markdown file summarizing the parameters """ # load the parameters into a convenient structure parameters = DictXpath(dict_class=OrderedDict) deprecated = {} for parameter in PARAMETER_LIST: if parameter.unit is UNIT.DEPRECATED: deprecated[parameter.key] = parameter else: parameters[parameter.key] = parameter def param_line(parameter): return "`%s = %s`: %s\n" % (parameter.key, parameter.default_value, parameter.description) with open(outfile, 'w') as fp: # list all the important parameters fp.write('Parameters\n====\n') for key1, value1 in parameters.iteritems(): if isinstance(value1, DictXpath): # we got a sub category category, subset = key1, value1 fp.write("\n%s\n" % category) fp.write("%s\n" % ("-" * len(category))) for key2, value2 in subset.iteritems(): if isinstance(value2, DictXpath): fp.write("* `%s`\n" % key2) for parameter in value2.itervalues(): fp.write(" * " + param_line(parameter)) else: fp.write("* " + param_line(value2)) else: fp.write("* " + param_line(value1)) # list the deprecated parameters fp.write('\nDeprecated parameters\n====\n') for _, parameter in sorted(deprecated.iteritems()): fp.write("* " + param_line(parameter))
class HPCProjectBase(object): """ class that manages a high performance computing project """ files_job = tuple() #< files that need to be set up for the project files_cleanup = tuple() #< files that need to be deleted to clean the work folder file_parameters = '%s_results.yaml' #< pattern for the file where the parameters are stored file_log_pass = "******" #< pattern for the log file of each pass default_passes = 4 def __init__(self, folder, name=None, parameters=None, passes=None): """ initializes a project with all necessary information """ self.logger = logging.getLogger(self.__class__.__name__) self.folder = folder self.name = name if passes is None: passes = self.default_passes if not hasattr(passes, '__iter__'): self.passes = range(1, passes + 1) else: self.passes = passes # save tracking parameters self.parameters = DictXpath(PARAMETERS_DEFAULT) if parameters is not None: self.parameters.from_dict(parameters) def clean_workfolder(self, purge=False): """ clears the project folder """ # determine which files to delete if purge: try: files_to_delete = os.listdir(self.folder) except OSError: files_to_delete = tuple() else: files_to_delete = [] for p in self.passes: files_to_delete.extend(self.files_job[p]) files_to_delete.extend(self.files_cleanup[p]) # iteratively delete these files for pattern in files_to_delete: file_pattern = os.path.join(self.folder, pattern) for file_path in glob.iglob(file_pattern): if os.path.isfile(file_path): try: os.remove(file_path) except OSError: pass @classmethod def create(cls, video_file, result_folder, video_name=None, parameters=None, passes=None, prepare_workfolder='auto'): """ creates a new project from data video_file is the filename of the video to scan result_folder is a general folder in which the results will be stored. Note that a subfolder will be used for all results video_name denotes a name associated with this video, which will be used to name folders and such. If no name is given, the filename is used. parameters is a dictionary that sets the parameters that are used for tracking. There is a special parameter 'scale_length' that we look for, which is applied in the first pass only. passes is an integer which is 1, 2 or 3, indicating whether only the first tracking pass or also the second one should be initialized prepare_workfolder can be 'none', 'clean', or 'purge', which indicates increasing amounts of files that will be deleted before creating the project. If it is set to 'auto', the folder will be purged if a first pass run is requested. specific_parameters are extra parameters that """ video_file = os.path.abspath(video_file) # determine the name of the video if video_name is None: filename = os.path.split(video_file)[1] video_name = os.path.splitext(filename)[0] # setup the project instance result_folder = os.path.abspath(os.path.expanduser(result_folder)) folder = os.path.join(result_folder, video_name) project = cls(folder, video_name, parameters, passes) # prepare the project project.prepare_project(video_file, result_folder, video_name, parameters, passes, prepare_workfolder) return project def prepare_project(self, video_file, result_folder, video_name=None, parameters=None, passes=None, prepare_workfolder='auto'): """ prepare the work directory by setting up all necessary files """ if prepare_workfolder == 'auto': pass1_requested = (1 in self.passes) self.clean_workfolder(purge=pass1_requested) elif 'clean' in prepare_workfolder: self.clean_workfolder() elif 'purge' in prepare_workfolder: self.clean_workfolder(purge=True) # get the folder in which the current project resides folder_code = os.path.abspath(utils.files.MAIN_DIRECTORY) # setup tracking parameters tracking_parameters = self.parameters.to_dict(flatten=True) # extract the factor for the lengths and provide it separately scale_length = tracking_parameters.pop('scale_length', 1) scale_length = parameters.pop('scale_length', scale_length) # setup all variables that might be used in the templates params = {'FOLDER_CODE': folder_code, 'JOB_DIRECTORY': self.folder, 'NAME': self.name, 'VIDEO_FILE': video_file, 'TRACKING_PARAMETERS': pprint.pformat(tracking_parameters), 'SPECIFIC_PARAMETERS': pprint.pformat(parameters), 'SCALE_LENGTH': scale_length} # setup job resources resource_iter = self.parameters['resources'].iteritems(flatten=True) for key, value in resource_iter: params[key.upper()] = value # ensure that the result folder exists try: os.makedirs(self.folder) except OSError: pass # set up job scripts for pass_id in self.passes: # add job files to parameters for k, filename in enumerate(self.files_job[pass_id]): params['JOB_FILE_%d' % k] = filename params['LOG_FILE'] = os.path.join(self.folder, self.file_log_pass % pass_id) # create the job scripts for filename in self.files_job[pass_id]: script = self.get_template(filename) script = script.format(**params) open(os.path.join(self.folder, filename), 'w').write(script) # create symbolic link if requested symlink_folder = self.parameters['project/symlink_folder'] if symlink_folder: dst = os.path.join(symlink_folder, self.name) if os.path.exists(dst): os.remove(dst) try: os.symlink(self.folder, dst) except OSError as err: self.logger.warn('Symlink creation failed: %s', err) self.logger.info('Prepared project in folder %s', self.folder) def get_template(self, template): """ return the content of a chosen template """ # check whether the template is given as an absolute path if os.path.isabs(template): filename = template else: filename = os.path.join( os.path.dirname(__file__), 'templates/%s' % template ) return open(filename).read() def submit(self): """ submit the job to the cluster """ raise NotImplementedError