def start_job(self, request): # split sdm jobs across multiple algorithms, # and multiple species input datasets # TODO: rethink and maybe split jobs based on enviro input datasets? if not self.is_active(): for func in (uuidToObject(f) for f in self.context.functions): # get utility to execute this experiment method = queryUtility(IComputeMethod, name=ISDMExperiment.__identifier__) if method is None: return ('error', u"Can't find method to run SDM Experiment") # create result object: # TODO: refactor this out into helper method title = u'{} - {} {}'.format( self.context.title, func.getId(), datetime.now().strftime('%Y-%m-%dT%H:%M:%S')) result = self._create_result_container(title) # Build job_params store them on result and submit job result.job_params = { 'resolution': IBCCVLMetadata(self.context)['resolution'], 'function': func.getId(), 'species_occurrence_dataset': self.context.species_occurrence_dataset, 'species_absence_dataset': self.context.species_absence_dataset, 'species_pseudo_absence_points': self.context.species_pseudo_absence_points, 'species_number_pseudo_absence_points': self.context.species_number_pseudo_absence_points, 'environmental_datasets': self.context.environmental_datasets, } # add toolkit params: result.job_params.update(self.context.parameters[IUUID(func)]) self._createProvenance(result) # submit job LOG.info("Submit JOB %s to queue", func.getId()) method(result, func) resultjt = IJobTracker(result) resultjt.new_job('TODO: generate id', 'generate taskname: sdm_experiment') resultjt.set_progress('PENDING', u'{} pending'.format(func.getId())) return 'info', u'Job submitted {0} - {1}'.format( self.context.title, self.state) else: return 'error', u'Current Job is still running'
def start_job(self, request): if not self.is_active(): # get utility to execute this experiment method = queryUtility(IComputeMethod, name=IProjectionExperiment.__identifier__) if method is None: # TODO: lookup by script type (Perl, Python, etc...) return ('error', u"Can't find method to run Projection Experiment") expuuid = self.context.species_distribution_models.keys()[0] exp = uuidToObject(expuuid) # TODO: what if two datasets provide the same layer? # start a new job for each sdm and future dataset for sdmuuid in self.context.species_distribution_models[expuuid]: for dsuuid in self.context.future_climate_datasets: dsbrain = uuidToCatalogBrain(dsuuid) dsmd = IBCCVLMetadata(dsbrain.getObject()) futurelayers = set(dsmd['layers'].keys()) # match sdm exp layers with future dataset layers projlayers = {} for ds, dslayerset in exp.environmental_datasets.items(): # add matching layers projlayers.setdefault(dsuuid, set()).update( dslayerset.intersection(futurelayers)) # remove matching layers projlayers[ds] = dslayerset - futurelayers if not projlayers[ds]: # remove if all layers replaced del projlayers[ds] # create result result = self._create_result_container( sdmuuid, dsbrain, projlayers) # update provenance self._createProvenance(result) # submit job LOG.info("Submit JOB project to queue") method(result, "project") # TODO: wrong interface resultjt = IJobTracker(result) resultjt.new_job( 'TODO: generate id', 'generate taskname: projection experiment') resultjt.set_progress('PENDING', u'projection pending') return 'info', u'Job submitted {0} - {1}'.format( self.context.title, self.state) else: # TODO: in case there is an error should we abort the transaction # to cancel previously submitted jobs? return 'error', u'Current Job is still running'
def start_job(self, request): if not self.is_active(): # get utility to execute this experiment method = queryUtility(IComputeMethod, name=IProjectionExperiment.__identifier__) if method is None: # TODO: lookup by script type (Perl, Python, etc...) return ('error', u"Can't find method to run Projection Experiment") expuuid = self.context.species_distribution_models.keys()[0] exp = uuidToObject(expuuid) # TODO: what if two datasets provide the same layer? # start a new job for each sdm and future dataset for sdmuuid in self.context.species_distribution_models[expuuid]: for dsuuid in self.context.future_climate_datasets: dsbrain = uuidToCatalogBrain(dsuuid) dsmd = IBCCVLMetadata(dsbrain.getObject()) futurelayers = set(dsmd['layers'].keys()) # match sdm exp layers with future dataset layers projlayers = {} for ds, dslayerset in exp.environmental_datasets.items(): # add matching layers projlayers.setdefault(dsuuid, set()).update(dslayerset.intersection(futurelayers)) # remove matching layers projlayers[ds] = dslayerset - futurelayers if not projlayers[ds]: # remove if all layers replaced del projlayers[ds] # create result result = self._create_result_container(sdmuuid, dsbrain, projlayers) # update provenance self._createProvenance(result) # submit job LOG.info("Submit JOB project to queue") method(result, "project") # TODO: wrong interface resultjt = IJobTracker(result) resultjt.new_job('TODO: generate id', 'generate taskname: projection experiment') resultjt.set_progress('PENDING', u'projection pending') return 'info', u'Job submitted {0} - {1}'.format(self.context.title, self.state) else: # TODO: in case there is an error should we abort the transaction # to cancel previously submitted jobs? return 'error', u'Current Job is still running'
def start_job(self, request): if not self.is_active(): # get utility to execute this experiment method = queryUtility(IComputeMethod, name=IEnsembleExperiment.__identifier__) if method is None: return ('error', u"Can't find method to run Ensemble Experiment") # create result container title = u'{} - ensemble {}'.format( self.context.title, datetime.now().strftime('%Y-%m-%dT%H:%M:%S')) result = createContentInContainer(self.context, 'Folder', title=title) # build job_params and store on result # FIXME: probably should split ensemble jobs based on resolution # for now pick first one to make result import work # datasets is dict with expkey and list of datasets... # can probably get resolution from exp or ds dsid = self.context.datasets.values()[0][0] dsmd = IBCCVLMetadata(uuidToObject(dsid)) result.job_params = { 'datasets': list(chain.from_iterable(self.context.datasets.values())), 'resolution': dsmd['resolution'] } # update provenance self._createProvenance(result) # submit job to queue LOG.info("Submit JOB Ensemble to queue") method(result, "ensemble") # TODO: wrong interface resultjt = IJobTracker(result) resultjt.new_job('TODO: generate id', 'generate taskname: ensemble') resultjt.set_progress('PENDING', 'ensemble pending') return 'info', u'Job submitted {0} - {1}'.format( self.context.title, self.state) else: return 'error', u'Current Job is still running'
def start_job(self, request): if not self.is_active(): # get utility to execute this experiment method = queryUtility(IComputeMethod, name=ISpeciesTraitsExperiment.__identifier__) if method is None: return ('error', u"Can't find method to run Species Traits Experiment") # iterate over all datasets and group them by emsc,gcm,year algorithm = uuidToCatalogBrain(self.context.algorithm) # create result object: # TODO: refactor this out into helper method title = u'{} - {} {}'.format( self.context.title, algorithm.id, datetime.now().strftime('%Y-%m-%dT%H:%M:%S')) result = createContentInContainer(self.context, 'Folder', title=title) # Build job_params store them on result and submit job result.job_params = { 'algorithm': algorithm.id, 'formula': self.context.formula, 'data_table': self.context.data_table, } # add toolkit params: result.job_params.update(self.context.parameters[algorithm.UID]) # update provenance self._createProvenance(result) # submit job LOG.info("Submit JOB %s to queue", algorithm.id) method(result, algorithm.getObject()) resultjt = IJobTracker(result) resultjt.new_job('TODO: generate id', 'generate taskname: sdm_experiment') resultjt.set_progress('PENDING', u'{} pending'.format(algorithm.id)) return 'info', u'Job submitted {0} - {1}'.format( self.context.title, self.state) else: return 'error', u'Current Job is still running'
def start_job(self, request): if not self.is_active(): # get utility to execute this experiment method = queryUtility(IComputeMethod, name=IEnsembleExperiment.__identifier__) if method is None: return ('error', u"Can't find method to run Ensemble Experiment") # create result container title = u'{} - ensemble {}'.format( self.context.title, datetime.now().strftime('%Y-%m-%dT%H:%M:%S')) result = createContentInContainer( self.context, 'Folder', title=title) # build job_params and store on result # FIXME: probably should split ensemble jobs based on resolution # for now pick first one to make result import work # datasets is dict with expkey and list of datasets... # can probably get resolution from exp or ds dsid = self.context.datasets.values()[0][0] dsmd = IBCCVLMetadata(uuidToObject(dsid)) result.job_params = { 'datasets': list(chain.from_iterable(self.context.datasets.values())), 'resolution': dsmd['resolution'] } # update provenance self._createProvenance(result) # submit job to queue LOG.info("Submit JOB Ensemble to queue") method(result, "ensemble") # TODO: wrong interface resultjt = IJobTracker(result) resultjt.new_job('TODO: generate id', 'generate taskname: ensemble') resultjt.set_progress('PENDING', 'ensemble pending') return 'info', u'Job submitted {0} - {1}'.format(self.context.title, self.state) else: return 'error', u'Current Job is still running'
def start_job(self, request): # split sdm jobs across multiple algorithms, # and multiple species input datasets # TODO: rethink and maybe split jobs based on enviro input datasets? if not self.is_active(): for func in (uuidToObject(f) for f in self.context.functions): # get utility to execute this experiment method = queryUtility(IComputeMethod, name=ISDMExperiment.__identifier__) if method is None: return ('error', u"Can't find method to run SDM Experiment") # create result object: # TODO: refactor this out into helper method title = u'{} - {} {}'.format(self.context.title, func.getId(), datetime.now().strftime('%Y-%m-%dT%H:%M:%S')) result = self._create_result_container(title) # Build job_params store them on result and submit job result.job_params = { 'resolution': IBCCVLMetadata(self.context)['resolution'], 'function': func.getId(), 'species_occurrence_dataset': self.context.species_occurrence_dataset, 'species_absence_dataset': self.context.species_absence_dataset, 'species_pseudo_absence_points': self.context.species_pseudo_absence_points, 'species_number_pseudo_absence_points': self.context.species_number_pseudo_absence_points, 'environmental_datasets': self.context.environmental_datasets, } # add toolkit params: result.job_params.update(self.context.parameters[IUUID(func)]) self._createProvenance(result) # submit job LOG.info("Submit JOB %s to queue", func.getId()) method(result, func) resultjt = IJobTracker(result) resultjt.new_job('TODO: generate id', 'generate taskname: sdm_experiment') resultjt.set_progress('PENDING', u'{} pending'.format(func.getId())) return 'info', u'Job submitted {0} - {1}'.format(self.context.title, self.state) else: return 'error', u'Current Job is still running'
def start_job(self, request): if not self.is_active(): # get utility to execute this experiment method = queryUtility(IComputeMethod, name=ISpeciesTraitsExperiment.__identifier__) if method is None: return ('error', u"Can't find method to run Species Traits Experiment") # iterate over all datasets and group them by emsc,gcm,year algorithm = uuidToCatalogBrain(self.context.algorithm) # create result object: # TODO: refactor this out into helper method title = u'{} - {} {}'.format(self.context.title, algorithm.id, datetime.now().strftime('%Y-%m-%dT%H:%M:%S')) result = createContentInContainer( self.context, 'Folder', title=title) # Build job_params store them on result and submit job result.job_params = { 'algorithm': algorithm.id, 'formula': self.context.formula, 'data_table': self.context.data_table, } # add toolkit params: result.job_params.update(self.context.parameters[algorithm.UID]) # update provenance self._createProvenance(result) # submit job LOG.info("Submit JOB %s to queue", algorithm.id) method(result, algorithm.getObject()) resultjt = IJobTracker(result) resultjt.new_job('TODO: generate id', 'generate taskname: sdm_experiment') resultjt.set_progress('PENDING', u'{} pending'.format(algorithm.id)) return 'info', u'Job submitted {0} - {1}'.format(self.context.title, self.state) else: return 'error', u'Current Job is still running'
def start_job(self, request): # TODO: split biodiverse job across years, gcm, emsc if not self.is_active(): # get utility to execute this experiment method = queryUtility(IComputeMethod, name=IBiodiverseExperiment.__identifier__) if method is None: return ('error', u"Can't find method to run Biodiverse Experiment") # iterate over all datasets and group them by emsc,gcm,year # FIXME: add resolution grouping? datasets = {} for projds, threshold in chain.from_iterable( map(lambda x: x.items(), self.context.projection.itervalues())): dsobj = uuidToObject(projds) dsmd = IBCCVLMetadata(dsobj) emsc = dsmd.get('emsc') gcm = dsmd.get('gcm') period = dsmd.get('temporal') resolution = dsmd.get('resolution') if not period: year = 'current' else: year = Period(period).start if period else None key = (emsc, gcm, year, resolution) datasets.setdefault(key, []).append((projds, threshold)) # create one job per dataset group for key, datasets in datasets.items(): (emsc, gcm, year, resolution) = key # create result object: if year == 'current': title = u'{} - biodiverse {} {}'.format( self.context.title, year, datetime.now().strftime('%Y-%m-%dT%H:%M:%S')) else: title = u'{} - biodiverse {}_{}_{} {}'.format( self.context.title, emsc, gcm, year, datetime.now().strftime('%Y-%m-%dT%H:%M:%S')) result = createContentInContainer(self.context, 'Folder', title=title) dss = [] for ds, thresh in datasets: dss.append({'dataset': ds, 'threshold': thresh}) # build job_params and store on result result.job_params = { # datasets is a list of dicts with 'threshold' and 'uuid' 'projections': dss, 'cluster_size': self.context.cluster_size, } # update provenance self._createProvenance(result) # submit job to queue LOG.info("Submit JOB Biodiverse to queue") method(result, "biodiverse") # TODO: wrong interface resultjt = IJobTracker(result) resultjt.new_job('TODO: generate id', 'generate taskname: biodiverse') resultjt.set_progress('PENDING', 'biodiverse pending') return 'info', u'Job submitted {0} - {1}'.format( self.context.title, self.state) else: return 'error', u'Current Job is still running'
def start_job(self, request): # TODO: split biodiverse job across years, gcm, emsc if not self.is_active(): # get utility to execute this experiment method = queryUtility(IComputeMethod, name=IBiodiverseExperiment.__identifier__) if method is None: return ('error', u"Can't find method to run Biodiverse Experiment") # iterate over all datasets and group them by emsc,gcm,year # FIXME: add resolution grouping? datasets = {} for projds, threshold in chain.from_iterable(map(lambda x: x.items(), self.context.projection.itervalues())): dsobj = uuidToObject(projds) dsmd = IBCCVLMetadata(dsobj) emsc = dsmd.get('emsc') gcm = dsmd.get('gcm') period = dsmd.get('temporal') resolution = dsmd.get('resolution') if not period: year = 'current' else: year = Period(period).start if period else None key = (emsc, gcm, year, resolution) datasets.setdefault(key, []).append((projds, threshold)) # create one job per dataset group for key, datasets in datasets.items(): (emsc, gcm, year, resolution) = key # create result object: if year == 'current': title = u'{} - biodiverse {} {}'.format( self.context.title, year, datetime.now().strftime('%Y-%m-%dT%H:%M:%S')) else: title = u'{} - biodiverse {}_{}_{} {}'.format( self.context.title, emsc, gcm, year, datetime.now().strftime('%Y-%m-%dT%H:%M:%S')) result = createContentInContainer( self.context, 'Folder', title=title) dss = [] for ds, thresh in datasets: dss.append({ 'dataset': ds, 'threshold': thresh }) # build job_params and store on result result.job_params = { # datasets is a list of dicts with 'threshold' and 'uuid' 'projections': dss, 'cluster_size': self.context.cluster_size, } # update provenance self._createProvenance(result) # submit job to queue LOG.info("Submit JOB Biodiverse to queue") method(result, "biodiverse") # TODO: wrong interface resultjt = IJobTracker(result) resultjt.new_job('TODO: generate id', 'generate taskname: biodiverse') resultjt.set_progress('PENDING', 'biodiverse pending') return 'info', u'Job submitted {0} - {1}'.format(self.context.title, self.state) else: return 'error', u'Current Job is still running'