def start_job(self, request):
# split sdm jobs across multiple algorithms,
# and multiple species input datasets
# TODO: rethink and maybe split jobs based on enviro input datasets?
if not self.is_active():
for func in (uuidToObject(f) for f in self.context.functions):
# get utility to execute this experiment
method = queryUtility(IComputeMethod,
name=ISDMExperiment.__identifier__)
if method is None:
return ('error',
u"Can't find method to run SDM Experiment")
# create result object:
# TODO: refactor this out into helper method
title = u'{} - {} {}'.format(
self.context.title, func.getId(),
datetime.now().strftime('%Y-%m-%dT%H:%M:%S'))
result = self._create_result_container(title)
# Build job_params store them on result and submit job
result.job_params = {
'resolution': IBCCVLMetadata(self.context)['resolution'],
'function': func.getId(),
'species_occurrence_dataset':
self.context.species_occurrence_dataset,
'species_absence_dataset':
self.context.species_absence_dataset,
'species_pseudo_absence_points':
self.context.species_pseudo_absence_points,
'species_number_pseudo_absence_points':
self.context.species_number_pseudo_absence_points,
'environmental_datasets':
self.context.environmental_datasets,
}
# add toolkit params:
result.job_params.update(self.context.parameters[IUUID(func)])
self._createProvenance(result)
# submit job
LOG.info("Submit JOB %s to queue", func.getId())
method(result, func)
resultjt = IJobTracker(result)
resultjt.new_job('TODO: generate id',
'generate taskname: sdm_experiment')
resultjt.set_progress('PENDING',
u'{} pending'.format(func.getId()))
return 'info', u'Job submitted {0} - {1}'.format(
self.context.title, self.state)
else:
return 'error', u'Current Job is still running'
def start_job(self, request):
if not self.is_active():
# get utility to execute this experiment
method = queryUtility(IComputeMethod,
name=IProjectionExperiment.__identifier__)
if method is None:
# TODO: lookup by script type (Perl, Python, etc...)
return ('error',
u"Can't find method to run Projection Experiment")
expuuid = self.context.species_distribution_models.keys()[0]
exp = uuidToObject(expuuid)
# TODO: what if two datasets provide the same layer?
# start a new job for each sdm and future dataset
for sdmuuid in self.context.species_distribution_models[expuuid]:
for dsuuid in self.context.future_climate_datasets:
dsbrain = uuidToCatalogBrain(dsuuid)
dsmd = IBCCVLMetadata(dsbrain.getObject())
futurelayers = set(dsmd['layers'].keys())
# match sdm exp layers with future dataset layers
projlayers = {}
for ds, dslayerset in exp.environmental_datasets.items():
# add matching layers
projlayers.setdefault(dsuuid, set()).update(
dslayerset.intersection(futurelayers))
# remove matching layers
projlayers[ds] = dslayerset - futurelayers
if not projlayers[ds]:
# remove if all layers replaced
del projlayers[ds]
# create result
result = self._create_result_container(
sdmuuid, dsbrain, projlayers)
# update provenance
self._createProvenance(result)
# submit job
LOG.info("Submit JOB project to queue")
method(result, "project") # TODO: wrong interface
resultjt = IJobTracker(result)
resultjt.new_job(
'TODO: generate id',
'generate taskname: projection experiment')
resultjt.set_progress('PENDING', u'projection pending')
return 'info', u'Job submitted {0} - {1}'.format(
self.context.title, self.state)
else:
# TODO: in case there is an error should we abort the transaction
# to cancel previously submitted jobs?
return 'error', u'Current Job is still running'
def start_job(self, request):
if not self.is_active():
# get utility to execute this experiment
method = queryUtility(IComputeMethod,
name=IProjectionExperiment.__identifier__)
if method is None:
# TODO: lookup by script type (Perl, Python, etc...)
return ('error',
u"Can't find method to run Projection Experiment")
expuuid = self.context.species_distribution_models.keys()[0]
exp = uuidToObject(expuuid)
# TODO: what if two datasets provide the same layer?
# start a new job for each sdm and future dataset
for sdmuuid in self.context.species_distribution_models[expuuid]:
for dsuuid in self.context.future_climate_datasets:
dsbrain = uuidToCatalogBrain(dsuuid)
dsmd = IBCCVLMetadata(dsbrain.getObject())
futurelayers = set(dsmd['layers'].keys())
# match sdm exp layers with future dataset layers
projlayers = {}
for ds, dslayerset in exp.environmental_datasets.items():
# add matching layers
projlayers.setdefault(dsuuid, set()).update(dslayerset.intersection(futurelayers))
# remove matching layers
projlayers[ds] = dslayerset - futurelayers
if not projlayers[ds]:
# remove if all layers replaced
del projlayers[ds]
# create result
result = self._create_result_container(sdmuuid, dsbrain, projlayers)
# update provenance
self._createProvenance(result)
# submit job
LOG.info("Submit JOB project to queue")
method(result, "project") # TODO: wrong interface
resultjt = IJobTracker(result)
resultjt.new_job('TODO: generate id',
'generate taskname: projection experiment')
resultjt.set_progress('PENDING',
u'projection pending')
return 'info', u'Job submitted {0} - {1}'.format(self.context.title, self.state)
else:
# TODO: in case there is an error should we abort the transaction
# to cancel previously submitted jobs?
return 'error', u'Current Job is still running'
def start_job(self, request):
if not self.is_active():
# get utility to execute this experiment
method = queryUtility(IComputeMethod,
name=IEnsembleExperiment.__identifier__)
if method is None:
return ('error',
u"Can't find method to run Ensemble Experiment")
# create result container
title = u'{} - ensemble {}'.format(
self.context.title,
datetime.now().strftime('%Y-%m-%dT%H:%M:%S'))
result = createContentInContainer(self.context,
'Folder',
title=title)
# build job_params and store on result
# FIXME: probably should split ensemble jobs based on resolution
# for now pick first one to make result import work
# datasets is dict with expkey and list of datasets...
# can probably get resolution from exp or ds
dsid = self.context.datasets.values()[0][0]
dsmd = IBCCVLMetadata(uuidToObject(dsid))
result.job_params = {
'datasets':
list(chain.from_iterable(self.context.datasets.values())),
'resolution':
dsmd['resolution']
}
# update provenance
self._createProvenance(result)
# submit job to queue
LOG.info("Submit JOB Ensemble to queue")
method(result, "ensemble") # TODO: wrong interface
resultjt = IJobTracker(result)
resultjt.new_job('TODO: generate id',
'generate taskname: ensemble')
resultjt.set_progress('PENDING', 'ensemble pending')
return 'info', u'Job submitted {0} - {1}'.format(
self.context.title, self.state)
else:
return 'error', u'Current Job is still running'
def start_job(self, request):
if not self.is_active():
# get utility to execute this experiment
method = queryUtility(IComputeMethod,
name=ISpeciesTraitsExperiment.__identifier__)
if method is None:
return ('error',
u"Can't find method to run Species Traits Experiment")
# iterate over all datasets and group them by emsc,gcm,year
algorithm = uuidToCatalogBrain(self.context.algorithm)
# create result object:
# TODO: refactor this out into helper method
title = u'{} - {} {}'.format(
self.context.title, algorithm.id,
datetime.now().strftime('%Y-%m-%dT%H:%M:%S'))
result = createContentInContainer(self.context,
'Folder',
title=title)
# Build job_params store them on result and submit job
result.job_params = {
'algorithm': algorithm.id,
'formula': self.context.formula,
'data_table': self.context.data_table,
}
# add toolkit params:
result.job_params.update(self.context.parameters[algorithm.UID])
# update provenance
self._createProvenance(result)
# submit job
LOG.info("Submit JOB %s to queue", algorithm.id)
method(result, algorithm.getObject())
resultjt = IJobTracker(result)
resultjt.new_job('TODO: generate id',
'generate taskname: sdm_experiment')
resultjt.set_progress('PENDING',
u'{} pending'.format(algorithm.id))
return 'info', u'Job submitted {0} - {1}'.format(
self.context.title, self.state)
else:
return 'error', u'Current Job is still running'
def start_job(self, request):
if not self.is_active():
# get utility to execute this experiment
method = queryUtility(IComputeMethod,
name=IEnsembleExperiment.__identifier__)
if method is None:
return ('error',
u"Can't find method to run Ensemble Experiment")
# create result container
title = u'{} - ensemble {}'.format(
self.context.title, datetime.now().strftime('%Y-%m-%dT%H:%M:%S'))
result = createContentInContainer(
self.context,
'Folder',
title=title)
# build job_params and store on result
# FIXME: probably should split ensemble jobs based on resolution
# for now pick first one to make result import work
# datasets is dict with expkey and list of datasets...
# can probably get resolution from exp or ds
dsid = self.context.datasets.values()[0][0]
dsmd = IBCCVLMetadata(uuidToObject(dsid))
result.job_params = {
'datasets': list(chain.from_iterable(self.context.datasets.values())),
'resolution': dsmd['resolution']
}
# update provenance
self._createProvenance(result)
# submit job to queue
LOG.info("Submit JOB Ensemble to queue")
method(result, "ensemble") # TODO: wrong interface
resultjt = IJobTracker(result)
resultjt.new_job('TODO: generate id',
'generate taskname: ensemble')
resultjt.set_progress('PENDING',
'ensemble pending')
return 'info', u'Job submitted {0} - {1}'.format(self.context.title, self.state)
else:
return 'error', u'Current Job is still running'
def start_job(self, request):
# split sdm jobs across multiple algorithms,
# and multiple species input datasets
# TODO: rethink and maybe split jobs based on enviro input datasets?
if not self.is_active():
for func in (uuidToObject(f) for f in self.context.functions):
# get utility to execute this experiment
method = queryUtility(IComputeMethod,
name=ISDMExperiment.__identifier__)
if method is None:
return ('error',
u"Can't find method to run SDM Experiment")
# create result object:
# TODO: refactor this out into helper method
title = u'{} - {} {}'.format(self.context.title, func.getId(),
datetime.now().strftime('%Y-%m-%dT%H:%M:%S'))
result = self._create_result_container(title)
# Build job_params store them on result and submit job
result.job_params = {
'resolution': IBCCVLMetadata(self.context)['resolution'],
'function': func.getId(),
'species_occurrence_dataset': self.context.species_occurrence_dataset,
'species_absence_dataset': self.context.species_absence_dataset,
'species_pseudo_absence_points': self.context.species_pseudo_absence_points,
'species_number_pseudo_absence_points': self.context.species_number_pseudo_absence_points,
'environmental_datasets': self.context.environmental_datasets,
}
# add toolkit params:
result.job_params.update(self.context.parameters[IUUID(func)])
self._createProvenance(result)
# submit job
LOG.info("Submit JOB %s to queue", func.getId())
method(result, func)
resultjt = IJobTracker(result)
resultjt.new_job('TODO: generate id',
'generate taskname: sdm_experiment')
resultjt.set_progress('PENDING',
u'{} pending'.format(func.getId()))
return 'info', u'Job submitted {0} - {1}'.format(self.context.title, self.state)
else:
return 'error', u'Current Job is still running'
def start_job(self, request):
if not self.is_active():
# get utility to execute this experiment
method = queryUtility(IComputeMethod,
name=ISpeciesTraitsExperiment.__identifier__)
if method is None:
return ('error',
u"Can't find method to run Species Traits Experiment")
# iterate over all datasets and group them by emsc,gcm,year
algorithm = uuidToCatalogBrain(self.context.algorithm)
# create result object:
# TODO: refactor this out into helper method
title = u'{} - {} {}'.format(self.context.title, algorithm.id,
datetime.now().strftime('%Y-%m-%dT%H:%M:%S'))
result = createContentInContainer(
self.context,
'Folder',
title=title)
# Build job_params store them on result and submit job
result.job_params = {
'algorithm': algorithm.id,
'formula': self.context.formula,
'data_table': self.context.data_table,
}
# add toolkit params:
result.job_params.update(self.context.parameters[algorithm.UID])
# update provenance
self._createProvenance(result)
# submit job
LOG.info("Submit JOB %s to queue", algorithm.id)
method(result, algorithm.getObject())
resultjt = IJobTracker(result)
resultjt.new_job('TODO: generate id',
'generate taskname: sdm_experiment')
resultjt.set_progress('PENDING',
u'{} pending'.format(algorithm.id))
return 'info', u'Job submitted {0} - {1}'.format(self.context.title, self.state)
else:
return 'error', u'Current Job is still running'
def start_job(self, request):
# TODO: split biodiverse job across years, gcm, emsc
if not self.is_active():
# get utility to execute this experiment
method = queryUtility(IComputeMethod,
name=IBiodiverseExperiment.__identifier__)
if method is None:
return ('error',
u"Can't find method to run Biodiverse Experiment")
# iterate over all datasets and group them by emsc,gcm,year
# FIXME: add resolution grouping?
datasets = {}
for projds, threshold in chain.from_iterable(
map(lambda x: x.items(),
self.context.projection.itervalues())):
dsobj = uuidToObject(projds)
dsmd = IBCCVLMetadata(dsobj)
emsc = dsmd.get('emsc')
gcm = dsmd.get('gcm')
period = dsmd.get('temporal')
resolution = dsmd.get('resolution')
if not period:
year = 'current'
else:
year = Period(period).start if period else None
key = (emsc, gcm, year, resolution)
datasets.setdefault(key, []).append((projds, threshold))
# create one job per dataset group
for key, datasets in datasets.items():
(emsc, gcm, year, resolution) = key
# create result object:
if year == 'current':
title = u'{} - biodiverse {} {}'.format(
self.context.title, year,
datetime.now().strftime('%Y-%m-%dT%H:%M:%S'))
else:
title = u'{} - biodiverse {}_{}_{} {}'.format(
self.context.title, emsc, gcm, year,
datetime.now().strftime('%Y-%m-%dT%H:%M:%S'))
result = createContentInContainer(self.context,
'Folder',
title=title)
dss = []
for ds, thresh in datasets:
dss.append({'dataset': ds, 'threshold': thresh})
# build job_params and store on result
result.job_params = {
# datasets is a list of dicts with 'threshold' and 'uuid'
'projections': dss,
'cluster_size': self.context.cluster_size,
}
# update provenance
self._createProvenance(result)
# submit job to queue
LOG.info("Submit JOB Biodiverse to queue")
method(result, "biodiverse") # TODO: wrong interface
resultjt = IJobTracker(result)
resultjt.new_job('TODO: generate id',
'generate taskname: biodiverse')
resultjt.set_progress('PENDING', 'biodiverse pending')
return 'info', u'Job submitted {0} - {1}'.format(
self.context.title, self.state)
else:
return 'error', u'Current Job is still running'
def start_job(self, request):
# TODO: split biodiverse job across years, gcm, emsc
if not self.is_active():
# get utility to execute this experiment
method = queryUtility(IComputeMethod,
name=IBiodiverseExperiment.__identifier__)
if method is None:
return ('error',
u"Can't find method to run Biodiverse Experiment")
# iterate over all datasets and group them by emsc,gcm,year
# FIXME: add resolution grouping?
datasets = {}
for projds, threshold in chain.from_iterable(map(lambda x: x.items(), self.context.projection.itervalues())):
dsobj = uuidToObject(projds)
dsmd = IBCCVLMetadata(dsobj)
emsc = dsmd.get('emsc')
gcm = dsmd.get('gcm')
period = dsmd.get('temporal')
resolution = dsmd.get('resolution')
if not period:
year = 'current'
else:
year = Period(period).start if period else None
key = (emsc, gcm, year, resolution)
datasets.setdefault(key, []).append((projds, threshold))
# create one job per dataset group
for key, datasets in datasets.items():
(emsc, gcm, year, resolution) = key
# create result object:
if year == 'current':
title = u'{} - biodiverse {} {}'.format(
self.context.title, year,
datetime.now().strftime('%Y-%m-%dT%H:%M:%S'))
else:
title = u'{} - biodiverse {}_{}_{} {}'.format(
self.context.title, emsc, gcm, year,
datetime.now().strftime('%Y-%m-%dT%H:%M:%S'))
result = createContentInContainer(
self.context,
'Folder',
title=title)
dss = []
for ds, thresh in datasets:
dss.append({
'dataset': ds,
'threshold': thresh
})
# build job_params and store on result
result.job_params = {
# datasets is a list of dicts with 'threshold' and 'uuid'
'projections': dss,
'cluster_size': self.context.cluster_size,
}
# update provenance
self._createProvenance(result)
# submit job to queue
LOG.info("Submit JOB Biodiverse to queue")
method(result, "biodiverse") # TODO: wrong interface
resultjt = IJobTracker(result)
resultjt.new_job('TODO: generate id',
'generate taskname: biodiverse')
resultjt.set_progress('PENDING',
'biodiverse pending')
return 'info', u'Job submitted {0} - {1}'.format(self.context.title, self.state)
else:
return 'error', u'Current Job is still running'
