def checkCompletedApp(self, app):
task = self._getParent()
j = app._getParent()
for odat in j.outputdata.outputdata:
# Look out: if this is changed, there is anothher one like it below!
if 0 == len([
f for f in j.outputdata.output
if ".".join(odat.split(".")[:-1]) in f
]):
logger.error("Job %s has not produced %s file, only: %s" %
(j.id, odat, j.outputdata.output))
return False
# if this is the first app to complete the partition...
if self.getPartitionStatus(self._app_partition[app.id]) != "completed":
task_container, subtask_dsname = task.container_name, self.dataset_name
infos = {}
for oinfo in j.outputdata.output:
try:
dq2_lock.acquire()
info = oinfo.split(",")
# get master replica from dataset - info not set to SE; but to ANALY_XYZ from panda
master_replica = dq2.getMasterReplicaLocation(info[0])
if master_replica:
info[5] = master_replica
else:
replicas = dq2.listDatasetReplicas(info[0]).values()
if len(replicas) == 0:
try:
info[5] = getPandaClient().PandaSites[
info[5]]["ddm"]
except KeyError:
pass
else:
complete, incomplete = replicas[0].values()
info[5] = (complete + incomplete)[0]
if info[4][:3] == "ad:":
info[4] = info[4][3:]
finally:
dq2_lock.release()
datasetname = subtask_dsname + '.' + info[5]
info[0] = datasetname
infos.setdefault(datasetname, []).append(",".join(info))
for ds in infos.keys():
outputdata = DQ2OutputDataset()
try:
outputdata.create_dataset(ds)
except DQDatasetExistsException:
pass
try:
outputdata.register_datasets_details(None, infos[ds])
except DQFileExistsInDatasetException:
pass
# Register Container
try:
containerinfo = {}
dq2_lock.acquire()
try:
containerinfo = dq2.listDatasets(task_container)
except:
containerinfo = {}
if containerinfo == {}:
try:
dq2.registerContainer(task_container)
logger.debug('Registered container for Task %i: %s' %
(task.id, task_container))
except Exception as x:
logger.error(
'Problem registering container for Task %i, %s : %s %s'
% (task.id, task_container, x.__class__, x))
for ds in infos.keys():
try:
dq2.registerDatasetsInContainer(task_container, [ds])
except DQContainerAlreadyHasDataset:
pass
except Exception as x:
logger.error(
'Problem registering dataset %s in container %s: %s %s'
% (subtask_dsname, task_container, x.__class__, x))
finally:
dq2_lock.release()
return True
def checkCompletedApp(self, app):
task = self._getParent()
j = app._getParent()
for odat in j.outputdata.outputdata:
# Look out: if this is changed, there is anothher one like it below!
if 0==len([f for f in j.outputdata.output if ".".join(odat.split(".")[:-1]) in f]):
logger.error("Job %s has not produced %s file, only: %s" % (j.id, odat, j.outputdata.output))
return False
# if this is the first app to complete the partition...
if self.getPartitionStatus(self._app_partition[app.id]) != "completed":
task_container, subtask_dsname = task.container_name, self.dataset_name
infos = {}
for oinfo in j.outputdata.output:
try:
dq2_lock.acquire()
info = oinfo.split(",")
# get master replica from dataset - info not set to SE; but to ANALY_XYZ from panda
master_replica = dq2.getMasterReplicaLocation(info[0])
if master_replica:
info[5] = master_replica
else:
replicas = dq2.listDatasetReplicas(info[0]).values()
if len(replicas) == 0:
try:
info[5] = getPandaClient().PandaSites[info[5]]["ddm"]
except KeyError:
pass
else:
complete, incomplete = replicas[0].values()
info[5] = (complete + incomplete)[0]
if info[4][:3] == "ad:":
info[4] = info[4][3:]
finally:
dq2_lock.release()
datasetname = subtask_dsname + '.' + info[5]
info[0] = datasetname
infos.setdefault(datasetname, []).append(",".join(info))
for ds in infos.keys():
outputdata = DQ2OutputDataset()
try:
outputdata.create_dataset(ds)
except DQDatasetExistsException:
pass
try:
outputdata.register_datasets_details(None, infos[ds])
except DQFileExistsInDatasetException:
pass
# Register Container
try:
containerinfo = {}
dq2_lock.acquire()
try:
containerinfo = dq2.listDatasets(task_container)
except:
containerinfo = {}
if containerinfo == {}:
try:
dq2.registerContainer(task_container)
logger.debug('Registered container for Task %i: %s' % (task.id, task_container))
except Exception as x:
logger.error('Problem registering container for Task %i, %s : %s %s' % (task.id, task_container,x.__class__, x))
for ds in infos.keys():
try:
dq2.registerDatasetsInContainer(task_container, [ ds ] )
except DQContainerAlreadyHasDataset:
pass
except Exception as x:
logger.error('Problem registering dataset %s in container %s: %s %s' %( subtask_dsname, task_container, x.__class__, x))
finally:
dq2_lock.release()
return True
def registerDataset(self):
"""Register in the transform container"""
trf = self._getParent()
trf_container = trf.getContainerName()
fail = False
try:
containerinfo = {}
dq2_lock.acquire()
try:
containerinfo = dq2.listDatasets(trf_container)
except:
containerinfo = {}
if containerinfo == {}:
try:
dq2.registerContainer(trf_container)
logger.info('Registered container for Unit %i of Transform %i: %s' % (self.getID(), trf.getID(), trf_container))
except Exception as x:
logger.error('Problem registering container for Unit %i of Transform %i, %s : %s %s' % (self.getID(), trf.getID(), trf_container,x.__class__, x))
fail = True
except DQException as x:
logger.error('DQ2 Problem registering container for Unit %i of Transform %i, %s : %s %s' % (self.getID(), trf.getID(), trf_container,x.__class__, x))
fail = True
job = GPI.jobs(self.active_job_ids[0])
ds_list = self.getOutputDatasetList()
for ds in ds_list:
try:
dq2.registerDatasetsInContainer(trf_container, [ ds ] )
except DQContainerAlreadyHasDataset:
pass
except Exception as x:
logger.error('Problem registering dataset %s in container %s: %s %s' %( job.outputdata.datasetname, trf_container, x.__class__, x))
fail = True
except DQException as x:
logger.error('DQ2 Problem registering dataset %s in container %s: %s %s' %( job.outputdata.datasetname, trf_container, x.__class__, x))
fail = True
finally:
dq2_lock.release()
if fail:
return not fail
# add dataset to the task container
task = trf._getParent()
task_container = task.getContainerName()
try:
containerinfo = {}
dq2_lock.acquire()
try:
containerinfo = dq2.listDatasets(task_container)
except:
containerinfo = {}
if containerinfo == {}:
try:
dq2.registerContainer(task_container)
logger.info('Registered container for Unit %i of Transform %i: %s' % (self.getID(), trf.getID(), task_container))
except Exception as x:
logger.error('Problem registering container for Unit %i of Transform %i in Task %i, %s : %s %s' %
(self.getID(), trf.getID(), task.getID(), task_container, x.__class__, x))
fail = True
except DQException as x:
logger.error('DQ2 Problem registering container for Unit %i of Transform %i in Task %i, %s : %s %s' %
(self.getID(), trf.getID(), task.getID(), task_container, x.__class__, x))
fail = True
ds_list = self.getOutputDatasetList()
for ds in ds_list:
try:
dq2.registerDatasetsInContainer(task_container, [ ds ] )
except DQContainerAlreadyHasDataset:
pass
except Exception as x:
logger.error('Problem registering dataset %s in container %s: %s %s' %( job.outputdata.datasetname, task_container, x.__class__, x))
fail = True
except DQException as x:
logger.error('DQ2 Problem registering dataset %s in container %s: %s %s' %( job.outputdata.datasetname, task_container, x.__class__, x))
fail = True
finally:
dq2_lock.release()
return not fail
def registerDataset(self):
"""Register in the transform container"""
trf = self._getParent()
trf_container = trf.getContainerName()
fail = False
try:
containerinfo = {}
dq2_lock.acquire()
try:
containerinfo = dq2.listDatasets(trf_container)
except:
containerinfo = {}
if containerinfo == {}:
try:
dq2.registerContainer(trf_container)
logger.info('Registered container for Unit %i of Transform %i: %s' % (self.getID(), trf.getID(), trf_container))
except Exception as x:
logger.error('Problem registering container for Unit %i of Transform %i, %s : %s %s' % (self.getID(), trf.getID(), trf_container,x.__class__, x))
fail = True
except DQException as x:
logger.error('DQ2 Problem registering container for Unit %i of Transform %i, %s : %s %s' % (self.getID(), trf.getID(), trf_container,x.__class__, x))
fail = True
job = GPI.jobs(self.active_job_ids[0])
ds_list = self.getOutputDatasetList()
for ds in ds_list:
try:
dq2.registerDatasetsInContainer(trf_container, [ ds ] )
except DQContainerAlreadyHasDataset:
pass
except Exception as x:
logger.error('Problem registering dataset %s in container %s: %s %s' %( job.outputdata.datasetname, trf_container, x.__class__, x))
fail = True
except DQException as x:
logger.error('DQ2 Problem registering dataset %s in container %s: %s %s' %( job.outputdata.datasetname, trf_container, x.__class__, x))
fail = True
finally:
dq2_lock.release()
if fail:
return not fail
# add dataset to the task container
task = trf._getParent()
task_container = task.getContainerName()
try:
containerinfo = {}
dq2_lock.acquire()
try:
containerinfo = dq2.listDatasets(task_container)
except:
containerinfo = {}
if containerinfo == {}:
try:
dq2.registerContainer(task_container)
logger.info('Registered container for Unit %i of Transform %i: %s' % (self.getID(), trf.getID(), task_container))
except Exception as x:
logger.error('Problem registering container for Unit %i of Transform %i in Task %i, %s : %s %s' %
(self.getID(), trf.getID(), task.getID(), task_container, x.__class__, x))
fail = True
except DQException as x:
logger.error('DQ2 Problem registering container for Unit %i of Transform %i in Task %i, %s : %s %s' %
(self.getID(), trf.getID(), task.getID(), task_container, x.__class__, x))
fail = True
ds_list = self.getOutputDatasetList()
for ds in ds_list:
try:
dq2.registerDatasetsInContainer(task_container, [ ds ] )
except DQContainerAlreadyHasDataset:
pass
except Exception as x:
logger.error('Problem registering dataset %s in container %s: %s %s' %( job.outputdata.datasetname, task_container, x.__class__, x))
fail = True
except DQException as x:
logger.error('DQ2 Problem registering dataset %s in container %s: %s %s' %( job.outputdata.datasetname, task_container, x.__class__, x))
fail = True
finally:
dq2_lock.release()
return not fail
