def checkCompletedApp(self, app): task = self._getParent() j = app._getParent() for odat in j.outputdata.outputdata: # Look out: if this is changed, there is anothher one like it below! if 0 == len([ f for f in j.outputdata.output if ".".join(odat.split(".")[:-1]) in f ]): logger.error("Job %s has not produced %s file, only: %s" % (j.id, odat, j.outputdata.output)) return False # if this is the first app to complete the partition... if self.getPartitionStatus(self._app_partition[app.id]) != "completed": task_container, subtask_dsname = task.container_name, self.dataset_name infos = {} for oinfo in j.outputdata.output: try: dq2_lock.acquire() info = oinfo.split(",") # get master replica from dataset - info not set to SE; but to ANALY_XYZ from panda master_replica = dq2.getMasterReplicaLocation(info[0]) if master_replica: info[5] = master_replica else: replicas = dq2.listDatasetReplicas(info[0]).values() if len(replicas) == 0: try: info[5] = getPandaClient().PandaSites[ info[5]]["ddm"] except KeyError: pass else: complete, incomplete = replicas[0].values() info[5] = (complete + incomplete)[0] if info[4][:3] == "ad:": info[4] = info[4][3:] finally: dq2_lock.release() datasetname = subtask_dsname + '.' + info[5] info[0] = datasetname infos.setdefault(datasetname, []).append(",".join(info)) for ds in infos.keys(): outputdata = DQ2OutputDataset() try: outputdata.create_dataset(ds) except DQDatasetExistsException: pass try: outputdata.register_datasets_details(None, infos[ds]) except DQFileExistsInDatasetException: pass # Register Container try: containerinfo = {} dq2_lock.acquire() try: containerinfo = dq2.listDatasets(task_container) except: containerinfo = {} if containerinfo == {}: try: dq2.registerContainer(task_container) logger.debug('Registered container for Task %i: %s' % (task.id, task_container)) except Exception as x: logger.error( 'Problem registering container for Task %i, %s : %s %s' % (task.id, task_container, x.__class__, x)) for ds in infos.keys(): try: dq2.registerDatasetsInContainer(task_container, [ds]) except DQContainerAlreadyHasDataset: pass except Exception as x: logger.error( 'Problem registering dataset %s in container %s: %s %s' % (subtask_dsname, task_container, x.__class__, x)) finally: dq2_lock.release() return True
def checkCompletedApp(self, app): task = self._getParent() j = app._getParent() for odat in j.outputdata.outputdata: # Look out: if this is changed, there is anothher one like it below! if 0==len([f for f in j.outputdata.output if ".".join(odat.split(".")[:-1]) in f]): logger.error("Job %s has not produced %s file, only: %s" % (j.id, odat, j.outputdata.output)) return False # if this is the first app to complete the partition... if self.getPartitionStatus(self._app_partition[app.id]) != "completed": task_container, subtask_dsname = task.container_name, self.dataset_name infos = {} for oinfo in j.outputdata.output: try: dq2_lock.acquire() info = oinfo.split(",") # get master replica from dataset - info not set to SE; but to ANALY_XYZ from panda master_replica = dq2.getMasterReplicaLocation(info[0]) if master_replica: info[5] = master_replica else: replicas = dq2.listDatasetReplicas(info[0]).values() if len(replicas) == 0: try: info[5] = getPandaClient().PandaSites[info[5]]["ddm"] except KeyError: pass else: complete, incomplete = replicas[0].values() info[5] = (complete + incomplete)[0] if info[4][:3] == "ad:": info[4] = info[4][3:] finally: dq2_lock.release() datasetname = subtask_dsname + '.' + info[5] info[0] = datasetname infos.setdefault(datasetname, []).append(",".join(info)) for ds in infos.keys(): outputdata = DQ2OutputDataset() try: outputdata.create_dataset(ds) except DQDatasetExistsException: pass try: outputdata.register_datasets_details(None, infos[ds]) except DQFileExistsInDatasetException: pass # Register Container try: containerinfo = {} dq2_lock.acquire() try: containerinfo = dq2.listDatasets(task_container) except: containerinfo = {} if containerinfo == {}: try: dq2.registerContainer(task_container) logger.debug('Registered container for Task %i: %s' % (task.id, task_container)) except Exception as x: logger.error('Problem registering container for Task %i, %s : %s %s' % (task.id, task_container,x.__class__, x)) for ds in infos.keys(): try: dq2.registerDatasetsInContainer(task_container, [ ds ] ) except DQContainerAlreadyHasDataset: pass except Exception as x: logger.error('Problem registering dataset %s in container %s: %s %s' %( subtask_dsname, task_container, x.__class__, x)) finally: dq2_lock.release() return True
def registerDataset(self): """Register in the transform container""" trf = self._getParent() trf_container = trf.getContainerName() fail = False try: containerinfo = {} dq2_lock.acquire() try: containerinfo = dq2.listDatasets(trf_container) except: containerinfo = {} if containerinfo == {}: try: dq2.registerContainer(trf_container) logger.info('Registered container for Unit %i of Transform %i: %s' % (self.getID(), trf.getID(), trf_container)) except Exception as x: logger.error('Problem registering container for Unit %i of Transform %i, %s : %s %s' % (self.getID(), trf.getID(), trf_container,x.__class__, x)) fail = True except DQException as x: logger.error('DQ2 Problem registering container for Unit %i of Transform %i, %s : %s %s' % (self.getID(), trf.getID(), trf_container,x.__class__, x)) fail = True job = GPI.jobs(self.active_job_ids[0]) ds_list = self.getOutputDatasetList() for ds in ds_list: try: dq2.registerDatasetsInContainer(trf_container, [ ds ] ) except DQContainerAlreadyHasDataset: pass except Exception as x: logger.error('Problem registering dataset %s in container %s: %s %s' %( job.outputdata.datasetname, trf_container, x.__class__, x)) fail = True except DQException as x: logger.error('DQ2 Problem registering dataset %s in container %s: %s %s' %( job.outputdata.datasetname, trf_container, x.__class__, x)) fail = True finally: dq2_lock.release() if fail: return not fail # add dataset to the task container task = trf._getParent() task_container = task.getContainerName() try: containerinfo = {} dq2_lock.acquire() try: containerinfo = dq2.listDatasets(task_container) except: containerinfo = {} if containerinfo == {}: try: dq2.registerContainer(task_container) logger.info('Registered container for Unit %i of Transform %i: %s' % (self.getID(), trf.getID(), task_container)) except Exception as x: logger.error('Problem registering container for Unit %i of Transform %i in Task %i, %s : %s %s' % (self.getID(), trf.getID(), task.getID(), task_container, x.__class__, x)) fail = True except DQException as x: logger.error('DQ2 Problem registering container for Unit %i of Transform %i in Task %i, %s : %s %s' % (self.getID(), trf.getID(), task.getID(), task_container, x.__class__, x)) fail = True ds_list = self.getOutputDatasetList() for ds in ds_list: try: dq2.registerDatasetsInContainer(task_container, [ ds ] ) except DQContainerAlreadyHasDataset: pass except Exception as x: logger.error('Problem registering dataset %s in container %s: %s %s' %( job.outputdata.datasetname, task_container, x.__class__, x)) fail = True except DQException as x: logger.error('DQ2 Problem registering dataset %s in container %s: %s %s' %( job.outputdata.datasetname, task_container, x.__class__, x)) fail = True finally: dq2_lock.release() return not fail