def __additionalParams(self, prodid):
callback = {}
tsClient = TransformationClient()
res = tsClient.getAdditionalParameters(prodid)
if not res['OK']:
callback = {"success": "false", "error": res["Message"]}
else:
result = res["Value"]
back = []
for i in sorted(result.keys()):
back.append([i, result[i]])
callback = {"success": "true", "result": back}
return callback
def __additionalParams(self, prodid):
callback = {}
tsClient = TransformationClient()
res = tsClient.getAdditionalParameters(prodid)
if not res['OK']:
callback = {"success": "false", "error": res["Message"]}
else:
result = res["Value"]
back = []
for i in sorted(result.keys()):
back.append([i, result[i]])
callback = {"success": "true", "result": back}
return callback
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
fc = FileCatalogClient()
fmeta = {}
trans = None
info = []
if clip.prodid:
res = tc.getTransformation(clip.prodid)
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
trans = res['Value']
res = tc.getTransformationInputDataQuery( clip.prodid )
if res['OK']:
trans['InputDataQuery'] = res['Value']
res = tc.getAdditionalParameters ( clip.prodid )
if res['OK']:
trans['AddParams'] = res['Value']
#do something with transf
res = fc.findDirectoriesByMetadata({'ProdID':clip.prodid})
if res['OK']:
if len(res['Value'].values()):
gLogger.verbose("Found some directory matching the metadata")
for dirs in res['Value'].values():
res = fc.getDirectoryMetadata(dirs)
if res['OK']:
fmeta.update(res['Value'])
else:
gLogger.warn("Failed to get dir metadata")
res = fc.listDirectory(dirs)
if not res['OK']:
def _getInfo():
"""gets info about transformation"""
clip = _Params()
clip.registerSwitches()
Script.parseCommandLine()
if not clip.prodid and not clip.filename:
Script.showHelp()
dexit(1)
from DIRAC import gLogger
import os
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
tc = TransformationClient()
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
fc = FileCatalogClient()
fmeta = {}
trans = None
info = []
if clip.prodid:
res = tc.getTransformation(clip.prodid)
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
trans = res['Value']
res = tc.getTransformationInputDataQuery( clip.prodid )
if res['OK']:
trans['InputDataQuery'] = res['Value']
res = tc.getAdditionalParameters ( clip.prodid )
if res['OK']:
trans['AddParams'] = res['Value']
#do something with transf
res1 = fc.findDirectoriesByMetadata({'ProdID':clip.prodid})
if res1['OK'] and len(res1['Value'].values()):
gLogger.verbose("Found %i directory matching the metadata" % len(res1['Value'].values()) )
for dirs in res1['Value'].values():
res = fc.getDirectoryUserMetadata(dirs)
if res['OK']:
fmeta.update(res['Value'])
else:
gLogger.error("Failed to get metadata for %s, SKIPPING" % dirs)
continue
res = fc.listDirectory(dirs)
if not res['OK']:
continue
content = res['Value']['Successful'][dirs]
if content["Files"]:
for f_ex in content["Files"].keys():
res = fc.getFileUserMetadata(f_ex)
if res['OK']:
fmeta.update(res['Value'])
break
#here we have trans and fmeta
info.append("")
info.append("Production %s has the following parameters:" % trans['TransformationID'])
info.extend(_createTransfoInfo(trans))
if fmeta:
info.append('The files created by this production have the following metadata:')
info.extend(_createFileInfo(fmeta))
info.append("It's possible that some meta data was not brought back,")
info.append("in particular file level metadata, so check some individual files")
if clip.filename:
pid = ""
if clip.filename.count("/"):
fpath = os.path.dirname(clip.filename)
res = fc.getDirectoryUserMetadata(fpath)
if not res['OK']:
gLogger.error(res['Message'])
dexit(0)
fmeta.update(res['Value'])
res = fc.getFileUserMetadata(clip.filename)
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
fmeta.update(res['Value'])
if 'ProdID' in fmeta:
pid = str(fmeta['ProdID'])
res = fc.getFileAncestors([clip.filename], 1)
if res["OK"]:
for dummy_lfn,ancestorsDict in res['Value']['Successful'].items():
if ancestorsDict.keys():
fmeta["Ancestors"] = ancestorsDict.keys()
res = fc.getFileDescendents([clip.filename], 1)
if res["OK"]:
for dummy_lfn,descendDict in res['Value']['Successful'].items():
if descendDict.keys():
fmeta['Descendants'] = descendDict.keys()
else:
ext = clip.filename.split(".")[-1]
fitems = []
for i in clip.filename.split('.')[:-1]:
fitems.extend(i.split('_'))
pid = ''
if ext == 'stdhep':
pid = fitems[fitems.index('gen')+1]
if ext == 'slcio':
if 'rec' in fitems:
pid = fitems[fitems.index('rec')+1]
elif 'dst' in fitems:
pid = fitems[fitems.index('dst')+1]
elif 'sim' in fitems:
pid = fitems[fitems.index('sim')+1]
else:
gLogger.error("This file does not follow the ILCDIRAC production conventions!")
gLogger.error("Please specify a prod ID directly or check the file.")
dexit(0)
if not pid:
gLogger.error("This file does not follow the ILCDIRAC production conventions!")
gLogger.error("Please specify a prod ID directly or check the file.")
dexit(0)
#as task follows the prod id, to get it we need
tid = fitems[fitems.index(pid)+1]
last_folder = str(int(tid)/1000).zfill(3)
res = fc.findDirectoriesByMetadata({'ProdID':int(pid)})
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
dir_ex = res['Value'].values()[0]
fpath = ""
if int(dir_ex.split("/")[-1]) == int(pid):
fpath = dir_ex+last_folder+"/"
elif int(dir_ex.split("/")[-2]) == int(pid):
fpath = "/".join(dir_ex.split('/')[:-2])+"/"+pid.zfill(8)+"/"+last_folder+"/"
else:
gLogger.error('Path does not follow conventions, will not get file family')
if fpath:
fpath += clip.filename
res = fc.getFileAncestors([fpath], 1)
if res["OK"]:
for dummy_lfn,ancestorsDict in res['Value']['Successful'].items():
fmeta["Ancestors"] = ancestorsDict.keys()
res = fc.getFileDescendents([fpath], 1)
if res["OK"]:
for dummy_lfn,descendDict in res['Value']['Successful'].items():
fmeta['Descendants'] = descendDict.keys()
res = fc.getDirectoryUserMetadata(dir_ex)
if not res['OK']:
gLogger.error(res['Message'])
else:
fmeta.update(res['Value'])
res = tc.getTransformation(pid)
if not res['OK']:
gLogger.error(res['Message'])
gLogger.error('Will proceed anyway')
else:
trans = res['Value']
res = tc.getTransformationInputDataQuery( pid )
if res['OK']:
trans['InputDataQuery'] = res['Value']
res = tc.getAdditionalParameters ( pid )
if res['OK']:
trans['AddParams'] = res['Value']
info.append("")
info.append("Input file has the following properties:")
info.extend(_createFileInfo(fmeta))
info.append("")
info.append('It was created with the production %s:' % pid)
if trans:
info.extend(_createTransfoInfo(trans))
gLogger.notice("\n".join(info))
dexit(0)
def _getInfo():
"""gets info about transformation"""
clip = _Params()
clip.registerSwitches()
Script.parseCommandLine()
if not clip.prodid and not clip.filename:
Script.showHelp()
dexit(1)
from DIRAC import gLogger
import os
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
tc = TransformationClient()
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
fc = FileCatalogClient()
fmeta = {}
trans = None
info = []
if clip.prodid:
res = tc.getTransformation(clip.prodid)
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
trans = res['Value']
res = tc.getTransformationInputDataQuery( clip.prodid )
if res['OK']:
trans['InputDataQuery'] = res['Value']
res = tc.getAdditionalParameters ( clip.prodid )
if res['OK']:
trans['AddParams'] = res['Value']
#do something with transf
res1 = fc.findDirectoriesByMetadata({'ProdID':clip.prodid})
if res1['OK'] and len(res1['Value'].values()):
gLogger.verbose("Found %i directory matching the metadata" % len(res1['Value'].values()) )
for dirs in res1['Value'].values():
res = fc.getDirectoryUserMetadata(dirs)
if res['OK']:
fmeta.update(res['Value'])
else:
gLogger.error("Failed to get metadata for %s, SKIPPING" % dirs)
continue
res = fc.listDirectory(dirs)
if not res['OK']:
continue
content = res['Value']['Successful'][dirs]
if content["Files"]:
for f_ex in content["Files"].keys():
res = fc.getFileUserMetadata(f_ex)
if res['OK']:
fmeta.update(res['Value'])
break
#here we have trans and fmeta
info.append("")
info.append("Production %s has the following parameters:" % trans['TransformationID'])
info.extend(_createTransfoInfo(trans))
if fmeta:
info.append('The files created by this production have the following metadata:')
info.extend(_createFileInfo(fmeta))
info.append("It's possible that some meta data was not brought back,")
info.append("in particular file level metadata, so check some individual files")
if clip.filename:
pid = ""
if clip.filename.count("/"):
fpath = os.path.dirname(clip.filename)
res = fc.getDirectoryUserMetadata(fpath)
if not res['OK']:
gLogger.error(res['Message'])
dexit(0)
fmeta.update(res['Value'])
res = fc.getFileUserMetadata(clip.filename)
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
fmeta.update(res['Value'])
if 'ProdID' in fmeta:
pid = str(fmeta['ProdID'])
res = fc.getFileAncestors([clip.filename], 1)
if res["OK"]:
for dummy_lfn,ancestorsDict in res['Value']['Successful'].items():
if ancestorsDict.keys():
fmeta["Ancestors"] = ancestorsDict.keys()
res = fc.getFileDescendents([clip.filename], 1)
if res["OK"]:
for dummy_lfn,descendDict in res['Value']['Successful'].items():
if descendDict.keys():
fmeta['Descendants'] = descendDict.keys()
else:
ext = clip.filename.split(".")[-1]
fitems = []
for i in clip.filename.split('.')[:-1]:
fitems.extend(i.split('_'))
pid = ''
if ext == 'stdhep':
pid = fitems[fitems.index('gen')+1]
if ext == 'slcio':
if 'rec' in fitems:
pid = fitems[fitems.index('rec')+1]
elif 'dst' in fitems:
pid = fitems[fitems.index('dst')+1]
elif 'sim' in fitems:
pid = fitems[fitems.index('sim')+1]
else:
gLogger.error("This file does not follow the ILCDIRAC production conventions!")
gLogger.error("Please specify a prod ID directly or check the file.")
dexit(0)
if not pid:
gLogger.error("This file does not follow the ILCDIRAC production conventions!")
gLogger.error("Please specify a prod ID directly or check the file.")
dexit(0)
#as task follows the prod id, to get it we need
tid = fitems[fitems.index(pid)+1]
last_folder = str(int(tid)/1000).zfill(3)
res = fc.findDirectoriesByMetadata({'ProdID':int(pid)})
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
dir_ex = res['Value'].values()[0]
fpath = ""
if int(dir_ex.split("/")[-1]) == int(pid):
fpath = dir_ex+last_folder+"/"
elif int(dir_ex.split("/")[-2]) == int(pid):
fpath = "/".join(dir_ex.split('/')[:-2])+"/"+pid.zfill(8)+"/"+last_folder+"/"
else:
gLogger.error('Path does not follow conventions, will not get file family')
if fpath:
fpath += clip.filename
res = fc.getFileAncestors([fpath], 1)
if res["OK"]:
for dummy_lfn,ancestorsDict in res['Value']['Successful'].items():
fmeta["Ancestors"] = ancestorsDict.keys()
res = fc.getFileDescendents([fpath], 1)
if res["OK"]:
for dummy_lfn,descendDict in res['Value']['Successful'].items():
fmeta['Descendants'] = descendDict.keys()
res = fc.getDirectoryUserMetadata(dir_ex)
if not res['OK']:
gLogger.error(res['Message'])
else:
fmeta.update(res['Value'])
res = tc.getTransformation(pid)
if not res['OK']:
gLogger.error(res['Message'])
gLogger.error('Will proceed anyway')
else:
trans = res['Value']
res = tc.getTransformationInputDataQuery( pid )
if res['OK']:
trans['InputDataQuery'] = res['Value']
res = tc.getAdditionalParameters ( pid )
if res['OK']:
trans['AddParams'] = res['Value']
info.append("")
info.append("Input file has the following properties:")
info.extend(_createFileInfo(fmeta))
info.append("")
info.append('It was created with the production %s:' % pid)
if trans:
info.extend(_createTransfoInfo(trans))
gLogger.notice("\n".join(info))
dexit(0)
