def select(self, minid=None, maxid=None, **attrs):
import repr
from Ganga.GPIDev.Lib.Job.Job import Job
if isType(minid, Job):
if minid.master:
minid = minid.master.id
else:
minid = minid.id
if maxid is None:
maxid = minid
if isType(maxid, Job):
if maxid.master:
maxid = maxid.master.id
else:
maxid = maxid.id
logger = getLogger()
this_repr = repr.Repr()
from Ganga.GPIDev.Base.Proxy import GPIProxyObjectFactory
attrs_str = ""
## Loop through all possible input combinations to constructa string representation of the attrs from possible inputs
## Reuired to flatten the additional arguments into a flat string in attrs_str
for a in attrs:
from inspect import isclass
if isclass(attrs[a]):
this_attr = GPIProxyObjectFactory(attrs[a]())
else:
from Ganga.GPIDev.Base.Objects import GangaObject
if isType(attrs[a], GangaObject):
this_attr = GPIProxyObjectFactory(attrs[a])
else:
if type(attrs[a]) is str:
from Ganga.GPIDev.Base.Proxy import getRuntimeGPIObject
this_attr = getRuntimeGPIObject(attrs[a], True)
else:
this_attr = attrs[a]
full_str = str(this_attr)
split_str = full_str.split('\n')
for line in split_str:
line = line.strip()
flat_str = ''.join(split_str)
attrs_str += ", %s=\"%s\"" % (str(a), flat_str)
logger.debug("Attrs_Str: %s" % str(attrs_str))
logger.debug("Constructing slice: %s" %
str("%s.select(minid='%s', maxid='%s'%s)" %
(self.name, this_repr.repr(minid),
this_repr.repr(maxid), attrs_str)))
this_slice = self.__class__("%s.select(minid='%s', maxid='%s'%s)" %
(self.name, this_repr.repr(minid),
this_repr.repr(maxid), attrs_str))
def append(id, obj):
this_slice.objects[id] = obj
self.do_select(append, minid, maxid, **attrs)
return this_slice
def mass_line_processor(line, mass_file):
lineParts = line.split(' ')
pattern = lineParts[1]
outputPath = lineParts[2]
name = os.path.basename(outputPath).strip('.gz')
if regex.search(mass_file.namePattern) is not None:
if outputPath == 'ERROR':
logger.error("Failed to upload file to mass storage")
logger.error(line[line.find('ERROR') + 5:])
d = MassStorageFile(namePattern=pattern)
d.compressed = mass_file.compressed
d.failureReason = line[line.find('ERROR') + 5:]
mass_file.subfiles.append(GPIProxyObjectFactory(d))
else:
d = MassStorageFile(namePattern=name)
d.compressed = mass_file.compressed
d.outputfilenameformat = mass_file.outputfilenameformat
mass_file.subfiles.append(GPIProxyObjectFactory(d))
mass_line_processor(line, d)
elif name == mass_file.namePattern:
if outputPath == 'ERROR':
logger.error("Failed to upload file to mass storage")
logger.error(line[line.find('ERROR') + 5:])
mass_file.failureReason = line[line.find('ERROR') + 5:]
return
mass_file.locations = [outputPath.strip('\n')]
def __getitem__(self, i):
'''Proivdes scripting (e.g. ds[2] returns the 3rd file) '''
if isinstance(i, type(slice(0))):
ds = GangaDataset(files=self.files[i])
return GPIProxyObjectFactory(ds)
else:
return GPIProxyObjectFactory(self.files[i])
def listShareDirContents(self, shared_directory_name):
"""
Function which is used to list the contents of the prepared sandbox folder managed by this app
Args:
shared_directory_name (str): full name of directory managed by this app
"""
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.ls(shared_directory_name)
def listShareDirContents(self, shared_directory_name):
"""
Function which is used to list the contents of the prepared sandbox folder managed by this app
Args:
shared_directory_name (str): full name of directory managed by this app
"""
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.ls(shared_directory_name)
def __getitem__(self, i):
'''Proivdes scripting (e.g. ds[2] returns the 3rd file) '''
if type(i) == type(slice(0)):
ds = BesDataset(files=self.files[i])
ds.depth = self.depth
ds.XMLCatalogueSlice = self.XMLCatalogueSlice
return GPIProxyObjectFactory(ds)
else:
return GPIProxyObjectFactory(self.files[i])
def __incrementShareRef(obj, attr_name):
shared_dir = getattr(obj, attr_name)
if hasattr(shared_dir, 'name'):
from Ganga.Core.GangaRepository import getRegistry
from Ganga.GPIDev.Base.Proxy import GPIProxyObjectFactory
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
logger.debug("Increasing shareref")
shareref.increase(shared_dir.name)
def decrementShareCounter(self, shared_directory_name, remove=0):
"""
Function which is used to decrement the number of (sub)jobs which share the prepared sandbox
managed by this app
Args:
shared_directory_name (str): full name of directory managed by this app
"""
remove = remove
logger.debug('Decrementing shared directory reference counter')
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.decrease(shared_directory_name, remove)
def decrementShareCounter(self, shared_directory_name, remove=0):
"""
Function which is used to decrement the number of (sub)jobs which share the prepared sandbox
managed by this app
Args:
shared_directory_name (str): full name of directory managed by this app
"""
remove = remove
logger.debug('Decrementing shared directory reference counter')
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.decrease(shared_directory_name, remove)
def union(self, other):
'''Returns a new data set w/ files from this and other.'''
files = set(self.getFullFileNames()).union(other.getFullFileNames())
data = BesDataset()
data.__construct__([list(files)])
data.depth = self.depth
return GPIProxyObjectFactory(data)
def __deepcopy__(self, memo = None):
self._getReadAccess()
c = super(GangaObject,self).__deepcopy__(memo)
for name,item in self._schema.allItems():
if not item['copyable']:
setattr(c,name,self._schema.getDefaultValue(name))
if item.isA(Schema.SharedItem):
shared_dir=getattr(c,name)
try:
from Ganga.Core.GangaRepository import getRegistry
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
logger.debug("Increasing shareref")
shareref.increase(shared_dir.name)
except AttributeError:
pass
return c
def difference(self, other):
'''Returns a new data set w/ files in this that are not in other.'''
other_files = other.getFullFileNames()
files = set(self.getFullFileNames()).difference(other_files)
data = BesDataset()
data.__construct__([list(files)])
data.depth = self.depth
return GPIProxyObjectFactory(data)
def intersection(self, other):
'''Returns a new data set w/ files common to this and other.'''
other_files = other.getFullFileNames()
files = set(self.getFullFileNames()).intersection(other_files)
data = BesDataset()
data.__construct__([list(files)])
data.depth = self.depth
return GPIProxyObjectFactory(data)
def getOutputDataLFNs(self):
"""Get a list of outputdata that has been uploaded by Dirac. Excludes
the outputsandbox if it is there."""
lfns = super(Dirac, self).getOutputDataLFNs()
ds = LHCbDataset()
for f in lfns:
ds.files.append(DiracFile(lfn=f))
return GPIProxyObjectFactory(ds)
def download(self, dir='.'):
'Downloads the LFN to dir (default is current directory).'
dir = expandfilename(dir)
dir = os.path.abspath(dir)
cmd = 'result = DiracCommands.getFile("%s","%s")' % (self.name, dir)
result = get_result(cmd, 'Problem during download', 'Download error.')
from PhysicalFile import PhysicalFile
return GPIProxyObjectFactory(PhysicalFile(name=result['Value']))
def symmetricDifference(self, other):
'''Returns a new data set w/ files in either this or other but not
both.'''
other_files = other.getFullFileNames()
files = set(self.getFullFileNames()).symmetric_difference(other_files)
data = BesDataset()
data.__construct__([list(files)])
data.depth = self.depth
return GPIProxyObjectFactory(data)
def getOutputDataLFNs(self, force_query=False):
"""Get a list of outputdata that has been uploaded by Dirac. Excludes
the outputsandbox if it is there."""
global dirac_ganga_server
lfns = self._getOutputDataLFNs(dirac_ganga_server, force_query)
ds = BesDataset()
for f in lfns:
ds.files.append(LogicalFile(f))
return GPIProxyObjectFactory(ds)
def getOutputData(self, outputDir=None, names=None, force=False):
"""Retrieve data stored on SE to outputDir (default=job output workspace).
If names=None, then all outputdata is downloaded otherwise names should
be a list of files to download. If force is True then download performed
even if data already exists."""
downloaded_files = super(Dirac, self).getOutputData(outputDir, names, force)
ds = LHCbDataset()
for f in downloaded_files:
ds.files.append(DiracFile(lfn=f))
return GPIProxyObjectFactory(ds)
def readInputData(self, optsfiles, extraopts=False):
'''Returns a LHCbDataSet object from a list of options files. The
optional argument extraopts will decide if the extraopts string inside
the application is considered or not.
Usage examples:
# Create an LHCbDataset object with the data found in the optionsfile
l=DaVinci(version='v22r0p2').readInputData([\"~/cmtuser/\" \
\"DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"])
# Get the data from an options file and assign it to the jobs inputdata
field
j.inputdata = j.application.readInputData([\"~/cmtuser/\" \
\"DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"])
# Assuming you have data in your extraopts, you can use the extraopts.
# In this case your extraopts need to be fully parseable by gaudirun.py
# So you must make sure that you have the proper import statements.
# e.g.
from Gaudi.Configuration import *
# If you mix optionsfiles and extraopts, as usual extraopts may
# overwright your options
#
# Use this to create a new job with data from extraopts of an old job
j=Job(inputdata=jobs[-1].application.readInputData([],True))
'''
def dummyfile():
temp_fd, temp_filename = tempfile.mkstemp(text=True, suffix='.py')
os.write(temp_fd, "Dummy file to keep the Optionsparser happy")
os.close(temp_fd)
return temp_filename
if type(optsfiles) != type([]):
optsfiles = [optsfiles]
# use a dummy file to keep the parser happy
if len(optsfiles) == 0:
optsfiles.append(dummyfile())
if extraopts:
extraopts = self.extraopts
else:
extraopts = ""
# parser = check_inputs(optsfiles, extraopts, self.env)
try:
parser = PythonOptionsParser(optsfiles, extraopts,
self.getenv(False))
except Exception as err:
msg = 'Unable to parse the job options. Please check options ' \
'files and extraopts.'
logger.error("PythonOptionsParserError:\n%s" % str(err))
raise ApplicationConfigurationError(msg)
return GPIProxyObjectFactory(parser.get_input_data())
def add(self, input):
if not isType(input, list):
input = [input]
for item in input:
if isType(item, str):
if os.path.isfile(expandfilename(item)):
logger.info('Copying file %s to shared directory %s'%(item, self.name))
shutil.copy2(expandfilename(item), os.path.join(shared_path,self.name))
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.increase(self.name)
shareref.decrease(self.name)
else:
logger.error('File %s not found' % expandfilename(item))
elif isType(item,File) and item.name is not '' and os.path.isfile(expandfilename(item.name)):
logger.info('Copying file object %s to shared directory %s'%(item.name,self.name))
shutil.copy2(expandfilename(item.name), os.path.join(shared_path,self.name))
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.increase(self.name)
shareref.decrease(self.name)
else:
logger.error('File %s not found' % expandfilename(item.name))
def upload(self, lfn, diracSE, guid=None):
'Upload PFN to LFC on SE "diracSE" w/ LFN "lfn".'
from LogicalFile import get_result
if guid is None:
cmd = 'result = DiracCommands.addFile("%s","%s","%s",None)' % \
(lfn,self.name,diracSE)
else:
cmd = 'result = DiracCommands.addFile("%s","%s","%s","%s")' % \
(lfn,self.name,diracSE,guid)
result = get_result(cmd, 'Problem w/ upload', 'Error uploading file.')
from LogicalFile import LogicalFile
return GPIProxyObjectFactory(LogicalFile(name=lfn))
def __deepcopy__(self, memo=None):
self = stripProxy(self)
self._getReadAccess()
c = super(GangaObject, self).__deepcopy__(memo)
if self._schema is not None:
for name, item in self._schema.allItems():
if not item['copyable']:
setattr(c, name, self._schema.getDefaultValue(name))
if item.isA(Schema.SharedItem):
shared_dir = getattr(c, name)
if hasattr(shared_dir, 'name'):
from Ganga.Core.GangaRepository import getRegistry
from Ganga.GPIDev.Base.Proxy import GPIProxyObjectFactory
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
logger.debug("Increasing shareref")
shareref.increase(shared_dir.name)
return c
def processWildcardMatches(self):
if self.subfiles:
return self.subfiles
if regex.search(self.namePattern):
ls_cmd = getConfig('Output')['MassStorageFile']['uploadOptions']['ls_cmd']
exitcode, output, m = self.shell.cmd1(ls_cmd + ' ' + self.inputremotedirectory, capture_stderr=True)
for filename in output.split('\n'):
if fnmatch(filename, self.namePattern):
subfile = MassStorageFile(namePattern=filename)
subfile.inputremotedirectory = self.inputremotedirectory
self.subfiles.append(GPIProxyObjectFactory(subfile))
def getDataset(self):
'''Gets the dataset from the bookkeeping for current path, etc.'''
files = []
if not self.path: return None
if not self.type in ['Path','RunsByDate','Run','Production']:
raise GangaException('Type="%s" is not valid.' % self.type)
lfnf = open('/afs/ihep.ac.cn/users/z/zhangxm/ganga/Ganga/install/5.6.2/python/GangaBoss/Lib/Dataset/lfns','r')
for line in lfnf.readlines():
newline = line.strip()
files.append(newline)
ds = BesDataset()
for f in files: ds.files.append(LogicalFile(f))
return GPIProxyObjectFactory(ds)
def getDataset(self):
'''Gets the dataset from the bookkeeping for current dict.'''
if not self.name: return None
badger = Badger()
files = []
files = badger.getFilesByDatasetName(self.name)
ds = BesDataset()
for f in files:
logicalFile = "LFN:"+f
logger.debug("zhangxm log: data files LFN: %s", f)
ds.files.append(logicalFile)
return GPIProxyObjectFactory(ds)
def processOutputWildcardMatches(self):
"""This collects the subfiles for wildcarded output LocalFile"""
import glob
fileName = self.namePattern
if self.compressed:
fileName = '%s.gz' % self.namePattern
sourceDir = self.getJobObject().outputdir
if regex.search(fileName) is not None:
for currentFile in glob.glob(os.path.join(sourceDir, fileName)):
d = LocalFile(namePattern=os.path.basename(currentFile))
d.compressed = self.compressed
self.subfiles.append(GPIProxyObjectFactory(d))
def processWildcardMatches(self):
if self.subfiles:
return self.subfiles
from fnmatch import fnmatch
if regex.search(self.namePattern):
#TODO namePattern shouldn't contain slashes and se_rpath should not contain wildcards
cmd = 'lcg-ls lfn:/grid/{vo}/{se_rpath}'.format(vo=self.credential_requirements.vo, se_rpath=self.se_rpath)
exitcode,output,m = getShell(self.credential_requirements).cmd1(cmd, capture_stderr=True)
for filename in output.split('\n'):
if fnmatch(filename, self.namePattern):
subfile = LCGSEFile(namePattern=filename)
subfile.se_rpath = self.se_rpath
subfile.lfc_host = self.lfc_host
self.subfiles.append(GPIProxyObjectFactory(subfile))
def lcgse_line_processor(line, lcgse_file):
guid = line[line.find('->') + 2:]
pattern = line.split(' ')[1]
name = line.split(' ')[2].strip('.gz')
if regex.search(lcgse_file.namePattern) is not None:
d = LCGSEFile(namePattern=name)
d.compressed = lcgse_file.compressed
d.lfc_host = lcgse_file.lfc_host
d.se = lcgse_file.se
# todo copy also the other attributes
lcgse_file.subfiles.append(GPIProxyObjectFactory(d))
lcgse_line_processor(line, d)
elif pattern == lcgse_file.namePattern:
if guid.startswith('ERROR'):
logger.error("Failed to upload file to LCG SE")
logger.error(guid[6:])
lcgse_file.failureReason = guid[6:]
return
lcgse_file.locations = guid
def processWildcardMatches(self):
if self.subfiles:
return self.subfiles
import glob
fileName = self.namePattern
if self.compressed:
fileName = '%s.gz' % self.namePattern
sourceDir = self.localDir
if regex.search(fileName) is not None:
for currentFile in glob.glob(os.path.join(sourceDir, fileName)):
d = LocalFile(namePattern=os.path.basename(currentFile),
localDir=os.path.dirname(currentFile))
d.compressed = self.compressed
self.subfiles.append(GPIProxyObjectFactory(d))
def processWildcardMatches(self):
if self.subfiles:
return self.subfiles
from fnmatch import fnmatch
if regex.search(self.namePattern):
# TODO namePattern shouldn't contain slashes and se_rpath should
# not contain wildcards
exitcode, output, m = self.shell.cmd1(
'lcg-ls lfn:/grid/' + getConfig('LCG')['VirtualOrganisation'] +
'/' + self.se_rpath,
capture_stderr=True)
for filename in output.split('\n'):
if fnmatch(filename, self.namePattern):
subfile = LCGSEFile(namePattern=filename)
subfile.se_rpath = self.se_rpath
subfile.lfc_host = self.lfc_host
self.subfiles.append(GPIProxyObjectFactory(subfile))
def getOutputData(self, dir=None, names=None):
"""Retrieve data stored on SE to dir (default=job output workspace).
If names=None, then all outputdata is downloaded otherwise names should
be a list of files to download."""
j = self.getJobObject()
if not names: names = ''
if not dir: dir = j.getOutputWorkspace().getPath()
global dirac_ganga_server
cmd = 'result = DiracCommands.getOutputData("%s","%s",%d)' \
% (names,dir,self.id)
result = dirac_ganga_server.execute(cmd)
downloaded_files = []
if not result_ok(result):
logger.error('Output download failed: %s' % str(result))
else:
downloaded_files = result.get('Value', [])
ds = BesDataset()
for f in downloaded_files:
ds.files.append(LogicalFile(f))
return GPIProxyObjectFactory(ds)
def listShareDirContents(self,shared_directory_name):
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.ls(shared_directory_name)
def put(self):
"""
Postprocesses (upload) output file to the desired destination from the client
"""
import hashlib
from apiclient.http import MediaFileUpload
service = self._setup_service()
# Sets the target directory
dir_path = self.localDir
if self.localDir == '':
dir_path = os.getcwd()
if self._getParent() is not None:
dir_path = self.getJobObject().getOutputWorkspace().getPath()
# Wildcard procedure
if regex.search(self.namePattern) is not None:
for wildfile in glob.glob(os.path.join(dir_path,
self.namePattern)):
FILENAME = wildfile
filename = os.path.basename(wildfile)
# Upload procedure
media_body = MediaFileUpload(FILENAME,
mimetype='text/plain',
resumable=True)
body = {
'title':
'%s' % filename,
'description':
'A test document',
'mimeType':
'text/plain',
'parents': [{
"kind": "drive#fileLink",
"id": "%s" % self.GangaFolderId
}]
}
# Metadata file and md5checksum intergrity check
file = service.files().insert(body=body,
media_body=media_body).execute()
with open(FILENAME, 'rb') as thefile:
if file.get('md5Checksum') == hashlib.md5(
thefile.read()).hexdigest():
logger.info("File \'%s\' uploaded successfully" %
filename)
else:
logger.error("File \'%s\' uploaded unsuccessfully" %
filename)
# Assign new schema components to each file and append to job
# subfiles
g = GoogleFile(filename)
g.downloadURL = file.get('downloadUrl', '')
g.id = file.get('id', '')
g.title = file.get('title', '')
self.subfiles.append(GPIProxyObjectFactory(g))
# For non-wildcard upload
else:
# Path to the file to upload
FILENAME = os.path.join(dir_path, self.namePattern)
# Upload procedure, can edit more of file metadata
media_body = MediaFileUpload(FILENAME,
mimetype='text/plain',
resumable=True)
body = {
'title':
'%s' % self.namePattern,
'description':
'A test document',
'mimeType':
'text/plain',
'parents': [{
"kind": "drive#fileLink",
"id": "%s" % self.GangaFolderId
}]
}
# Metadata storage and md5checksum integrity check
file = service.files().insert(body=body,
media_body=media_body).execute()
with open(FILENAME, 'rb') as thefile:
if file.get('md5Checksum') == hashlib.md5(
thefile.read()).hexdigest():
logger.info("File \'%s\' uploaded succesfully" %
self.namePattern)
else:
logger.error("Upload Unsuccessful")
# Assign values to new schema components
self.downloadURL = file.get('downloadUrl', '')
self.id = file.get('id', '')
self.title = file.get('title', '')
return
return GPIProxyObjectFactory(self.subfiles[:])
def decrementShareCounter(self, shared_directory_name, remove=0):
remove=remove
logger.debug('Decrementing shared directory reference counter')
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.decrease(shared_directory_name, remove)
def incrementShareCounter(self, shared_directory_name):
logger.debug('Incrementing shared directory reference counter')
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
logger.debug('within incrementShareCounter, calling increase')
shareref.increase(shared_directory_name)
'Exceptions')
exportToPublicInterface('ProtectedAttributeError', ProtectedAttributeError,
'Exceptions')
exportToPublicInterface('ReadOnlyObjectError', ReadOnlyObjectError,
'Exceptions')
exportToPublicInterface('JobError', JobError, 'Exceptions')
# ------------------------------------------------------------------------------------
# Import Monitoring Services
import Ganga.GPIDev.MonitoringServices
# ------------------------------------------------------------------------------------
# only the available credentials are exported
credential = getCredential(name='GridProxy', create=False)
if credential:
exportToPublicInterface('gridProxy', GPIProxyObjectFactory(credential),
'Objects', 'Grid proxy management object.')
credential = getCredential('AfsToken')
if credential:
exportToPublicInterface('afsToken', GPIProxyObjectFactory(credential),
'Objects', 'AFS token management object.')
# ------------------------------------------------------------------------------------
# Add Misc functions to public interface
exportToPublicInterface('license', ganga_license, 'Functions')
exportToPublicInterface('load', load, 'Functions')
exportToPublicInterface('export', export, 'Functions')
exportToPublicInterface('typename', typename, 'Functions')
exportToPublicInterface('categoryname', categoryname, 'Functions')
exportToPublicInterface('plugins', plugins, 'Functions')
def __getitem__(self, i):
'''Proivdes scripting (e.g. od[2] returns the 3rd file name) '''
if type(i) == type(slice(0)):
return GPIProxyObjectFactory(OutputData(files=self.files[i]))
else:
return self.files[i]
def put(self):
"""
Creates and executes commands for file upload to mass storage (Castor), this method will
be called on the client
"""
import glob
import re
sourceDir = ''
# if used as a stand alone object
if self._getParent() is None:
if self.localDir == '':
import os
_CWD = os.getcwd()
if os.path.isfile(os.path.join(_CWD, self.namePattern)):
sourceDir = _CWD
else:
logger.warning(
'localDir attribute is empty, don\'t know from which dir to take the file'
)
return
else:
sourceDir = self.localDir
(result, message) = self.validate()
if result == False:
logger.warning(message)
return
else:
job = self.getJobObject()
sourceDir = job.outputdir
# if there are subjobs, the put method will be called on every subjob
# and will upload the resulted output file
if len(job.subjobs) > 0:
return
massStorageConfig = getConfig(
'Output')['MassStorageFile']['uploadOptions']
mkdir_cmd = massStorageConfig['mkdir_cmd']
cp_cmd = massStorageConfig['cp_cmd']
ls_cmd = massStorageConfig['ls_cmd']
massStoragePath = massStorageConfig['path']
# create the last directory (if not exist) from the config path
import os.path
pathToDirName = os.path.dirname(massStoragePath)
dirName = os.path.basename(massStoragePath)
(exitcode, mystdout, mystderr) = self.execSyscmdSubprocess(
'%s %s' % (ls_cmd, pathToDirName))
if exitcode != 0:
self.handleUploadFailure(mystderr)
return
directoryExists = False
for directory in mystdout.split('\n'):
if directory.strip() == dirName:
directoryExists = True
break
if not directoryExists:
(exitcode, mystdout, mystderr) = self.execSyscmdSubprocess(
'%s %s' % (mkdir_cmd, massStoragePath))
if exitcode != 0:
self.handleUploadFailure(mystderr)
return
# the folder part of self.outputfilenameformat
folderStructure = ''
# the file name part of self.outputfilenameformat
filenameStructure = ''
if self._getParent() != None:
jobfqid = self.getJobObject().fqid
jobid = jobfqid
subjobid = ''
if (jobfqid.find('.') > -1):
jobid = jobfqid.split('.')[0]
subjobid = jobfqid.split('.')[1]
if self.outputfilenameformat is None:
filenameStructure = '{fname}'
# create jid/sjid directories
folderStructure = jobid
if subjobid != '':
folderStructure = os.path.join(jobid, subjobid)
else:
filenameStructure = os.path.basename(self.outputfilenameformat)
filenameStructure = filenameStructure.replace('{jid}', jobid)
folderStructure = os.path.dirname(self.outputfilenameformat)
folderStructure = folderStructure.replace('{jid}', jobid)
if subjobid != '':
filenameStructure = filenameStructure.replace(
'{sjid}', subjobid)
folderStructure = folderStructure.replace(
'{sjid}', subjobid)
else:
if self.outputfilenameformat != None:
folderStructure = os.path.dirname(self.outputfilenameformat)
filenameStructure = os.path.basename(self.outputfilenameformat)
else:
filenameStructure = '{fname}'
# create the folder structure
if folderStructure != '':
folderStructure = folderStructure.strip('/')
massStoragePath = os.path.join(massStoragePath, folderStructure)
command = '%s -p %s' % (mkdir_cmd, massStoragePath)
(exitcode, mystdout, mystderr) = self.execSyscmdSubprocess(command)
if exitcode != 0:
self.handleUploadFailure(mystderr)
return
# here filenameStructure has replaced jid and sjid if any, and only not
# replaced keyword is fname
fileName = self.namePattern
if self.compressed:
fileName = '%s.gz' % self.namePattern
if regex.search(fileName) is not None:
for currentFile in glob.glob(os.path.join(sourceDir, fileName)):
finalFilename = filenameStructure.replace(
'{fname}', os.path.basename(currentFile))
(exitcode, mystdout, mystderr) = self.execSyscmdSubprocess(
'%s %s %s' %
(cp_cmd, currentFile,
os.path.join(massStoragePath, finalFilename)))
d = MassStorageFile(namePattern=os.path.basename(currentFile))
d.compressed = self.compressed
if exitcode != 0:
self.handleUploadFailure(mystderr)
else:
logger.info(
'%s successfully uploaded to mass storage as %s' %
(currentFile,
os.path.join(massStoragePath, finalFilename)))
d.locations = os.path.join(massStoragePath,
os.path.basename(finalFilename))
# Alex removed this as more general approach in job.py after put() is called
# remove file from output dir if this object is attached to a job
# if self._getParent() != None:
# os.system('rm %s' % os.path.join(sourceDir, currentFile))
self.subfiles.append(GPIProxyObjectFactory(d))
else:
currentFile = os.path.join(sourceDir, fileName)
finalFilename = filenameStructure.replace(
'{fname}', os.path.basename(currentFile))
(exitcode, mystdout, mystderr) = self.execSyscmdSubprocess(
'%s %s %s' % (cp_cmd, currentFile,
os.path.join(massStoragePath, finalFilename)))
if exitcode != 0:
self.handleUploadFailure(mystderr)
else:
logger.info('%s successfully uploaded to mass storage as %s' %
(currentFile,
os.path.join(massStoragePath, finalFilename)))
location = os.path.join(massStoragePath,
os.path.basename(finalFilename))
if location not in self.locations:
self.locations.append(location)