def lsJobDir(self, jobId):
"""List files at a specific URL.
@type jobId: string
@param jobId: jobId, which is URL location of job dir.
@rtype: list
@return: list of FileInfo
"""
# the jobID is a valid URL to the job directory. We can use it to
# inspect its contents.
#
# For other directories (gmlog or other), using FTPControl, we do
# not get accurate file sizes, only for the real output
# and for scripts/files in the proper job directory.
logger.debug('ls in JobDir for job %s' % jobId )
ftp = arclib.FTPControl()
url = arclib.URL(jobId)
self.__lockArclib()
try:
files = ftp.ListDir(url)
except arclib.ARCLibError, err:
logger.debug('Error during file listing: %s' % err.what())
errmsg = arclib.FileInfo()
errmsg.filename = err.what
errmsg.size = 0
errmsg.isDir = False
files = [ errmsg ]
def getStandardError(self, jobId):
"""Get the standard error of a running job.
@type jobID: string
@param jobID: jobId URL identifier.
@rtype: list
@return: list of return value from ARC and output from job."""
logger.debug('get stderr output for %s' % jobId)
try:
xrsl = self.recoverXrsl(jobId)
try:
outname = xrsl.GetRelation('stderr').GetSingleValue()
except arclib.XrslError as err:
outname = 'stderr' # try default if name not found
logger.debug('output file name: %s' % outname)
try:
self.__lockArclib()
ftp = arclib.FTPControl()
ftp.Download(arclib.URL(jobId + '/' + outname))
except Exception as err:
self.__unlockArclib()
raise ARCWrapperError(err.__str__())
self.__unlockArclib()
logger.debug('output downloaded')
result = open(outname, 'rb').read()
os.remove(outname)
except arclib.ARCLibError as err:
result = 'failed to retrieve job output stderr: %s' % err.what()
logger.error('%s' % result)
logger.debug('output retrieved')
return result
def recoverXrsl(self, jobId):
""" retrieves the xrsl for a job (from the server), if possible"""
logger.debug('Trying to obtain xRSL for job %s' % jobId)
xrsl = arclib.Xrsl('')
self.__lockArclib()
try:
(jobPath,jobBasename) = splitJobId(jobId)
xrslURL = arclib.URL(jobPath + '/info/'
+ jobBasename + '/description')
ftp = arclib.FTPControl()
ftp.Download(xrslURL, 'tmp')
str = file('tmp').read()
xrsl = arclib.Xrsl(str)
os.remove('tmp')
except arclib.ARCLibError, err:
logger.error('Failed to get Xrsl: %s' % err.what())
def download(job):
gridJobId = job.getCurrentGID()
if os.path.isdir('results'):
os.system('mv results results.%s' % time.time())
os.system('mkdir results')
try:
jctrl = arclib.FTPControl()
debug('DownloadDirectory(%s)' % gridJobId)
jctrl.DownloadDirectory(gridJobId, 'results')
debug('CleanJob(%s)' % gridJobId)
arclib.CleanJob(gridJobId)
debug('RemoveJobID(%s)' % gridJobId)
arclib.RemoveJobID(gridJobId)
except arclib.FTPControlError:
err = sys.exc_info()[0]
warn('Job %s downloading failed: %r ' % (job.id, err))
return False
return True
def getStandardError(self, jobId):
"""Get the standard error of a running job.
@type jobID: string
@param jobID: jobId URL identifier.
@rtype: list
@return: list of return value from ARC and output from job."""
logger.debug('get stderr output for %s' % jobId)
try:
xrsl = self.recoverXrsl(jobId)
try:
outname = xrsl.GetRelation('stderr').GetSingleValue()
except arclib.XrslError, err:
outname = 'stderr' # try default if name not found
logger.debug('output file name: %s' % outname)
try:
self.__lockArclib()
ftp = arclib.FTPControl()
ftp.Download(arclib.URL(jobId + '/' + outname))
except Exception, err:
self.__unlockArclib()
raise ARCWrapperError(err.__str__())
def getResults(self, jobId, downloadDir=''):
"""Download results from grid job.
@type jobId: string
@param jobID: jobId URL identifier.
@type downloadDir: string
@param downloadDir: Download results to specified directory.
@rtype: list
@return: list of downloaded files (strings)"""
logger.debug('Downloading files from job %s' % jobId)
complete = []
currDir = os.getcwd()
# jobID is a valid URL for the job directory.
# we chop off the final number (should be unique enough)
# and use it as a directory name to download (emulates behaviour
# of ngget: downloaddir _prefixes_ the dir to which we download).
try:
(jobPath, jobBasename) = splitJobId(jobId)
jobInfoDir = jobPath + '/info/' + jobBasename
jobDir = jobPath + '/' + jobBasename
os.chdir(self._userdir)
if not downloadDir == '':
if not os.path.exists(downloadDir):
os.mkdir(downloadDir)
elif not os.path.isdir(downloadDir):
raise ARCWrapperError(downloadDir +
' exists, not a directory.')
os.chdir(downloadDir)
if not os.path.exists(jobBasename):
os.mkdir(jobBasename)
else:
if not os.path.isdir(jobBasename):
raise ARCWrapperError('Cannot create job directory,' +
' existing file %s in the way.' %
jobBasename)
os.chdir(jobBasename)
except Exception as err:
logger.error('Error creating job directory: %s' % err)
os.chdir(currDir)
raise ARCWrapperError(err.__str__())
logger.debug('downloading output summary file')
self.__lockArclib()
try:
ftp = arclib.FTPControl()
# We could just download the whole directory.
# But better use the contents of "output" in
# the info-directory... (specified by user)
# to avoid downloading large input files.
# ftp.DownloadDirectory(jobURL, jobBasename)
#
# We use a temp file to get this information first
(tmp, tmpname) = tempfile.mkstemp(prefix='output', text=True)
os.close(tmp)
ftp.Download(arclib.URL(jobInfoDir + '/output'), tmpname)
lines = open(tmpname, 'rb').readlines()
os.remove(tmpname)
files = [l.strip().strip('/') for l in lines]
# also get the entire directory listing from the server
dir = ftp.ListDir(arclib.URL(jobDir), True)
basenames = [os.path.basename(x.filename) for x in dir]
if '' in files:
logger.debug('downloading _all_ files')
# TODO for files which are already there?
ftp.DownloadDirectory(arclib.URL(jobDir), '.')
complete = basenames
else:
for f in files:
if f in basenames:
# we should download this one
try:
if f.isdir:
logger.debug('DownloadDir %s' % f)
ftp.DownloadDirectory(
arclib.URL(jobDir + '/' + f), f)
# ... which operates recursively
complete.append(f + '/ (dir)')
else:
logger.debug('Download %s' % f)
ftp.Download(arclib.URL(jobDir + '/' + f), f)
complete.append(f)
except arclib.ARCLibError as err:
logger.error('Error downloading %s: %s' %
(f, err.what()))
except arclib.ARCLibError as err:
logger.error('ARCLib error while downloading: %s' % err.what())
self.__unlockArclib()
os.chdir(currDir)
raise ARCWrapperError(err.what())
except Exception as err:
logger.error('Error while downloading.\n %s' % err)
self.__unlockArclib()
os.chdir(currDir)
raise ARCWrapperError(err.__str__())
# return
logger.debug(' '.join(['downloaded:'] + complete))
os.chdir(currDir)
return complete
os.mkdir(jobBasename)
else:
if not os.path.isdir(jobBasename):
raise ARCWrapperError('Cannot create job directory,'
+' existing file %s in the way.'\
% jobBasename)
os.chdir(jobBasename)
except Exception, err:
logger.error('Error creating job directory: %s' % err)
os.chdir(currDir)
raise ARCWrapperError(err.__str__())
logger.debug('downloading output summary file')
self.__lockArclib()
try:
ftp = arclib.FTPControl()
# We could just download the whole directory.
# But better use the contents of "output" in
# the info-directory... (specified by user)
# to avoid downloading large input files.
# ftp.DownloadDirectory(jobURL, jobBasename)
#
# We use a temp file to get this information first
(tmp,tmpname) = tempfile.mkstemp(prefix='output', text=True)
os.close(tmp)
ftp.Download(arclib.URL(jobInfoDir + '/output'), tmpname)
lines = file(tmpname).readlines()
os.remove(tmpname)
files = [ l.strip().strip('/') for l in lines ]
