def benchmark_pre (tid, test_cfg, bench_cfg, session) :
if not 'job_service_url' in test_cfg :
raise saga.NoSuccess ('no job service URL configured')
if not 'load' in bench_cfg :
raise saga.NoSuccess ('no benchmark load configured')
host = saga.Url(test_cfg['job_service_url']).host
n_j = int(bench_cfg['iterations'])
load = int(bench_cfg['load'])
exe = '/bin/sleep'
ssh = subprocess ("ssh %s" % host, stdin = subprocess.PIPE,
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT)
# find the ssh prompt
stdin = ssh.communicate[0]
stdout = ssh.communicate[1]
while <stdin> ~! /'>$'/io :
time.sleep (0.1)
# setup is done
return {'ssh' : ssh, 'cmd' : "%s %s" % (executable, load)}
def remove(self, path, flags):
'''This method is called upon logicaldir.remove() '''
complete_path = rs.Url(path).get_path()
try:
self._logger.debug("Executing: irm -r %s" % complete_path)
ret, out, _ = self.shell.run_sync("irm -r %s" % complete_path)
if ret != 0:
raise rs.NoSuccess("Could not remove directory %s [%s]: %s" %
(complete_path, str(ret), out))
except Exception as ex:
# was there no directory to delete?
if "does not exist" in str(ex):
raise rs.DoesNotExist("Directory %s does not exist." %
complete_path)
# couldn't delete for unspecificed reason
raise rs.NoSuccess("Couldn't delete directory %s because %s" %
(complete_path, ex))
return
def benchmark_pre(tid, app_cfg, bench_cfg):
if not 'saga.tests' in app_cfg:
raise saga.NoSuccess('no tests configured')
if not 'job_service_url' in app_cfg['saga.tests']:
raise saga.NoSuccess('no job service URL configured')
host = str(app_cfg['saga.tests']['job_service_url'])
app_cfg['host'] = host
def remove_self (self, flags) :
'''This method is called upon logicalfile.remove() '''
complete_path = self._url.get_path()
self._logger.debug("Attempting to remove file at: %s" % complete_path)
try:
ret, out, _ = self.shell.run_sync("irm %s" % complete_path)
if ret:
raise rs.NoSuccess("Could not remove file %s [%s]: %s"
% (complete_path, str(ret), out))
except Exception, ex:
# couldn't delete for unspecificed reason
raise rs.NoSuccess("delete %s failed: %s" % (complete_path, ex))
def list (self, npat, flags) :
# TODO: Make this use the irods_get_directory_listing
complete_path = self._url.path
result = []
self._logger.debug("Attempting to get directory listing for logical"
"path %s" % complete_path)
try:
ret, out, _ = self.shell.run_sync("ils %s" % complete_path)
if ret != 0:
raise Exception("Could not open directory")
# strip extra linebreaks from stdout, make a list w/ linebreaks,
# skip first entry which tells us the current directory
for item in out.strip().split("\n")[1:]:
item = item.strip()
if item.startswith("C- "):
# result.append("dir " + item[3:])
result.append(item[3:])
else:
# result.append("file " + item)
result.append(item)
except Exception, ex:
raise rs.NoSuccess ("Couldn't list directory: %s " % (str(ex)))
def download (self, name, source, flags) :
'''Downloads a file from the REMOTE REPLICA FILESYSTEM to a local
directory.
@param target: param containing a local path/filename
to save the file to
@param source: Optional param containing a remote replica to retrieve
the file from (not evaluated, yet)
'''
target = name
# TODO: Make sure that the target URL is a local/file:// URL
# extract the path from the LogicalFile object, excluding
# the filename
logical_path = self._url.get_path()
# fill in our local path if one was specified
local_path = ""
if target:
local_path = rs.Url(target).get_path()
try:
# var to hold our command result, placed here to keep in scope
ret = out = _ = 0
# mark that this is experimental/may not be part of official API
self._logger.debug("Beginning download operation "
"will download logical file: %s, "
"specified local target is %s"
% (logical_path, target))
if target: cmd = "iget %s %s" % (logical_path, local_path)
else : cmd = "iget %s" % logical_path
self._logger.debug("Executing: %s" % cmd)
ret, out, _ = self.shell.run_sync(cmd)
# check our result
if ret:
raise rs.NoSuccess("download failed: %s [%s]: %s"
% (logical_path, str(ret), out))
except Exception, ex:
# couldn't download for unspecificed reason
raise rs.NoSuccess ("Couldn't download file. %s" % ex)
def upload (self, source, target, flags) :
'''Uploads a file from the LOCAL, PHYSICAL filesystem to
the replica management system.
@param source: URL (should be file:// or local path) of local file
@param target: Optional param containing ?resource=myresource query
This will upload the file to a specified iRODS
resource or group.
'''
# TODO: Make sure that the source URL is a local/file:// URL
complete_path = rs.Url(source).get_path()
# extract the path from the LogicalFile object, excluding
# the filename
destination_path = self._url.get_path()[0:string.rfind(
self._url.get_path(), "/") + 1]
try:
# note we're uploading
self._logger.debug("Beginning upload operation "
"will register file in logical dir: %s"
% destination_path)
# the query holds our target resource
query = rs.Url(target).get_query()
# list of args we will generate
arg_list = ""
# parse flags + generate args
if flags:
if flags & rs.namespace.OVERWRITE:
arg_list += "-f "
# was no resource selected?
if not query:
self._logger.debug("Attempting to upload to default resource")
ret, out, _ = self.shell.run_sync("iput %s %s %s" %
(arg_list, complete_path, destination_path))
# resource was selected, have to parse it and supply to iput -R
else:
# TODO: Verify correctness
resource = query.split("=")[1]
self._logger.debug("upload to %s" % resource)
ret, out, _ = self.shell.run_sync("iput -R %s %s %s %s" %
(resource, arg_list, complete_path,
destination_path))
if ret:
raise rs.NoSuccess("Could not upload file %s, errorcode %s: %s"
% (complete_path, str(ret), out))
except Exception, ex:
# couldn't upload for unspecificed reason
raise rs.NoSuccess._log (self._logger, "upload failed: %s" % ex)
def make_dir (self, path, flags) :
# complete_path = dir_obj._url.path
complete_path = rs.Url(path).get_path()
# attempt to run iRODS mkdir command
try:
ret, out, _ = self.shell.run_sync("imkdir %s" % complete_path)
if ret != 0:
raise rs.NoSuccess("Could not create dir %s, [%s]: %s"
% (complete_path, str(ret), out))
except Exception, ex:
# did the directory already exist?
if "CATALOG_ALREADY_HAS_ITEM_BY_THAT_NAME" in str(ex):
raise rs.AlreadyExists ("Directory already exists.")
# couldn't create for unspecificed reason
raise rs.NoSuccess ("Couldn't create directory. %s" % ex)
def move_self (self, target, flags) :
'''This method is called upon logicaldir.move() '''
# path to file we are moving on iRODS
source_path = self._url.get_path()
dest_path = rs.Url(target).get_path()
self._logger.debug("Attempting to move logical file %s to location %s"
% (source_path, dest_path))
try:
ret, out, _ = self.shell.run_sync("imv %s %s"
% (source_path, dest_path))
if ret:
raise rs.NoSuccess("Could not move logical file %s to location"
"%s, errorcode %s: %s"
% (source_path, dest_path, str(ret), out))
except Exception, ex:
raise rs.NoSuccess ("Couldn't move file: %s" % ex)
def benchmark_pre(tid, app_cfg, bench_cfg):
if not 'saga.tests' in app_cfg:
raise saga.NoSuccess('no tests configured')
if not 'job_service_url' in app_cfg['saga.tests']:
raise saga.NoSuccess('no job service URL configured')
if not 'load' in bench_cfg:
raise saga.NoSuccess('no benchmark load configured')
host = str(app_cfg['saga.tests']['job_service_url'])
load = int(bench_cfg['load'])
js = saga.job.Service(host)
jd = saga.job.Description()
jd.executable = '/bin/sleep'
jd.arguments = [load]
app_cfg['js'] = js
app_cfg['jd'] = jd
def sanity_check (self) :
try:
# temporarily silence logger
lvl = self._logger.getEffectiveLevel ()
self._logger.setLevel (rs.utils.logger.ERROR)
# open a temporary shell
self.shell = rs.utils.pty_shell.PTYShell(
rs.url.Url("ssh://localhost"), logger=self._logger)
# run ils, see if we get any errors -- if so, fail the
# sanity check
rc, _, _ = self.shell.run_sync("ils")
if rc != 0:
raise Exception("sanity check error")
except Exception, ex:
raise rs.NoSuccess ("Could not run iRODS/ils. Check iRODS"
"environment and certificates (%s)" % ex)
def irods_get_resource_listing(self):
''' Return a list of irods resources and resource groups with
information stored in irods_resource_entry format.
It uses commandline tool ilsresc -l
:param self: reference to adaptor which called this function
'''
result = []
try:
# execute the ilsresc -l command
ret, out, _ = self.shell.run_sync("ilsresc -l")
# make sure we ran ok
if ret != 0:
raise Exception("Could not obtain resources with ilsresc -l")
# convert our command's stdout to a list of text lines
cw_result_list = out.strip().split("\n")
# list of resource entries we will save our results to
result = []
# while loop instead of for loop so we can mutate the list
# as we iterate
while cw_result_list:
entry = irods_resource_entry()
# get our next line from the FRONT of the list
line = cw_result_list.pop(0)
# check to see if this is the beginning of a
# singular resource entry
if line.startswith("resource name: "):
# singular resource entry output from ilsresc -l
# LINE NUMBERS AND PADDING ADDED
# ex. actual output, line 0 starts like "resource name"
# 0 resource name: BNL_ATLAS_2_FTP
# 1 resc id: 16214
# 2 zone: osg
# 3 type: MSS universal driver
# 4 class: compound
# 5 location: gw014k1.fnal.gov
# 6 vault: /data/cache/BNL_ATLAS_2_FTPplaceholder
# 7 free space:
# 8 status: up
# 9 info:
# 10 comment:
# 11 create time: 01343055975: 2012-07-23.09:06:15
# 12 modify time: 01347480717: 2012-09-12.14:11:57
# 13 ----
entry.name = line[len("resource name: "):].strip()
entry.is_resource_group = False
# TODO: SAVE ALL THE OTHER INFO
for i in range(13):
cw_result_list.pop(0)
# add our resource to the list
result.append(entry)
# check to see if this is an entry for a resource group
elif line.startswith("resource group: "):
entry.name = line[len("resource group: "):].strip()
entry.is_resource_group = True
# continue processing ilsresc -l results until we
# are at the end of the resource group information
# ----- is not printed if there are no further entries
# so we have to make sure to check we don't pop off
# an empty stack too
#
# TODO: SAVE THE LIST OF RESOURCES IN A RESOURCE GROUP
while cw_result_list and not line.startswith("-----"):
line = cw_result_list.pop(0)
result.append(entry)
# for some reason, we're at a line which we have
# no idea how to handle this is bad -- throw an error
else:
raise rs.NoSuccess("Error parsing iRODS ilsresc -l output")
return result
except Exception, e:
raise rs.NoSuccess ("Couldn't get resource listing: %s " % (str(e)))
def irods_get_directory_listing (self, irods_dir, wrapper) :
'''Function takes an iRODS logical directory as an argument,
and returns a list of irods_logical_entry instances containing
information on files/directories found in the directory argument.
It uses the commandline tool ils.
:param self: reference to adaptor which called this function
:param dir: iRODS directory we want to get a listing of
:param wrapper: the wrapper we will make our iRODS
commands with
'''
result = []
try:
cw = wrapper
# execute the ils -L command
ret, out, _ = cw.run_sync ("ils -L %s" % irods_dir)
# make sure we ran ok
if ret != 0:
raise rs.NoSuccess("Could not open directory %s [%s]: %s"
% (irods_dir, str(ret), out))
# strip extra linebreaks from stdout, make a list from the
# linebreaks that remain, skip first entry which just tells us the
# current directory
for item in out.strip().split("\n"):
# if we are listing a directory or remote resource file
# location i.e.
#
# [azebro1@gw68]$ ils -L /osg/home/azebro1
# /osg/home/azebro1:
# azebro1 1 UFlorida-SSERCA_FTP 12 2012-11-14.09:55 & irods-test.txt
# /data/cache/UFlorida-SSERCA_FTPplaceholder/home/azebro1/irods-test.txt osgGridFtpGroup
#
# then we want to ignore that entry (not using it for now)
if item.strip().startswith("/"):
continue
# remove whitespace
item = item.strip()
# entry for file or directory
dir_entry = irods_logical_entry()
# if we have a directory here
if item.startswith("C- "):
dir_entry.name = item[3:]
dir_entry.is_directory = True
# if we have a file here
else:
# ils -L output looks like this after you split it:
# 0 1 2 3 4 5 6
# ['azebro1', '1', 'UFlorida-SSERCA_FTP', '12', '2012-11-14.09:55', '&', 'irods-test.txt']
# not sure what 1 and 5 are ...
dir_entry.owner = item.split()[0]
dir_entry.locations = [item.split()[2]]
dir_entry.size = item.split()[3]
dir_entry.date = item.split()[4]
dir_entry.name = item.split()[6]
result.append(dir_entry)
# merge all entries on the list with duplicate filenames into a
# single entry with one filename and multiple resource locations
final_list = []
for item in result:
if item.name in [i.name for i in final_list]:
# duplicate name, merge this entry with the previous one
for final_list_item in final_list:
if final_list_item.name == item.name:
final_list_item.locations.append(item.locations[0])
else:
final_list.append(item)
return final_list
except Exception, e:
raise rs.NoSuccess ("Couldn't get directory listing: %s %s " % e)
