def benchmark_pre (tid, test_cfg, bench_cfg, session) : if not 'job_service_url' in test_cfg : raise saga.NoSuccess ('no job service URL configured') if not 'load' in bench_cfg : raise saga.NoSuccess ('no benchmark load configured') host = saga.Url(test_cfg['job_service_url']).host n_j = int(bench_cfg['iterations']) load = int(bench_cfg['load']) exe = '/bin/sleep' ssh = subprocess ("ssh %s" % host, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.STDOUT) # find the ssh prompt stdin = ssh.communicate[0] stdout = ssh.communicate[1] while <stdin> ~! /'>$'/io : time.sleep (0.1) # setup is done return {'ssh' : ssh, 'cmd' : "%s %s" % (executable, load)}
def remove(self, path, flags): '''This method is called upon logicaldir.remove() ''' complete_path = rs.Url(path).get_path() try: self._logger.debug("Executing: irm -r %s" % complete_path) ret, out, _ = self.shell.run_sync("irm -r %s" % complete_path) if ret != 0: raise rs.NoSuccess("Could not remove directory %s [%s]: %s" % (complete_path, str(ret), out)) except Exception as ex: # was there no directory to delete? if "does not exist" in str(ex): raise rs.DoesNotExist("Directory %s does not exist." % complete_path) # couldn't delete for unspecificed reason raise rs.NoSuccess("Couldn't delete directory %s because %s" % (complete_path, ex)) return
def benchmark_pre(tid, app_cfg, bench_cfg): if not 'saga.tests' in app_cfg: raise saga.NoSuccess('no tests configured') if not 'job_service_url' in app_cfg['saga.tests']: raise saga.NoSuccess('no job service URL configured') host = str(app_cfg['saga.tests']['job_service_url']) app_cfg['host'] = host
def remove_self (self, flags) : '''This method is called upon logicalfile.remove() ''' complete_path = self._url.get_path() self._logger.debug("Attempting to remove file at: %s" % complete_path) try: ret, out, _ = self.shell.run_sync("irm %s" % complete_path) if ret: raise rs.NoSuccess("Could not remove file %s [%s]: %s" % (complete_path, str(ret), out)) except Exception, ex: # couldn't delete for unspecificed reason raise rs.NoSuccess("delete %s failed: %s" % (complete_path, ex))
def list (self, npat, flags) : # TODO: Make this use the irods_get_directory_listing complete_path = self._url.path result = [] self._logger.debug("Attempting to get directory listing for logical" "path %s" % complete_path) try: ret, out, _ = self.shell.run_sync("ils %s" % complete_path) if ret != 0: raise Exception("Could not open directory") # strip extra linebreaks from stdout, make a list w/ linebreaks, # skip first entry which tells us the current directory for item in out.strip().split("\n")[1:]: item = item.strip() if item.startswith("C- "): # result.append("dir " + item[3:]) result.append(item[3:]) else: # result.append("file " + item) result.append(item) except Exception, ex: raise rs.NoSuccess ("Couldn't list directory: %s " % (str(ex)))
def download (self, name, source, flags) : '''Downloads a file from the REMOTE REPLICA FILESYSTEM to a local directory. @param target: param containing a local path/filename to save the file to @param source: Optional param containing a remote replica to retrieve the file from (not evaluated, yet) ''' target = name # TODO: Make sure that the target URL is a local/file:// URL # extract the path from the LogicalFile object, excluding # the filename logical_path = self._url.get_path() # fill in our local path if one was specified local_path = "" if target: local_path = rs.Url(target).get_path() try: # var to hold our command result, placed here to keep in scope ret = out = _ = 0 # mark that this is experimental/may not be part of official API self._logger.debug("Beginning download operation " "will download logical file: %s, " "specified local target is %s" % (logical_path, target)) if target: cmd = "iget %s %s" % (logical_path, local_path) else : cmd = "iget %s" % logical_path self._logger.debug("Executing: %s" % cmd) ret, out, _ = self.shell.run_sync(cmd) # check our result if ret: raise rs.NoSuccess("download failed: %s [%s]: %s" % (logical_path, str(ret), out)) except Exception, ex: # couldn't download for unspecificed reason raise rs.NoSuccess ("Couldn't download file. %s" % ex)
def upload (self, source, target, flags) : '''Uploads a file from the LOCAL, PHYSICAL filesystem to the replica management system. @param source: URL (should be file:// or local path) of local file @param target: Optional param containing ?resource=myresource query This will upload the file to a specified iRODS resource or group. ''' # TODO: Make sure that the source URL is a local/file:// URL complete_path = rs.Url(source).get_path() # extract the path from the LogicalFile object, excluding # the filename destination_path = self._url.get_path()[0:string.rfind( self._url.get_path(), "/") + 1] try: # note we're uploading self._logger.debug("Beginning upload operation " "will register file in logical dir: %s" % destination_path) # the query holds our target resource query = rs.Url(target).get_query() # list of args we will generate arg_list = "" # parse flags + generate args if flags: if flags & rs.namespace.OVERWRITE: arg_list += "-f " # was no resource selected? if not query: self._logger.debug("Attempting to upload to default resource") ret, out, _ = self.shell.run_sync("iput %s %s %s" % (arg_list, complete_path, destination_path)) # resource was selected, have to parse it and supply to iput -R else: # TODO: Verify correctness resource = query.split("=")[1] self._logger.debug("upload to %s" % resource) ret, out, _ = self.shell.run_sync("iput -R %s %s %s %s" % (resource, arg_list, complete_path, destination_path)) if ret: raise rs.NoSuccess("Could not upload file %s, errorcode %s: %s" % (complete_path, str(ret), out)) except Exception, ex: # couldn't upload for unspecificed reason raise rs.NoSuccess._log (self._logger, "upload failed: %s" % ex)
def make_dir (self, path, flags) : # complete_path = dir_obj._url.path complete_path = rs.Url(path).get_path() # attempt to run iRODS mkdir command try: ret, out, _ = self.shell.run_sync("imkdir %s" % complete_path) if ret != 0: raise rs.NoSuccess("Could not create dir %s, [%s]: %s" % (complete_path, str(ret), out)) except Exception, ex: # did the directory already exist? if "CATALOG_ALREADY_HAS_ITEM_BY_THAT_NAME" in str(ex): raise rs.AlreadyExists ("Directory already exists.") # couldn't create for unspecificed reason raise rs.NoSuccess ("Couldn't create directory. %s" % ex)
def move_self (self, target, flags) : '''This method is called upon logicaldir.move() ''' # path to file we are moving on iRODS source_path = self._url.get_path() dest_path = rs.Url(target).get_path() self._logger.debug("Attempting to move logical file %s to location %s" % (source_path, dest_path)) try: ret, out, _ = self.shell.run_sync("imv %s %s" % (source_path, dest_path)) if ret: raise rs.NoSuccess("Could not move logical file %s to location" "%s, errorcode %s: %s" % (source_path, dest_path, str(ret), out)) except Exception, ex: raise rs.NoSuccess ("Couldn't move file: %s" % ex)
def benchmark_pre(tid, app_cfg, bench_cfg): if not 'saga.tests' in app_cfg: raise saga.NoSuccess('no tests configured') if not 'job_service_url' in app_cfg['saga.tests']: raise saga.NoSuccess('no job service URL configured') if not 'load' in bench_cfg: raise saga.NoSuccess('no benchmark load configured') host = str(app_cfg['saga.tests']['job_service_url']) load = int(bench_cfg['load']) js = saga.job.Service(host) jd = saga.job.Description() jd.executable = '/bin/sleep' jd.arguments = [load] app_cfg['js'] = js app_cfg['jd'] = jd
def sanity_check (self) : try: # temporarily silence logger lvl = self._logger.getEffectiveLevel () self._logger.setLevel (rs.utils.logger.ERROR) # open a temporary shell self.shell = rs.utils.pty_shell.PTYShell( rs.url.Url("ssh://localhost"), logger=self._logger) # run ils, see if we get any errors -- if so, fail the # sanity check rc, _, _ = self.shell.run_sync("ils") if rc != 0: raise Exception("sanity check error") except Exception, ex: raise rs.NoSuccess ("Could not run iRODS/ils. Check iRODS" "environment and certificates (%s)" % ex)
def irods_get_resource_listing(self): ''' Return a list of irods resources and resource groups with information stored in irods_resource_entry format. It uses commandline tool ilsresc -l :param self: reference to adaptor which called this function ''' result = [] try: # execute the ilsresc -l command ret, out, _ = self.shell.run_sync("ilsresc -l") # make sure we ran ok if ret != 0: raise Exception("Could not obtain resources with ilsresc -l") # convert our command's stdout to a list of text lines cw_result_list = out.strip().split("\n") # list of resource entries we will save our results to result = [] # while loop instead of for loop so we can mutate the list # as we iterate while cw_result_list: entry = irods_resource_entry() # get our next line from the FRONT of the list line = cw_result_list.pop(0) # check to see if this is the beginning of a # singular resource entry if line.startswith("resource name: "): # singular resource entry output from ilsresc -l # LINE NUMBERS AND PADDING ADDED # ex. actual output, line 0 starts like "resource name" # 0 resource name: BNL_ATLAS_2_FTP # 1 resc id: 16214 # 2 zone: osg # 3 type: MSS universal driver # 4 class: compound # 5 location: gw014k1.fnal.gov # 6 vault: /data/cache/BNL_ATLAS_2_FTPplaceholder # 7 free space: # 8 status: up # 9 info: # 10 comment: # 11 create time: 01343055975: 2012-07-23.09:06:15 # 12 modify time: 01347480717: 2012-09-12.14:11:57 # 13 ---- entry.name = line[len("resource name: "):].strip() entry.is_resource_group = False # TODO: SAVE ALL THE OTHER INFO for i in range(13): cw_result_list.pop(0) # add our resource to the list result.append(entry) # check to see if this is an entry for a resource group elif line.startswith("resource group: "): entry.name = line[len("resource group: "):].strip() entry.is_resource_group = True # continue processing ilsresc -l results until we # are at the end of the resource group information # ----- is not printed if there are no further entries # so we have to make sure to check we don't pop off # an empty stack too # # TODO: SAVE THE LIST OF RESOURCES IN A RESOURCE GROUP while cw_result_list and not line.startswith("-----"): line = cw_result_list.pop(0) result.append(entry) # for some reason, we're at a line which we have # no idea how to handle this is bad -- throw an error else: raise rs.NoSuccess("Error parsing iRODS ilsresc -l output") return result except Exception, e: raise rs.NoSuccess ("Couldn't get resource listing: %s " % (str(e)))
def irods_get_directory_listing (self, irods_dir, wrapper) : '''Function takes an iRODS logical directory as an argument, and returns a list of irods_logical_entry instances containing information on files/directories found in the directory argument. It uses the commandline tool ils. :param self: reference to adaptor which called this function :param dir: iRODS directory we want to get a listing of :param wrapper: the wrapper we will make our iRODS commands with ''' result = [] try: cw = wrapper # execute the ils -L command ret, out, _ = cw.run_sync ("ils -L %s" % irods_dir) # make sure we ran ok if ret != 0: raise rs.NoSuccess("Could not open directory %s [%s]: %s" % (irods_dir, str(ret), out)) # strip extra linebreaks from stdout, make a list from the # linebreaks that remain, skip first entry which just tells us the # current directory for item in out.strip().split("\n"): # if we are listing a directory or remote resource file # location i.e. # # [azebro1@gw68]$ ils -L /osg/home/azebro1 # /osg/home/azebro1: # azebro1 1 UFlorida-SSERCA_FTP 12 2012-11-14.09:55 & irods-test.txt # /data/cache/UFlorida-SSERCA_FTPplaceholder/home/azebro1/irods-test.txt osgGridFtpGroup # # then we want to ignore that entry (not using it for now) if item.strip().startswith("/"): continue # remove whitespace item = item.strip() # entry for file or directory dir_entry = irods_logical_entry() # if we have a directory here if item.startswith("C- "): dir_entry.name = item[3:] dir_entry.is_directory = True # if we have a file here else: # ils -L output looks like this after you split it: # 0 1 2 3 4 5 6 # ['azebro1', '1', 'UFlorida-SSERCA_FTP', '12', '2012-11-14.09:55', '&', 'irods-test.txt'] # not sure what 1 and 5 are ... dir_entry.owner = item.split()[0] dir_entry.locations = [item.split()[2]] dir_entry.size = item.split()[3] dir_entry.date = item.split()[4] dir_entry.name = item.split()[6] result.append(dir_entry) # merge all entries on the list with duplicate filenames into a # single entry with one filename and multiple resource locations final_list = [] for item in result: if item.name in [i.name for i in final_list]: # duplicate name, merge this entry with the previous one for final_list_item in final_list: if final_list_item.name == item.name: final_list_item.locations.append(item.locations[0]) else: final_list.append(item) return final_list except Exception, e: raise rs.NoSuccess ("Couldn't get directory listing: %s %s " % e)