def read(self, lfn, local=False): """Reads a report from storage""" if local: cat = file(lfn).read() else: cat = castortools.cat(castortools.lfnToCastor(lfn)) return json.loads(cat)
def publish(self, report): """Publish a file""" for path in report['PathList']: _, name = tempfile.mkstemp('.txt', text=True) json.dump(report, file(name,'w'), sort_keys=True, indent=4) fname = '%s_%s.txt' % (self.parent, report['DateCreated']) #rename the file locally - TODO: This is a potential problem nname = os.path.join(os.path.dirname(name),fname) os.rename(name, nname) castor_path = castortools.lfnToCastor(path) new_name = '%s/%s' % (castor_path, fname) castortools.xrdcp(nname,path) time.sleep(1) if castortools.fileExists(new_name): #castortools.move(old_name, new_name) #castortools.chmod(new_name, '644') print "File published: '%s'" % castortools.castorToLFN(new_name) os.remove(nname) else: pathhash = path.replace('/','.') hashed_name = 'PublishToFileSystem-%s-%s' % (pathhash, fname) shutil.move(nname, hashed_name) print >> sys.stderr, "Cannot write to directory '%s' - written to local file '%s' instead." % (castor_path, hashed_name)
def publish(self, report): """Publish a file""" for path in report['PathList']: _, name = tempfile.mkstemp('.txt', text=True) json.dump(report, file(name, 'w'), sort_keys=True, indent=4) fname = '%s_%s.txt' % (self.parent, report['DateCreated']) #rename the file locally - TODO: This is a potential problem nname = os.path.join(os.path.dirname(name), fname) os.rename(name, nname) castor_path = castortools.lfnToCastor(path) new_name = '%s/%s' % (castor_path, fname) castortools.xrdcp(nname, path) time.sleep(1) if castortools.fileExists(new_name): #castortools.move(old_name, new_name) #castortools.chmod(new_name, '644') print "File published: '%s'" % castortools.castorToLFN( new_name) os.remove(nname) else: pathhash = path.replace('/', '.') hashed_name = 'PublishToFileSystem-%s-%s' % (pathhash, fname) shutil.move(nname, hashed_name) print >> sys.stderr, "Cannot write to directory '%s' - written to local file '%s' instead." % ( castor_path, hashed_name)
def read(self, lfn, local = False): """Reads a report from storage""" if local: cat = file(lfn).read() else: cat = castortools.cat(castortools.lfnToCastor(lfn)) return json.loads(cat)
def run(self, input): if self.user == 'CMS': return {'Topdir':None,'Directory':None} topdir = castortools.lfnToCastor(castorBaseDir(user=self.user)) directory = '%s/%s' % (topdir,self.dataset) # directory = directory.replace('//','/') if not castortools.fileExists(directory): if hasattr(self,'create') and self.create: castortools.createCastorDir(directory) #castortools.chmod(directory,'775') if not castortools.isDirectory(directory): raise Exception("Dataset directory '%s' does not exist or could not be created" % directory) return {'Topdir':topdir,'Directory':directory}
def run(self, input): if self.user == 'CMS': return {'Topdir':None,'Directory':None} topdir = castortools.lfnToCastor(castorBaseDir(user=self.user)) directory = '%s/%s' % (topdir,self.dataset) directory = directory.replace('//','/') if not castortools.fileExists(directory): if hasattr(self,'create') and self.create: castortools.createCastorDir(directory) #castortools.chmod(directory,'775') if not castortools.isDirectory(directory): raise Exception("Dataset directory '%s' does not exist or could not be created" % directory) return {'Topdir':topdir,'Directory':directory}
def __init__(self, dataset, options): if not dataset.startswith(os.sep): dataset = os.sep + dataset self.dataset = dataset self.options = options self.topdir = castortools.lfnToCastor(castorBaseDir(user=options.user)) self.directory = os.path.join(self.topdir, *self.dataset.split(os.sep)) #event counters self.eventsTotal = -1 self.eventsSeen = 0 self.test_result = None
def __init__(self, dataset, options): if not dataset.startswith(os.sep): dataset = os.sep + dataset self.dataset = dataset self.options = options self.topdir = castortools.lfnToCastor( castorBaseDir(user=options.user) ) self.directory = os.path.join(self.topdir, *self.dataset.split(os.sep)) #event counters self.eventsTotal = -1 self.eventsSeen = 0 self.test_result = None
def castorBaseDir( user=os.environ['USER'], area = None): """Gets the top level directory to use for writing for 'user'""" if area is None: user, area = getUserAndArea(user) d = '/store/cmst3/%s/%s/CMG' % (area,user) exists = castortools.isDirectory( castortools.lfnToCastor(d) ) if exists: return d else: msg = "The directory '%s' does not exist. Please check the username and area (user/group). You may need to create the directory yourself." % d print >> sys.stderr, msg raise NameError(msg)
def castorBaseDir(user=os.environ['USER'], area=None): """Gets the top level directory to use for writing for 'user'""" if area is None: user, area = getUserAndArea(user) d = 'root://eoscms.cern.ch//eos/cms/store/cmst3/%s/%s/CMG' % (area, user) exists = castortools.isDirectory(castortools.lfnToCastor(d)) if exists: return d else: msg = "The directory '%s' does not exist. Please check the username and area (user/group). You may need to create the directory yourself." % d print >> sys.stderr, msg raise NameError(msg)
default="empty_datasets.txt") parser.add_option_group(group) (options, args) = parser.parse_args() cmgdbApi = CmgdbApi(options.development) cmgdbApi.connect() columns, rows = cmgdbApi.sql( "select file_owner, path_name from dataset_details where dataset_is_open='Y' and (file_owner='cmgtools' or file_owner='cmgtools_group')" ) f = open(options.filename, 'w') for row in rows: fileown = row[0] dsName = row[1] if fileown == "--cmgtools": fileown = "cmgtools" lfnDir = castorBaseDir(fileown) + dsName castorDir = castortools.lfnToCastor(lfnDir) if castortools.datasetNotEmpty(castorDir, ".*root"): continue else: f.write(fileown + "%" + dsName + '\n') f.close()
default=None, help="The user space to write into") parser.add_option_group(group) (opts, datasets) = parser.parse_args() import ConfigParser config = ConfigParser.SafeConfigParser() #set options to be case sensitive config.optionxform = str config.add_section('MULTICRAB') config.set('MULTICRAB', 'cfg', opts.cfg) from CMGTools.Production.castorBaseDir import castorBaseDir import CMGTools.Production.eostools as castortools topdir = castortools.lfnToCastor(castorBaseDir(user=opts.user)) output_dirs = [] for d in datasets: #accept the user%dataset syntax, but ignore the user name for grid tokens = d.split('%') if len(tokens) == 2: d = tokens[1] safe_name = d.replace('/', '_') if safe_name.startswith('_'): safe_name = safe_name[1:] if safe_name.endswith('_'): safe_name = safe_name[:-1]
savpass = getpass.getpass("Enter Savannah Password: "******"" if options.commented: comment = raw_input("Please enter your comment here:") # Store full dataset name dirOrFile = args[0] # Castor base directory for specified user baseDir = castortools.lfnToCastor(castorBaseDir.castorBaseDir( options.user)) #get the user, after taking account of the user_area hack user, _ = castorBaseDir.getUserAndArea(options.user) # Locations of Logger(.tgz) and containing folder on Castor targetDir = baseDir + "/" + args[0].lstrip("/").rstrip("/") targetTgz = targetDir + "/Logger.tgz" # Get array containing 1) Primary dataset name; 2) Dataset Name; 3+) Tiers details = dirOrFile.lstrip("/").split("/") # If element is not Primary dataset of dataset name, add to the Tiers array count = 0 tiers = [] for i in details: if count > 1:
def __init__(self, name, user, pattern='.*root'): self.lfnDir = castorBaseDir(user) + name self.castorDir = castortools.lfnToCastor( self.lfnDir ) self.maskExists = False self.report = None super(Dataset, self).__init__(name, user, pattern)
castorDir = "" localDir = "" # opions.user could be of the form user_area user,area = cBaseDir.getUserAndArea(options.user) dataSets = '/afs/cern.ch/user/{first}/{user}/public/DataSets.txt'.format( first = user[0], # first letter of the username user = user ) ifile=open(dataSets,'r') pattern = re.compile( args[0] ) castorBaseDir = castortools.lfnToCastor(cBaseDir.castorBaseDir( options.user )) for line in ifile.readlines(): line = line.rstrip() if len(line)==0 or line[0]!='/': continue if pattern.search( line ): sampleName = line try: castorDir = castorBaseDir + sampleName except: sys.exit(1) localDir = None try: localDir = os.environ['CMGLOCALBASEDIR'] localDir += sampleName except:
if options.tier != "": sampleNameDir += "/" + options.tier # testing that the crab file exists try: oldCrab = open('crab.cfg','r') except Exception, e: print "Cannot find crab.cfg file in current directory. Error was '%s'." % str(e) sys.exit(1) # preparing castor dir ----------------- import castorBaseDir cdir = castortools.lfnToCastor( castorBaseDir.castorBaseDir(user=options.user) ) cdir += sampleNameDir if castortools.isCastorFile( cdir ) and not options.force: print 'The destination castor directory already exists:' print cdir print 'Please check. If everything is fine, run again with the -f option.' sys.exit(1) rfmkdir = 'rfmkdir -m 775 -p ' + cdir print rfmkdir castortools.createCastorDir(cdir) castortools.chmod(cdir, '775') # making local crab directory --------- ldir = '.' + sampleNameDir
def __init__(self, name, user, pattern='.*root'): self.lfnDir = castorBaseDir(user) + name self.castorDir = castortools.lfnToCastor(self.lfnDir) self.maskExists = False self.report = None super(Dataset, self).__init__(name, user, pattern)
#!/usr/bin/env python import CMGTools.Production.eostools as castortools from CMGTools.Production.castorBaseDir import castorBaseDir if __name__ == '__main__': import sys from optparse import OptionParser parser = OptionParser() parser.usage = "%prog <user>\nPrints the castor base directory of a given user." parser.add_option("-c", "--castorpath", action="store_true", dest="castorpath", help="Print full castor path. Otherwise print LFN, starting by /store", default=False) (options,args) = parser.parse_args() if len(args)!=1: parser.print_help() sys.exit(1) user = args[0] dir = castorBaseDir(user) if options.castorpath: dir = castortools.lfnToCastor( dir ) print dir
group.add_option("-t", "--tier", dest="tier", default='',help="The data tier to use, e.g. 'PAT_CMG_5_6_0'") group.add_option("-u", "--user", dest="user", default=None,help="The user space to write into") parser.add_option_group(group) (opts, datasets) = parser.parse_args() import ConfigParser config = ConfigParser.SafeConfigParser() #set options to be case sensitive config.optionxform = str config.add_section('MULTICRAB') config.set('MULTICRAB','cfg',opts.cfg) from CMGTools.Production.castorBaseDir import castorBaseDir import CMGTools.Production.eostools as castortools topdir = castortools.lfnToCastor(castorBaseDir(user=opts.user)) output_dirs = [] for d in datasets: #accept the user%dataset syntax, but ignore the user name for grid tokens = d.split('%') if len(tokens) == 2: d = tokens[1] safe_name = d.replace('/','_') if safe_name.startswith('_'): safe_name = safe_name[1:] if safe_name.endswith('_'): safe_name = safe_name[:-1]
) parser.add_option_group(group) (options, args) = parser.parse_args() cmgdbApi = CmgdbApi(options.development) cmgdbApi.connect() columns, rows = cmgdbApi.sql( "select file_owner, path_name from dataset_details where dataset_is_open='Y' and (file_owner='cmgtools' or file_owner='cmgtools_group')" ) f = open(options.filename, "w") for row in rows: fileown = row[0] dsName = row[1] if fileown == "--cmgtools": fileown = "cmgtools" lfnDir = castorBaseDir(fileown) + dsName castorDir = castortools.lfnToCastor(lfnDir) if castortools.datasetNotEmpty(castorDir, ".*root"): continue else: f.write(fileown + "%" + dsName + "\n") f.close()