def publish(self, report):
"""Publish a file"""
for path in report['PathList']:
_, name = tempfile.mkstemp('.txt', text=True)
json.dump(report, file(name,'w'), sort_keys=True, indent=4)
fname = '%s_%s.txt' % (self.parent, report['DateCreated'])
#rename the file locally - TODO: This is a potential problem
nname = os.path.join(os.path.dirname(name),fname)
os.rename(name, nname)
castor_path = castortools.lfnToCastor(path)
new_name = '%s/%s' % (castor_path, fname)
castortools.xrdcp(nname,path)
time.sleep(1)
if castortools.fileExists(new_name):
#castortools.move(old_name, new_name)
#castortools.chmod(new_name, '644')
print "File published: '%s'" % castortools.castorToLFN(new_name)
os.remove(nname)
else:
pathhash = path.replace('/','.')
hashed_name = 'PublishToFileSystem-%s-%s' % (pathhash, fname)
shutil.move(nname, hashed_name)
print >> sys.stderr, "Cannot write to directory '%s' - written to local file '%s' instead." % (castor_path, hashed_name)
def publish(self, report):
"""Publish a file"""
for path in report['PathList']:
_, name = tempfile.mkstemp('.txt', text=True)
json.dump(report, file(name, 'w'), sort_keys=True, indent=4)
fname = '%s_%s.txt' % (self.parent, report['DateCreated'])
#rename the file locally - TODO: This is a potential problem
nname = os.path.join(os.path.dirname(name), fname)
os.rename(name, nname)
castor_path = castortools.lfnToCastor(path)
new_name = '%s/%s' % (castor_path, fname)
castortools.xrdcp(nname, path)
time.sleep(1)
if castortools.fileExists(new_name):
#castortools.move(old_name, new_name)
#castortools.chmod(new_name, '644')
print "File published: '%s'" % castortools.castorToLFN(
new_name)
os.remove(nname)
else:
pathhash = path.replace('/', '.')
hashed_name = 'PublishToFileSystem-%s-%s' % (pathhash, fname)
shutil.move(nname, hashed_name)
print >> sys.stderr, "Cannot write to directory '%s' - written to local file '%s' instead." % (
castor_path, hashed_name)
def read(self, lfn, local = False):
"""Reads a report from storage"""
if local:
cat = file(lfn).read()
else:
cat = castortools.cat(castortools.lfnToCastor(lfn))
#print "the cat is: ", cat
return json.loads(cat)
def read(self, lfn, local=False):
"""Reads a report from storage"""
if local:
cat = file(lfn).read()
else:
cat = castortools.cat(castortools.lfnToCastor(lfn))
#print "the cat is: ", cat
return json.loads(cat)
def run(self, input):
if self.user == 'CMS':
return {'Topdir':None,'Directory':None}
topdir = castortools.lfnToCastor(castorBaseDir(user=self.user))
directory = '%s/%s' % (topdir,self.dataset)
# directory = directory.replace('//','/')
if not castortools.fileExists(directory):
if hasattr(self,'create') and self.create:
castortools.createCastorDir(directory)
#castortools.chmod(directory,'775')
if not castortools.isDirectory(directory):
raise Exception("Dataset directory '%s' does not exist or could not be created" % directory)
return {'Topdir':topdir,'Directory':directory}
def run(self, input):
if self.user == 'CMS':
return {'Topdir':None,'Directory':None}
topdir = castortools.lfnToCastor(castorBaseDir(user=self.user))
directory = '%s/%s' % (topdir,self.dataset)
# directory = directory.replace('//','/')
if not castortools.fileExists(directory):
if hasattr(self,'create') and self.create:
castortools.createCastorDir(directory)
#castortools.chmod(directory,'775')
if not castortools.isDirectory(directory):
raise Exception("Dataset directory '%s' does not exist or could not be created" % directory)
return {'Topdir':topdir,'Directory':directory}
def castorBaseDir( user=os.environ['USER'], area = None):
"""Gets the top level directory to use for writing for 'user'"""
if area is None:
user, area = getUserAndArea(user)
d = 'root://eoscms.cern.ch//eos/cms/store/cmst3/%s/%s/CMG' % (area,user)
exists = castortools.isDirectory( castortools.lfnToCastor(d) )
if exists:
return d
else:
msg = "The directory '%s' does not exist. Please check the username and area (user/group). You may need to create the directory yourself." % d
print >> sys.stderr, msg
raise NameError(msg)
def __init__(self, dataset, options):
if not dataset.startswith(os.sep):
dataset = os.sep + dataset
self.dataset = dataset
self.options = options
self.topdir = castortools.lfnToCastor( castorBaseDir(user=options.user) )
self.directory = os.path.join(self.topdir, *self.dataset.split(os.sep))
#event counters
self.eventsTotal = -1
self.eventsSeen = 0
self.test_result = None
def __init__(self, dataset, options):
if not dataset.startswith(os.sep):
dataset = os.sep + dataset
self.dataset = dataset
self.options = options
self.topdir = castortools.lfnToCastor(castorBaseDir(user=options.user))
self.directory = os.path.join(self.topdir, *self.dataset.split(os.sep))
#event counters
self.eventsTotal = -1
self.eventsSeen = 0
self.test_result = None
def __init__(self, name, user, pattern='.*root'):
self.lfnDir = castorBaseDir(user) + name
self.castorDir = castortools.lfnToCastor(self.lfnDir)
self.maskExists = False
self.report = None
super(Dataset, self).__init__(name, user, pattern)
def __init__(self, name, user, pattern='.*root'):
self.lfnDir = castorBaseDir(user) + name
self.castorDir = castortools.lfnToCastor( self.lfnDir )
self.maskExists = False
self.report = None
super(Dataset, self).__init__(name, user, pattern)