Example #1
0
    def publish(self, report):
        """Publish a file"""
        for path in report['PathList']:
            _, name = tempfile.mkstemp('.txt', text=True)
            json.dump(report, file(name,'w'), sort_keys=True, indent=4)
            
            fname = '%s_%s.txt' % (self.parent, report['DateCreated'])
            #rename the file locally - TODO: This is a potential problem
            nname = os.path.join(os.path.dirname(name),fname)
            os.rename(name, nname)
            
            castor_path = castortools.lfnToCastor(path)
            new_name = '%s/%s' % (castor_path, fname)
            castortools.xrdcp(nname,path)
            time.sleep(1)
            
            if castortools.fileExists(new_name):
                
                #castortools.move(old_name, new_name)
                #castortools.chmod(new_name, '644')

                print "File published: '%s'" % castortools.castorToLFN(new_name)
                os.remove(nname)
            else:
                pathhash = path.replace('/','.')
                hashed_name = 'PublishToFileSystem-%s-%s' % (pathhash, fname)
                shutil.move(nname, hashed_name)
                print >> sys.stderr, "Cannot write to directory '%s' - written to local file '%s' instead." % (castor_path, hashed_name)
Example #2
0
    def publish(self, report):
        """Publish a file"""
        for path in report['PathList']:
            _, name = tempfile.mkstemp('.txt', text=True)
            json.dump(report, file(name, 'w'), sort_keys=True, indent=4)

            fname = '%s_%s.txt' % (self.parent, report['DateCreated'])
            #rename the file locally - TODO: This is a potential problem
            nname = os.path.join(os.path.dirname(name), fname)
            os.rename(name, nname)

            castor_path = castortools.lfnToCastor(path)
            new_name = '%s/%s' % (castor_path, fname)
            castortools.xrdcp(nname, path)
            time.sleep(1)

            if castortools.fileExists(new_name):

                #castortools.move(old_name, new_name)
                #castortools.chmod(new_name, '644')

                print "File published: '%s'" % castortools.castorToLFN(
                    new_name)
                os.remove(nname)
            else:
                pathhash = path.replace('/', '.')
                hashed_name = 'PublishToFileSystem-%s-%s' % (pathhash, fname)
                shutil.move(nname, hashed_name)
                print >> sys.stderr, "Cannot write to directory '%s' - written to local file '%s' instead." % (
                    castor_path, hashed_name)
Example #3
0
 def read(self, lfn, local = False):
     """Reads a report from storage"""
     if local:
         cat = file(lfn).read()
     else:
         cat = castortools.cat(castortools.lfnToCastor(lfn))
     #print "the cat is: ", cat
     return json.loads(cat)
Example #4
0
 def read(self, lfn, local=False):
     """Reads a report from storage"""
     if local:
         cat = file(lfn).read()
     else:
         cat = castortools.cat(castortools.lfnToCastor(lfn))
     #print "the cat is: ", cat
     return json.loads(cat)
Example #5
0
 def run(self, input):
     if self.user == 'CMS':
         return {'Topdir':None,'Directory':None}
     topdir = castortools.lfnToCastor(castorBaseDir(user=self.user))
     directory = '%s/%s' % (topdir,self.dataset)
     # directory = directory.replace('//','/')
     if not castortools.fileExists(directory):
         if hasattr(self,'create') and self.create:
             castortools.createCastorDir(directory)
             #castortools.chmod(directory,'775')
     if not castortools.isDirectory(directory): 
         raise Exception("Dataset directory '%s' does not exist or could not be created" % directory)
     return {'Topdir':topdir,'Directory':directory}  
 def run(self, input):
     if self.user == 'CMS':
         return {'Topdir':None,'Directory':None}
     topdir = castortools.lfnToCastor(castorBaseDir(user=self.user))
     directory = '%s/%s' % (topdir,self.dataset)
     # directory = directory.replace('//','/')
     if not castortools.fileExists(directory):
         if hasattr(self,'create') and self.create:
             castortools.createCastorDir(directory)
             #castortools.chmod(directory,'775')
     if not castortools.isDirectory(directory): 
         raise Exception("Dataset directory '%s' does not exist or could not be created" % directory)
     return {'Topdir':topdir,'Directory':directory}  
Example #7
0
def castorBaseDir( user=os.environ['USER'], area = None):
    """Gets the top level directory to use for writing for 'user'"""
    
    if area is None:
        user, area = getUserAndArea(user)
    
    d = 'root://eoscms.cern.ch//eos/cms/store/cmst3/%s/%s/CMG' % (area,user)
    exists = castortools.isDirectory( castortools.lfnToCastor(d) )
    if exists:
        return d
    else:
        msg = "The directory '%s' does not exist. Please check the username and area (user/group). You may need to create the directory yourself." % d
        print >> sys.stderr, msg
        raise NameError(msg)
Example #8
0
    def __init__(self, dataset, options):
        if not dataset.startswith(os.sep):
            dataset = os.sep + dataset

        self.dataset = dataset
        self.options = options
        self.topdir = castortools.lfnToCastor( castorBaseDir(user=options.user) )
        self.directory = os.path.join(self.topdir, *self.dataset.split(os.sep))
        
        #event counters
        self.eventsTotal = -1
        self.eventsSeen = 0
        
        self.test_result = None
Example #9
0
    def __init__(self, dataset, options):
        if not dataset.startswith(os.sep):
            dataset = os.sep + dataset

        self.dataset = dataset
        self.options = options
        self.topdir = castortools.lfnToCastor(castorBaseDir(user=options.user))
        self.directory = os.path.join(self.topdir, *self.dataset.split(os.sep))

        #event counters
        self.eventsTotal = -1
        self.eventsSeen = 0

        self.test_result = None
Example #10
0
 def __init__(self, name, user, pattern='.*root'):
     self.lfnDir = castorBaseDir(user) + name
     self.castorDir = castortools.lfnToCastor(self.lfnDir)
     self.maskExists = False
     self.report = None
     super(Dataset, self).__init__(name, user, pattern)
Example #11
0
 def __init__(self, name, user, pattern='.*root'):
     self.lfnDir = castorBaseDir(user) + name
     self.castorDir = castortools.lfnToCastor( self.lfnDir )
     self.maskExists = False
     self.report = None
     super(Dataset, self).__init__(name, user, pattern)