def __initLogger( self, componentName, logSection ): gLogger.initialize( componentName, logSection ) if self.__debugMode == 1: gLogger.setLevel( "VERBOSE" ) elif self.__debugMode == 2: gLogger.setLevel( "VERBOSE" ) gLogger.showHeaders( True ) elif self.__debugMode >= 3: gLogger.setLevel( "DEBUG" ) gLogger.showHeaders( True )
def setUp( self ): """c'tor :param self: self reference """ gLogger.showHeaders( True ) self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.processPool = ProcessPool( 4, 8, 8, poolCallback = self.poolCallback, poolExceptionCallback = self.poolExceptionCallback ) self.processPool.daemonize()
def copy_file(self, filename, source_se, dest_se, dry_run=True): """ copies a file from on SE to another and registers it in the dirac file catalogue """ res = self.__rpcclient.getReplicas(filename, False) if not res["OK"]: print "Could not get replica status for %s" % filename return False ses = res['Value']['Successful'][filename].keys() if not source_se in ses: # print "File %s not at source SE" % filename return False if (source_se in ses) and (not dest_se in ses): print "%s" % filename if not dry_run: res = self.__dm.replicateAndRegister(filename, dest_se, source_se) if not res['OK']: print "Replicate and register failed for: %s" % filename print res sleep(5) print "Trying again to register %s" %filename gLogger.setLevel( "DEBUG" ) gLogger.showHeaders( True ) res = self.__dm.replicateAndRegister(filename, dest_se, source_se) gLogger.setLevel( "INFO" ) gLogger.showHeaders( False ) if not res['OK']: print "Replicate and register failed again for: %s" % filename self.__badfiles.add(filename) return False return True # file already exists on destination SE return False
from DIRAC.Core.Base import Script # Register workflow parameter switch Script.registerSwitch('p:', 'parameter=', 'Parameters that are passed directly to the workflow') Script.parseCommandLine() # from DIRAC.Core.Workflow.Parameter import * from DIRAC import gLogger from DIRAC.Core.Workflow.Workflow import fromXMLFile from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport from DIRAC.AccountingSystem.Client.DataStoreClient import DataStoreClient from DIRAC.RequestManagementSystem.Client.Request import Request # Forcing the current directory to be the first in the PYTHONPATH sys.path.insert(0, os.path.realpath('.')) gLogger.showHeaders(True) def jobexec(jobxml, wfParameters): jobfile = os.path.abspath(jobxml) if not os.path.exists(jobfile): gLogger.warn('Path to specified workflow %s does not exist' % (jobfile)) sys.exit(1) workflow = fromXMLFile(jobfile) gLogger.debug(workflow) code = workflow.createCode() gLogger.debug(code) jobID = 0 if 'JOBID' in os.environ: jobID = os.environ['JOBID'] gLogger.info('DIRAC JobID %s is running at site %s' % (jobID, DIRAC.siteName()))
def setUp( self ): gLogger.showHeaders( True ) self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.processPool = ProcessPool( 4, 8, 8 ) self.processPool.daemonize()
# Register workflow parameter switch Script.registerSwitch('p:', 'parameter=', 'Parameters that are passed directly to the workflow') Script.parseCommandLine() # from DIRAC.Core.Workflow.Parameter import * from DIRAC import gLogger from DIRAC.Core.Workflow.Workflow import fromXMLFile from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport from DIRAC.AccountingSystem.Client.DataStoreClient import DataStoreClient from DIRAC.RequestManagementSystem.Client.Request import Request # Forcing the current directory to be the first in the PYTHONPATH sys.path.insert(0, os.path.realpath('.')) gLogger.showHeaders(True) def jobexec(jobxml, wfParameters): jobfile = os.path.abspath(jobxml) if not os.path.exists(jobfile): gLogger.warn('Path to specified workflow %s does not exist' % (jobfile)) sys.exit(1) workflow = fromXMLFile(jobfile) gLogger.debug(workflow) code = workflow.createCode() gLogger.debug(code) jobID = 0 if os.environ.has_key('JOBID'): jobID = os.environ['JOBID']
in the dirac file catalogue """ from DIRAC.Core.Base import Script Script.initialize() import sys import os import getopt from time import sleep from DIRAC.Core.DISET.RPCClient import RPCClient from DIRAC.DataManagementSystem.Client.DataManager import DataManager from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC import gLogger gLogger.setLevel( "INFO" ) gLogger.showHeaders( False ) class RecursiveCp(object): def __init__(self): self.__rpcclient = RPCClient( "DataManagement/FileCatalog" ) self.__dm = DataManager() self.__n_files = 0 self.__space_copied = 0L self.__badfiles = set() def search_directory(self, directory_path, source_se, dest_se, dry_run=True): """ finds all files and subdirectories in a directory """ print "Searching directory: %s" % directory_path
def processPool(): gLogger.showHeaders(True) log = gLogger.getSubLogger("TaskCallbacksTests") processPool = ProcessPool(4, 8, 8) processPool.daemonize() yield processPool
def main(): # Register workflow parameter switch Script.registerSwitch( "p:", "parameter=", "Parameters that are passed directly to the workflow") # Registering arguments will automatically add their description to the help menu Script.registerArgument( "jobXMLfile: specify path to the Job XML file description") Script.parseCommandLine() # from DIRAC.Core.Workflow.Parameter import * from DIRAC import gLogger from DIRAC.Core.Workflow.Workflow import fromXMLFile from DIRAC.Core.Utilities.Proxy import executeWithoutServerCertificate from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport from DIRAC.AccountingSystem.Client.DataStoreClient import DataStoreClient from DIRAC.RequestManagementSystem.Client.Request import Request # Forcing the current directory to be the first in the PYTHONPATH sys.path.insert(0, os.path.realpath(".")) gLogger.showHeaders(True) @executeWithoutServerCertificate def jobexec(jobxml, wfParameters): jobfile = os.path.abspath(jobxml) if not os.path.exists(jobfile): gLogger.warn("Path to specified workflow %s does not exist" % (jobfile)) sys.exit(1) workflow = fromXMLFile(jobfile) gLogger.debug(workflow) code = workflow.createCode() gLogger.debug(code) jobID = 0 if "JOBID" in os.environ: jobID = os.environ["JOBID"] gLogger.info("DIRAC JobID %s is running at site %s" % (jobID, DIRAC.siteName())) workflow.addTool("JobReport", JobReport(jobID)) workflow.addTool("AccountingReport", DataStoreClient()) workflow.addTool("Request", Request()) # Propagate the command line parameters to the workflow if any for pName, pValue in wfParameters.items(): workflow.setValue(pName, pValue) # Propagate the command line parameters to the workflow module instances of each step for stepdefinition in workflow.step_definitions.values(): for moduleInstance in stepdefinition.module_instances: for pName, pValue in wfParameters.items(): if moduleInstance.parameters.find(pName): moduleInstance.parameters.setValue(pName, pValue) return workflow.execute() positionalArgs = Script.getPositionalArgs() if len(positionalArgs) != 1: gLogger.debug("Positional arguments were %s" % (positionalArgs)) DIRAC.abort(1, "Must specify the Job XML file description") if "JOBID" in os.environ: gLogger.info("JobID: %s" % (os.environ["JOBID"])) jobXMLfile = positionalArgs[0] parList = Script.getUnprocessedSwitches() parDict = {} for switch, parameter in parList: if switch == "p": name, value = parameter.split("=") value = value.strip() # The comma separated list in curly brackets is interpreted as a list if value.startswith("{"): value = value[1:-1].replace('"', "").replace(" ", "").split(",") value = ";".join(value) parDict[name] = value gLogger.debug("PYTHONPATH:\n%s" % ("\n".join(sys.path))) jobExec = jobexec(jobXMLfile, parDict) if not jobExec["OK"]: gLogger.debug("Workflow execution finished with errors, exiting") if jobExec["Errno"]: os._exit(jobExec["Errno"]) else: os._exit(1) else: gLogger.debug("Workflow execution successful, exiting") # dirac_jobexec might interact with ARC library which cannot be closed using a simple sys.exit(0) # See https://bugzilla.nordugrid.org/show_bug.cgi?id=4022 for further details os._exit(0)