def execute(self, command, verbose=False, background=False): ## make sure the MOA_THREADS env var is set - this is used from inside ## the Makefiles later threads need to be treated different from the ## other parameters. multi-threaded operation is only allowed in the ## second phase of execution. self.job.env['MOA_THREADS'] = "%s" % self.job.options.threads #if moa is not verbose, make should be silent if not self.job.options.verbose: self.makeArgs.append('-s') background = self.job.options.background l.debug("Calling make for command %s" % command) actor = self.job.getActor() #dump the job env into the actor environment (sys env) actor.setEnv(self.job.env) #and the job configuration actor.setEnv(self.job.conf) cl = ['make', command] + self.makeArgs l.debug("executing %s" % " ".join(cl)) actor.run(cl, background = background)
def __init__(self, templateName = None): """ Initialze the template object, which means: * Check if the template exists, if not raise an Exception * Load template info """ super(Template, self).__init__() self.name = 'nojob' self.backend = 'nojob' self.valid = False self.parameters = {} if templateName: self.meta.name = templateName self.meta.templateDataFile = os.path.join(TEMPLATEDIR, '%s.moa' % templateName) if not os.path.exists(self.meta.templateDataFile): raise InvalidTemplate() self.meta.valid = True self.load(self.meta.templateDataFile) l.debug("set template to %s, backend %s" % (self.name, self.backend))
def __init__(self, templateName=None): """ Initialze the template object, which means: * Check if the template exists, if not raise an Exception * Load template info """ super(Template, self).__init__() self.name = "nojob" self.backend = "nojob" self.valid = False self.parameters = {} if templateName: self.meta.name = templateName self.meta.templateDataFile = os.path.join(TEMPLATEDIR, "%s.moa" % templateName) if not os.path.exists(self.meta.templateDataFile): raise InvalidTemplate() self.meta.valid = True self.load(self.meta.templateDataFile) l.debug("set template to %s, backend %s" % (self.name, self.backend))
def execute(self, command, verbose=False, background=False): ## make sure the MOA_THREADS env var is set - this is used from inside ## the Makefiles later threads need to be treated different from the ## other parameters. multi-threaded operation is only allowed in the ## second phase of execution. self.job.env['MOA_THREADS'] = "%s" % self.job.options.threads #if moa is not verbose, make should be silent if not self.job.options.verbose: self.makeArgs.append('-s') background = self.job.options.background l.debug("Calling make for command %s" % command) actor = self.job.getActor() #dump the job env into the actor environment (sys env) actor.setEnv(self.job.env) #and the job configuration actor.setEnv(self.job.conf) cl = ['make', command] + self.makeArgs l.debug("executing %s" % " ".join(cl)) actor.run(cl, background=background)
def postNew(data): """ To be executed just after the 'moa new' command """ wd = data['cwd'] global GITROOT global GITREPO l.debug("running git post new hook") l.debug('GITROOT %s' % GITROOT) template = 'traverse' for a in data['newargs']: if '=' in a: continue template = a if template == 'project': # creating new project - see if we need to create a # new repository return _gitInit(data) #try to commit this project if not GITROOT: #no git repository - not registering any files return info = moa.info.info(wd) index = GITREPO.index index.add(info['moa_files'].split()) index.commit('Inital commit of job "%s"' % info['title'])
def load(self): """ Load configuration from disk """ for f in self.meta.configFiles.keys(): fileName = self.meta.configFiles[f] l.debug("Considering config file %s / %s" % (f, fileName)) if os.path.exists(fileName): super(Config, self).load(fileName, set_name=f)
def getOutput(self): """ Get the output from a moa run """ if not os.path.exists(self.captureOutName): return "" l.debug("reading output from %s" % self.captureOutName) return open(self.captureOutName).read()
def setTemplate(self, name): """ Set a new template for this job """ self.checkConfDir() self.template = moa.template.Template(name) with open(os.path.join(self.confDir, 'template'), 'w') as F: F.write(name) l.debug('set job in %s to template %s' % (self.wd, name))
def execute(self, command): """ """ l.debug("executing %s" % command) if self.backend: self.backend.execute(command, verbose=self.options.verbose, background=self.options.background) else: l.error("No backend loaded - cannot execute %s" % command)
def execute(self, command): """ """ l.debug("executing %s" % command) if self.backend: self.backend.execute(command, verbose = self.options.verbose, background = self.options.background) else: l.error("No backend loaded - cannot execute %s" % command)
def getError(self): """ Get the stderr of a moa run @returns: stderr output of this job (if captured) @rtype: string """ if not os.path.exists(self.captureErrName): return "" l.debug("reading error from %s" % self.captureErrName) return open(self.captureErrName).read()
def __init__(self, job): self.job = job ## Determine what plugins are loaded self.plugins = {} self.data = {} self.allPlugins = self.job.conf.getPlugins() l.debug("Plugins %s" % ", ".join(self.allPlugins)) ## load the plugins as seperate modules. A plugin does not need to self.initialize()
def configUnset(data): """ remove variables from the configuration """ job = data['job'] for a in data['newargs']: if '=' in a: l.error("Invalid argument to unset %s" % a) else: l.debug("Unsetting %s" % a) job.conf.unset(a) job.conf.save()
def configUnset(data): """ remove variables from the configuration """ job = data["job"] for a in data["newargs"]: if "=" in a: l.error("Invalid argument to unset %s" % a) else: l.debug("Unsetting %s" % a) job.conf.unset(a) job.conf.save()
def run2(self, cl): """ Really run """ self.runStartTime = time.time() l.debug("Starting actor now in %s" % self.wd) l.debug(" - with command line: '%s'" % " ".join(cl)) if self.captureOut: self.FOUT = open(self.captureOutName, 'w') if self.captureErr: self.FERR = open(self.captureErrName, 'w') self.p = subprocess.Popen(cl, shell=False, cwd=self.wd, stdout=self.FOUT, stderr=self.FERR) l.debug("execution has started with pid %d" % self.p.pid) self.pid = self.p.pid self.out, self.err = self.p.communicate() self.rc = self.p.returncode self.runStopTime = time.time() if self.captureOut: self.FOUT.close() if self.captureErr: self.FERR.close() l.debug("Finished execution with rc %d " % self.rc) return self.rc
def run2(self, cl): """ Really run """ self.runStartTime = time.time() l.debug("Starting actor now in %s" % self.wd) l.debug(" - with command line: '%s'" % " ".join(cl)) if self.captureOut: self.FOUT = open(self.captureOutName, 'w') if self.captureErr: self.FERR = open(self.captureErrName, 'w') self.p = subprocess.Popen( cl, shell=False, cwd = self.wd, stdout = self.FOUT, stderr = self.FERR) l.debug("execution has started with pid %d" % self.p.pid) self.pid = self.p.pid self.out, self.err = self.p.communicate() self.rc = self.p.returncode self.runStopTime = time.time() if self.captureOut: self.FOUT.close() if self.captureErr: self.FERR.close() l.debug("Finished execution with rc %d " % self.rc) return self.rc
def finish(self): if self.rc == 0: l.debug("Succesfully finished make in %s" % (self.wd)) else: if self.verbose: l.error("Error running make in %s. Return code %s" % (self.wd, self.rc)) else: l.debug("Error running make in %s. Return code %s" % (self.wd, self.rc)) if self.captureName: if self.captureOut: os.unlink(self.captureOutName) if self.captureErr: os.unlink(self.captureErrName)
def postSet(data): """ Execute just after setting a parameter """ wd = data['cwd'] global GITROOT global GITREPO #try to commit this project if not GITROOT: #no git repository - not registering any files return l.debug("committing postset") index = GITREPO.index index.commit('Set %s ' % " ".join(data['newargs']))
def finish(self): if self.rc == 0: l.debug("Succesfully finished make in %s" % (self.wd)) else: if self.verbose: l.error("Error running make in %s. Return code %s" % ( self.wd, self.rc)) else: l.debug("Error running make in %s. Return code %s" % ( self.wd, self.rc)) if self.captureName: if self.captureOut: os.unlink(self.captureOutName) if self.captureErr: os.unlink(self.captureErrName)
def __init__(self, wd=None): """ Constructor """ l.debug("Initializing a GNU Make job") if not wd: self.wd = os.getcwd() else: self.wd = wd #This is most probably a makefile job self.isMoa = False self.template = None self.makeArgs = [] self.refresh()
def prepare(data): #check if we're inside a git repository if not data.has_key('projectRoot'): return wd = data['cwd'] projectRoot = data['projectRoot'] l.debug("seeing if %s is a git repos" % projectRoot) repo = git.Repo(projectRoot) global GITROOT global GITREPO GITROOT = projectRoot GITREPO = repo l.debug("found a git root at %s " % GITROOT)
def loadBackend(self): """ load the backend """ backendName = self.template.backend.value l.debug("attempt to load backend %s" % backendName) try: _moduleName = 'moa.backend.%s' % backendName _module = __import__( _moduleName, globals(), locals(), [_moduleName], -1) l.debug("Successfully Loaded module %s" % _moduleName) except ImportError, e: if str(e) == "No module named %s" % _moduleName: l.critical("Backend %s does not exists" % backendName) l.critical("Error loading backend %s" % backendName) raise sys.exit(-1)
def loadBackend(self): """ load the backend """ backendName = self.template.backend.value l.debug("attempt to load backend %s" % backendName) try: _moduleName = 'moa.backend.%s' % backendName _module = __import__(_moduleName, globals(), locals(), [_moduleName], -1) l.debug("Successfully Loaded module %s" % _moduleName) except ImportError, e: if str(e) == "No module named %s" % _moduleName: l.critical("Backend %s does not exists" % backendName) l.critical("Error loading backend %s" % backendName) raise sys.exit(-1)
def findProjectRoot(job): """ Find the project root of a certain moa job >>> wd = moa.job.newTestJob(template='project', ... title = 'test job creation') >>> subdir = os.path.join(wd, 'test') >>> os.mkdir(subdir) >>> job = moa.job.newJob(subdir, template='traverse', title='test') >>> result = findProjectRoot(subdir) >>> result == wd True >>> None == findProjectRoot('/usr') True @param path: The path for which we're looking for the project root, if omitted, use the cwd @type path: string @returns: a tuple, first value indicates if there is a parent project the second is the path to the project, the third value is the project title @rtype: tuple of a boolean and two strings """ l.debug("Finding project root for job in %s" % job.wd) # start walking up through the tree until we discover #start walking through the parent tree lookAt = job while True: res = _projectRoot(lookAt) if res == 'project': return lookAt newPath = os.path.split(lookAt.wd)[0] if newPath == '/': return None lookAt = moa.job.getJob(newPath) # assuming that we're not creating moa jobs in the system root # if you do want to, I'm not going to cooperate. if cwd == '/': return None
def run(self, cl, background=False): """ Start a run """ if background: # try to fork child = os.fork() if child != 0: # This is the parent thread - return return child l.debug("Child thread - start executing - start run %s" % child) #no questions asked - we're saving the output self.captureOut = True self.captureErr = True return self.run2(cl)
def run(self, cl, background = False): """ Start a run """ if background: # try to fork child = os.fork() if child != 0: # This is the parent thread - return return child l.debug("Child thread - start executing - start run %s" % child) #no questions asked - we're saving the output self.captureOut = True self.captureErr = True return self.run2(cl)
def isMoa(self): """ Is the job directory a Moa directory currently we assume if there is a moa.mk file >>> job = Job('/') >>> job.isMoa() False >>> import moa.job >>> jobdir = moa.job.newTestJob('traverse') >>> job = Job(jobdir) >>> job.isMoa() True """ makefile = os.path.join(self.wd, 'Makefile') if not os.access(makefile, os.R_OK): #ok, this might be a moa directory, but #you do not have sufficient permissions return False l.debug('isMoaDir: checking %s' % makefile) if not os.path.exists(makefile): return False #we could run make, but that is rather slow just to check if the #Makefile is a proper Moa Makefile - so, we' quickly read the #Makefile to get a quick indication isMoa = False F = open(makefile) for line in F.readlines(): if 'MOABASE' in line: isMoa = True break if '$(call moa_load' in line: isMoa = True break F.close() return isMoa
def __init__(self, wd, options, args): self.wd = wd self.options = options self.args = args self.job = None ### determine what the command is self.command = "" if len(args) > 0: self.command = args[0] self.args = args[1:] l.info('Starting moa command "%s"' % self.command) ### Determine project root (if there is one) projectRoot = moa.project.findProjectRoot(self.wd) if projectRoot: l.debug('Project root is %s' % projectRoot) os.putenv('MOAPROJECTROOT', projectRoot) self.refresh()
def get(): """ Initialize the Moa job Determine if there is a moa job installed here and if so, what kind. This is done so to keep the possibility open to have multiple backends (i.e. possibly replacing make at a certain point """ global moaJob if moaJob: return moaJob l.debug("Initalizing Moa job") if os.path.exists('Makefile'): moaJob = MOAMAKEFILEJOB() else: moaJob = None return moaJob
def _addPath(wd, pth, tar): l.info("Checking %s" % pth) if not moa.info.isMoaDir(pth): l.info("not Moa: ignoring %s" % pth) return if not wd in pth: raise Exception("Odd path?? %s, its not part of %s" % (pth, wd)) rpth = pth.replace(wd, '.') l.info("Adding directory %s" % rpth) jobInfo = moa.info.info(rpth) l.debug("files: %s" % str(jobInfo['moa_files'].split())) for f in jobInfo['moa_files'].split(): addFile = os.path.join(rpth, f) if not os.path.exists(addFile): continue l.info("Adding file %s" % addFile) tar.add(addFile)
def newJob(data): """ Create a new job """ wd = data['wd'] options = data['options'] args = data['newargs'] params = [] template = 'traverse' for a in args: if '=' in a: params.append(a) else: template = a l.debug("Creating a new '%s' job" % template) if options.directory: wd = options.directory if os.path.exists(os.path.join( wd, '.moa', 'template')) and \ not options.force: l.error("Seems that there is already a Moa job in") l.error(wd) l.error("") l.error("use -f to override") title = options.title job = moa.job.newJob(wd, template=template) if title: job.conf.set('title', title) for p in params: k, v = p.split('=', 1) job.conf.set(k, v) job.conf.save()
def newJob(data): """ Create a new job """ wd = data['wd'] options = data['options'] args = data['newargs'] params = [] template = 'traverse' for a in args: if '=' in a: params.append(a) else: template = a l.debug("Creating a new '%s' job" % template) if options.directory: wd = options.directory if os.path.exists(os.path.join( wd, '.moa', 'template')) and \ not options.force: l.error("Seems that there is already a Moa job in") l.error(wd) l.error("") l.error("use -f to override") title = options.title job = moa.job.newJob(wd, template = template) if title: job.conf.set('title', title) for p in params: k,v = p.split('=', 1) job.conf.set(k,v) job.conf.save()
def _addPath(wd, pth, tar): l.info("Checking %s" % pth) if not moa.info.isMoaDir(pth): l.info("not Moa: ignoring %s" % pth) return if not wd in pth: raise Exception("Odd path?? %s, its not part of %s" %( pth, wd)) rpth = pth.replace(wd, '.') l.info("Adding directory %s" % rpth) jobInfo = moa.info.info(rpth) l.debug("files: %s" % str(jobInfo['moa_files'].split())) for f in jobInfo['moa_files'].split(): addFile = os.path.join(rpth, f) if not os.path.exists(addFile): continue l.info("Adding file %s" % addFile) tar.add(addFile)
def testPlugins(args=[]): global pluginFailures global pluginTests #new style plugin tests plugins = moa.plugins.Plugins() for plugin, testCode in plugins.getAttr('TESTSCRIPT'): #if asking for a single plugin, test only that plugin if args and plugin not in args: continue l.info("Starting new style test of %s" % plugin) testDir = tempfile.mkdtemp() testScript = os.path.join(testDir, 'test.sh') with open(testScript, 'w') as F: F.write(TESTSCRIPTHEADER) F.write(testCode) l.debug("executing test.sh in %s" % testScript) p = subprocess.Popen('bash %s' % testScript, shell=True, cwd = testDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) out, err = p.communicate() rc = p.returncode if rc != 0: l.critical("Errors in plugin test %s (rc %d)" % (plugin, rc)) if out: l.critical("Stdout:\n" + out) if err: l.critical("Stderr:\n" + err) pluginFailures += 1 else: if out: l.debug("Stdout:\n" + out) if err: l.info("Stderr:\n" + err) l.info("Success testing %s (%d lines)" % ( plugin, len(testCode.strip().split("\n")))) pluginTests += 1
def prepareInvocation(self, invocation): self.invocation = invocation # Prepare the arguments for a moaMake call if invocation.options.remake: self.makeArgs.append('-B') if invocation.options.makedebug: self.makeArgs.append('-d') if invocation.options.threads > 1: l.debug("Running make with %d threads" % options.threads) self.makeArgs.append('-j %d' % invocation.options.threads) ## make sure the MOA_THREADS env var is set - this is used from inside ## the Makefiles later threads need to be treated different from the ## other parameters. multi-threaded operation is only allowed in the ## second phase of execution. os.putenv('MOA_THREADS', "%s" % options.threads) ## Store all arguments in the environment - we might want to have a ## look at them from the Makefiles os.putenv("MOAARGS", "%s" % " ".join(["'" + str(x) + "'" for x in args]))
def initialize(self): """ attempt to load the python part of the plugins """ ## do we have a python module?? l.debug('Start plugin init') for plugin in self.allPlugins: pyModule = 'moa.plugin.%s' % plugin try: _m = __import__( pyModule, globals(), locals(), ['git'], -1) self.plugins[plugin] = _m l.debug("Successfully Loaded module %s" % pyModule) except ImportError, e: if not str(e) == "No module named %s" % plugin: raise l.debug("No python plugin module found for %s" % plugin)
def initialize(self, template=None): """ Create a new GnuMake job in the `wd` """ l.debug("Creating a new job from template '%s'" % template) l.debug("- in wd %s" % self.wd) if not template: template = self.job.template if not template.valid: l.error("Invalid template") return False if not template.backend == 'gnumake': l.error("template backend mismatch") return False l.debug("Start writing %s" % self.makefile) with open(self.makefile, 'w') as F: F.write(NEW_MAKEFILE_HEADER) F.write("$(call moa_load,%s)\n" % template)
def initialize(self, template = None): """ Create a new GnuMake job in the `wd` """ l.debug("Creating a new job from template '%s'" % template) l.debug("- in wd %s" % self.wd) if not template: template = self.job.template if not template.valid: l.error("Invalid template") return False if not template.backend == 'gnumake': l.error("template backend mismatch") return False l.debug("Start writing %s" % self.makefile) with open(self.makefile, 'w') as F: F.write(NEW_MAKEFILE_HEADER) F.write("$(call moa_load,%s)\n" % template)
def checkTemplate(path): """ Run all current template tests - see if any needs updating All job Makefiles should look exactly alike and have only two lines of actual Makefile code: include $(MOABASE)/template/moa/prepare.mk $(call moa_load,emboss/revseq) """ makefilePath = os.path.join(path, 'Makefile') if not os.path.exists(makefilePath): l.error("template check failed - cannot find a Makefile") seenIncludePrepare = False seenIncludeTemplate = False superfluousLines = False template = None makefile = open(makefilePath) i = 0 nmf = [] for line in makefile.readlines(): line = line.strip() i += 1 if not line: continue if line[0] == '#': nmf += line continue if line == 'include $(MOABASE)/template/moa/prepare.mk': nmf += line seenIncludePrepare = True continue if 'include $(MOABASE)/template/' in line: if template: l.error("Duplicate template include??") l.error("Please fix this!") sys.exit(-1) template = line.replace('include $(MOABASE)/template/', '') template = template.replace('.mk', '') l.debug("Discovered template %s" % template) continue if '$(call moa_load,' in line: if template: l.error("Duplicate template include??") l.error("Please fix this!") sys.exit(-1) seenIncludeTemplate = True template = line.replace('call moa_load,', '').replace(')', '').strip() l.debug("Discovered template %s" % template) continue if line in [ '.PHONY: moa_preprocess', 'moa_preprocess:', '.PHONY: moa_postprocess', 'moa_postprocess:', '-include moa.mk', 'MOAMK_INCLUDE=done', '@echo preprocess commands go here', '@echo Postprocess commands go here..', ]: continue superfluousLines = True l.info("extra code: '%s'" % line) if seenIncludePrepare and seenIncludeTemplate and (not superfluousLines): return True if superfluousLines: l.error("Deprecated Makefile, discovered extra code") l.error("Please fix this manually") sys.exit(-1) l.error("Deprecated template, fixing. Please check!") shutil.move(makefilePath, makefilePath + '.old') moa.job.newJob(template=template, wd=path, noInit=True, titleCheck=False) sys.exit(0)
def run(self, command): for p in self.keys(): if not command in dir(self[p]): continue l.debug("plugin executing hook %s for %s" % (command, p)) getattr(self[p], command)(self.data)
def renumber(path, fr, to): """ Renumber a moa job >>> import tempfile >>> emptyDir = tempfile.mkdtemp() >>> removeFiles(emptyDir, recursive=True) >>> fromDir = os.path.join(emptyDir, '10.test') >>> problemDir = os.path.join(emptyDir, '20.problem') >>> toDir = os.path.join(emptyDir, '20.test') >>> os.mkdir(os.path.join(emptyDir, '10.test')) >>> os.path.exists(os.path.join(emptyDir, '10.test')) True >>> os.path.exists(toDir) False >>> renumber(emptyDir, '10', '20') >>> os.path.exists(fromDir) False >>> os.path.exists(toDir) True >>> os.mkdir(problemDir) >>> renumber(emptyDir, '20', '30') Traceback (most recent call last): File '/opt/moa/lib/python/moa/utils.py', line 114, in renumber raise MoaFileError(fullDir) MoaFileError: Moa error handling file @param path: the path to operate in @type path: String @param fr: number to rename from @type fr: String representing a number @param to: number to rename to @type to: String representing a number """ frDir = None toDir = None l.debug("moa ren %s %s" % (fr, to)) for x in os.listdir(path): if x[0] == '.' : continue fullDir = os.path.join(path, x) xsplit = x.split('.') if xsplit[0] == fr: if frDir: l.error("more than one directory starting with %s" % fr) raise MoaFileError(fullDir) frDir = fullDir toDir = os.path.join(path, to + "." + ".".join(xsplit[1:])) if xsplit[0] == to: l.error("target directory starting with %s already exists" % to) raise MoaFileError(fullDir) if not frDir: l.error("Cannot find a directory starting with %s" % fr) raise MoaFileError(path) if not toDir: l.error("Cannot find a directory starting with %s" % to) raise MoaFileError(path) if not os.path.isdir(frDir): l.error("%s is not a directory" % frDir) raise MoaFileError(frDir) #if not os.path.isdir(toDir): # l.error("%s is not a directory" % toDir) # raise MoaFileError(toDir) l.info("renaming: %s" % (frDir)) l.info(" to: %s" % (toDir)) os.rename(frDir, toDir)
def checkTemplate(path): """ Run all current template tests - see if any needs updating All job Makefiles should look exactly alike and have only two lines of actual Makefile code: include $(MOABASE)/template/moa/prepare.mk $(call moa_load,emboss/revseq) """ makefilePath = os.path.join(path, 'Makefile') if not os.path.exists(makefilePath): l.error("template check failed - cannot find a Makefile") seenIncludePrepare = False seenIncludeTemplate = False superfluousLines = False template = None makefile = open(makefilePath) i = 0 nmf = [] for line in makefile.readlines(): line = line.strip() i += 1 if not line: continue if line[0] == '#': nmf += line continue if line == 'include $(MOABASE)/template/moa/prepare.mk': nmf += line seenIncludePrepare = True continue if 'include $(MOABASE)/template/' in line: if template: l.error("Duplicate template include??") l.error("Please fix this!") sys.exit(-1) template = line.replace('include $(MOABASE)/template/', '') template = template.replace('.mk', '') l.debug("Discovered template %s" % template) continue if '$(call moa_load,' in line: if template: l.error("Duplicate template include??") l.error("Please fix this!") sys.exit(-1) seenIncludeTemplate = True template = line.replace('call moa_load,', '').replace(')', '').strip() l.debug("Discovered template %s" % template) continue if line in ['.PHONY: moa_preprocess', 'moa_preprocess:', '.PHONY: moa_postprocess', 'moa_postprocess:', '-include moa.mk', 'MOAMK_INCLUDE=done', '@echo preprocess commands go here', '@echo Postprocess commands go here..', ]: continue superfluousLines = True l.info("extra code: '%s'" % line) if seenIncludePrepare and seenIncludeTemplate and (not superfluousLines): return True if superfluousLines: l.error("Deprecated Makefile, discovered extra code") l.error("Please fix this manually") sys.exit(-1) l.error("Deprecated template, fixing. Please check!") shutil.move(makefilePath, makefilePath + '.old') moa.job.newJob(template=template, wd=path, noInit=True, titleCheck=False) sys.exit(0)
def new(self, template, title=None, parameters=[], force=False, titleCheck=True, noInit=False): """ Create a new template in the `wd` """ l.debug("Creating a new job from template '%s'" % template) l.debug("- in wd %s" % self.wd) if not os.path.exists(self.wd): l.debug("creating folder for %s" % self.wd) os.makedirs(self.wd) #TODO: do something with the results of this check if not moa.template.check(template): l.error("Invalid template") if os.path.exists(self.makefile): l.debug("Makefile exists!") if not force: l.error("makefile exists, use -f (--force) to overwrite") sys.exit(-1) l.debug("Start writing %s" % self.makefile) with open(self.makefile, 'w') as F: F.write(NEW_MAKEFILE_HEADER) F.write("$(call moa_load,%s)\n" % template) if title: self.conf.add('title', title) params = [] for par in parameters: if not '=' in par: continue self.conf.add(par) self.conf.save() if noInit: return l.debug("Running moa initialization") job = moa.runMake.MOAMAKE(wd=self.wd, target='initialize', captureOut=False, captureErr=False, stealth=True, verbose=False) job.run() job.finish() l.debug("Written %s, try: moa help" % self.makefile) # check if a title is defined as 'title=something' on the # commandline, as opposed to using the -t option if not title: for p in parameters: if p.find('title=') == 0: title = p.split('=', 1)[1].strip() parameters.remove(p) break if (not title) and titleCheck and (not template == 'traverse'): l.warning("You *must* specify a job title") l.warning("You can still do so by running: ") l.warning(" moa set title='something descriptive'") title = "" if title: l.debug('creating a new moa makefile with title "%s" in %s' % (title, self.wd)) else: l.debug('creating a new moa makefile in %s' % (self.wd))
def createAdhoc(data): """ Create an adhoc job """ wd = data['cwd'] options = data['options'] args = data['newargs'] command = " ".join(args).strip() if not command: command=moa.utils.askUser('adhoc_command:\n>', '') l.info('Parsing command: %s' % command) params = [] mode = None searchGlobs = True if options.mode: mode = options.mode if options.mode == 'simple': searchGlobs = False if not options.mode in ['seq', 'par', 'all', 'simple']: l.critical("Unknown adhoc mode: %s" % options.mode) sys.exit(-1) elif '$<' in command: mode = 'seq' searchGlobs = False elif ('$^' in command) or ('$?' in command): mode = 'all' searchGlobs = False l.warning("Observed '$^' or '$?', setting mode to 'all'") l.warning("Processing all files in one go") #see if we have to look for file globs if not searchGlobs: l.info("No recognizable globs found") else: #it appears to make sense to see if there is a glob in the command refindGlob = re.compile( r"([^ *]+" \ + os.sep \ + ")?([^ *]*\*[^ *]*?)((?:\.[^ .*]+)?)") globs = [] for g in refindGlob.finditer(command): globs.append(g) if globs: globReplace = '$<', '$t' mode = 'seq' if len(globs) > 2: raise Exception("Too many globs ?? I not understand :(") if len(globs) == 2: st1 = _sourceOrTarget(globs[0]) st2 = _sourceOrTarget(globs[1]) if st1 == st2: l.warn("Unsure wich is the source & target glob, assuming:") inGlob,outGlob = globs if st1 == 'source': inGlob,outGlob = globs else: outGlob,inGlob = globs globReplace = '$t', '$<' l.info("Input glob: %s" % inGlob.group()) l.info("Output glob: %s" % outGlob.group()) else: l.info("Input glob: %s" % globs[0].group()) inGlob, outGlob = globs[0], None inD, inG, inE = inGlob.groups() if not inD: inD = "" if not inE: inE = "" l.info(" - set input dir : %s" % inD) l.info(" - set input glob : %s" % inG) l.info(" - set input extension : %s" % inE[1:]) params += ['adhoc_input_dir=%s' % inD] params += ['adhoc_input_glob=%s' % inG] params += ['adhoc_input_extension=%s' % inE[1:]] if outGlob: ouD, ouG, ouE = outGlob.groups() if not ouD: ouD = "" if not ouE: ouE = "" ogg = outGlob.groups() ouG1, ouG2 = ouG.split('*') sed = r"s^\(.*\)%s^%s%s\1%s%s^g" % ( inE.replace('.', '\.'), ouD.replace('/', '/'), ouG.split('*')[0], ouG.split('*')[1], ouE ) l.info(" - set name_sed : %s " % sed) l.info(" - set output dir : %s " % ouD) params += ['adhoc_output_dir=%s' % ouD] params += ['adhoc_name_sed=%s' % sed] #hack the commandline for i in range(len(globs)-1, -1, -1): g = globs[i] command = command[:g.start()] + globReplace[i] + command[g.end():] if not mode: mode = 'simple' if command: l.info(" - set command : %s" % command) params.append('adhoc_process=%s' % command) params.append('adhoc_mode=%s' % mode) l.info(" - set mode : %s" % mode) if mode == 'seq': l.warning("Note: adhoc is running in sequential ('seq') mode. If ") l.warning("you are confident that the individual jobs do not interfere, you might ") l.warning("consider setting adhoc to parallel operation:") l.warning("$ moa set adhoc_mode=par") l.debug('setting parameters %s' % params) job = moa.job.newJob(wd, template='adhoc', title = options.title, force = options.force, parameters=params)
def renumber(path, fr, to): """ Renumber a moa job >>> import tempfile >>> emptyDir = tempfile.mkdtemp() >>> removeFiles(emptyDir, recursive=True) >>> fromDir = os.path.join(emptyDir, '10.test') >>> problemDir = os.path.join(emptyDir, '20.problem') >>> toDir = os.path.join(emptyDir, '20.test') >>> os.mkdir(os.path.join(emptyDir, '10.test')) >>> os.path.exists(os.path.join(emptyDir, '10.test')) True >>> os.path.exists(toDir) False >>> renumber(emptyDir, '10', '20') >>> os.path.exists(fromDir) False >>> os.path.exists(toDir) True >>> os.mkdir(problemDir) >>> renumber(emptyDir, '20', '30') Traceback (most recent call last): File '/opt/moa/lib/python/moa/utils.py', line 114, in renumber raise MoaFileError(fullDir) MoaFileError: Moa error handling file @param path: the path to operate in @type path: String @param fr: number to rename from @type fr: String representing a number @param to: number to rename to @type to: String representing a number """ frDir = None toDir = None l.debug("moa ren %s %s" % (fr, to)) for x in os.listdir(path): if x[0] == '.': continue fullDir = os.path.join(path, x) xsplit = x.split('.') if xsplit[0] == fr: if frDir: l.error("more than one directory starting with %s" % fr) raise MoaFileError(fullDir) frDir = fullDir toDir = os.path.join(path, to + "." + ".".join(xsplit[1:])) if xsplit[0] == to: l.error("target directory starting with %s already exists" % to) raise MoaFileError(fullDir) if not frDir: l.error("Cannot find a directory starting with %s" % fr) raise MoaFileError(path) if not toDir: l.error("Cannot find a directory starting with %s" % to) raise MoaFileError(path) if not os.path.isdir(frDir): l.error("%s is not a directory" % frDir) raise MoaFileError(frDir) #if not os.path.isdir(toDir): # l.error("%s is not a directory" % toDir) # raise MoaFileError(toDir) l.info("renaming: %s" % (frDir)) l.info(" to: %s" % (toDir)) os.rename(frDir, toDir)