def _gitInit(data): """ Initialize a git repository """ global GITREPO global GITROOT if GITROOT: l.critical("trying to initialize a git repository wihtin a git repository") sys.exit(-1) wd = data['cwd'] GITROOT = wd GITREPO = git.Repo.init(GITROOT) l.info("created a git repository at %s" % GITROOT) info = moa.info.info(wd) index = GITREPO.index index.add(info['moa_files'].split()) index.commit('Settin up project "%s"' % info['title']) #write some data to .gitignore with open(os.path.join(wd, '.gitignore'), 'w') as F: F.write(".*\n") F.write("moa.success\n") F.write("moa.out\n") F.write("moa.err\n") F.write("moa.failed\n") F.write("moa.runlock\n")
def gitlog(data): """ Print a log to screen """ global GITREPO global GITROOT if not GITREPO: l.info("noting to report - no repo") return tags = {} for t in GITREPO.tags: print t.commit tags[t.commit] = t for c in GITREPO.iter_commits(): #if str(c) in tags.keys() t = time.strftime("%d %b %Y %H:%M", time.localtime(c.authored_date)) tag = None if c in tags.keys(): print " tag| %s" % tags[c] print "%3s | %s | %s" % (c.count(), t, c.message)
def _gitInit(data): """ Initialize a git repository """ global GITREPO global GITROOT if GITROOT: l.critical( "trying to initialize a git repository wihtin a git repository") sys.exit(-1) wd = data['cwd'] GITROOT = wd GITREPO = git.Repo.init(GITROOT) l.info("created a git repository at %s" % GITROOT) info = moa.info.info(wd) index = GITREPO.index index.add(info['moa_files'].split()) index.commit('Settin up project "%s"' % info['title']) #write some data to .gitignore with open(os.path.join(wd, '.gitignore'), 'w') as F: F.write(".*\n") F.write("moa.success\n") F.write("moa.out\n") F.write("moa.err\n") F.write("moa.failed\n") F.write("moa.runlock\n")
def tag(data): global GITREPO if not GITREPO: l.info("no repository is initialized") return tagname = data['args'][1] message = data['options'].gitMessage l.info('tagging with "%s"' % tagname) GITREPO.create_tag(tagname, message=message)
def removeDirectory(path): """ Remove a directory with contents - no questions asked >>> import tempfile >>> tempdir = tempfile.mkdtemp() >>> os.path.exists(tempdir) True >>> os.path.isdir(tempdir) True >>> removeDirectory(tempdir) >>> os.path.exists(tempdir) False """ l.info("Removing %s" % path) shutil.rmtree(path) l.info("Finished removing %s" % path)
def __init__(self, wd, options, args): self.wd = wd self.options = options self.args = args self.job = None ### determine what the command is self.command = "" if len(args) > 0: self.command = args[0] self.args = args[1:] l.info('Starting moa command "%s"' % self.command) ### Determine project root (if there is one) projectRoot = moa.project.findProjectRoot(self.wd) if projectRoot: l.debug('Project root is %s' % projectRoot) os.putenv('MOAPROJECTROOT', projectRoot) self.refresh()
def unpack(data): args = data['newargs'] packName = args[0] originalPackname = packName packFile = _getPackFile(packName) if not packFile: l.error("cannot locate a packfile for %s" % packName) sys.exit(-1) options = data['options'] if options.directory: target = options.directory if not os.path.exists(target): os.makedirs(target) else: target = '.' if (not options.force) and (moa.info.isMoaDir(target)): l.error("There is already a moa job in %s" % target) l.error("Use force (-f) to unpack it here") sys.exit(-1) packArgs = _getPackArgs(packFile) providedArgs = data['newargs'][1:] if len(packArgs) != len(providedArgs): l.error("Provided the wrong amount of arguments") l.error("This packfile needs to be called with the") l.error("following arguments:") l.error(" %s" % " ".join(packArgs)) if not options.force: sys.exit(-1) else: l.error("continuing anyway (force)") for a in range(len(providedArgs)): l.info("setting %s to %s" % (packArgs[a], args[a])) moa.conf.setVar(target, packArgs[a], args[a])
def _addPath(wd, pth, tar): l.info("Checking %s" % pth) if not moa.info.isMoaDir(pth): l.info("not Moa: ignoring %s" % pth) return if not wd in pth: raise Exception("Odd path?? %s, its not part of %s" % (pth, wd)) rpth = pth.replace(wd, '.') l.info("Adding directory %s" % rpth) jobInfo = moa.info.info(rpth) l.debug("files: %s" % str(jobInfo['moa_files'].split())) for f in jobInfo['moa_files'].split(): addFile = os.path.join(rpth, f) if not os.path.exists(addFile): continue l.info("Adding file %s" % addFile) tar.add(addFile)
def _addPath(wd, pth, tar): l.info("Checking %s" % pth) if not moa.info.isMoaDir(pth): l.info("not Moa: ignoring %s" % pth) return if not wd in pth: raise Exception("Odd path?? %s, its not part of %s" %( pth, wd)) rpth = pth.replace(wd, '.') l.info("Adding directory %s" % rpth) jobInfo = moa.info.info(rpth) l.debug("files: %s" % str(jobInfo['moa_files'].split())) for f in jobInfo['moa_files'].split(): addFile = os.path.join(rpth, f) if not os.path.exists(addFile): continue l.info("Adding file %s" % addFile) tar.add(addFile)
def moacp(data): """ Copy a moa job - 0 create a new directory 1 copy the configuration TODO: adapt file & dir links """ wd = data['cwd'] options = data['options'] args = data['newargs'] if len(args) > 1: dirto = args[1] else: dirto = '.' dirfrom = args[0] #remove trailing slash & determine basename if dirfrom[-1] == '/': dirfrom = dirfrom[:-1] fromBase = os.path.basename(dirfrom) # trick - the second argument is a number # renumber the target directory if re.match("^[0-9]+$", dirto): dirto = re.sub("^[0-9]*\.", dirto + '.', fromBase) if not os.path.exists(dirto): l.info("creating directory %s" % dirto) os.makedirs(dirto) else: dirto = os.path.join(dirto, fromBase) os.makedirs(dirto) l.info("Copying from %s to %s" % (dirfrom, dirto)) for f in ['Makefile', 'moa.mk']: cfr = os.path.join(dirfrom, f) cto = os.path.join(dirto, f) l.info("copy %s to %s" % (cfr, cto)) shutil.copyfile(cfr, cto)
def testPlugins(args=[]): global pluginFailures global pluginTests #new style plugin tests plugins = moa.plugins.Plugins() for plugin, testCode in plugins.getAttr('TESTSCRIPT'): #if asking for a single plugin, test only that plugin if args and plugin not in args: continue l.info("Starting new style test of %s" % plugin) testDir = tempfile.mkdtemp() testScript = os.path.join(testDir, 'test.sh') with open(testScript, 'w') as F: F.write(TESTSCRIPTHEADER) F.write(testCode) l.debug("executing test.sh in %s" % testScript) p = subprocess.Popen('bash %s' % testScript, shell=True, cwd = testDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) out, err = p.communicate() rc = p.returncode if rc != 0: l.critical("Errors in plugin test %s (rc %d)" % (plugin, rc)) if out: l.critical("Stdout:\n" + out) if err: l.critical("Stderr:\n" + err) pluginFailures += 1 else: if out: l.debug("Stdout:\n" + out) if err: l.info("Stderr:\n" + err) l.info("Success testing %s (%d lines)" % ( plugin, len(testCode.strip().split("\n")))) pluginTests += 1
def pack(data): """ Create an adhoc job """ wd = data['cwd'] if wd[-1] == '/': wd = wd[:-1] options = data['options'] args = data['newargs'] #see if we're calling a pack command if args: if args[0] == 'unpack': return unpack(data) elif args[0] == 'list' : return listPacks(data) elif args[0] == 'rm' : return rmPack(data) elif args[0] == 'args' : return packArgs(data) elif args[0] == 'create': args = args[1:] packName, packPath = _getPackName(data) l.info('start creating pack in %s' % wd) l.info("pack name: %s" % packName) l.info("pack path: %s" % packPath) TF = tarfile.open( name = packPath, mode = 'w:bz2') if args: l.info("packing with args %s" % args) packArgsFile = os.path.join(wd, 'moa.packargs') with open(packArgsFile, 'w') as F: F.write(" ".join(args)) l.info("adding %s" % packArgsFile) TF.add(packArgsFile, os.path.basename(packArgsFile)) os.unlink(packArgsFile) if options.packRecursive: for dirpath, dirnames, filenames in os.walk(wd): _addPath(wd, dirpath, TF) else: _addPath(wd, wd, TF) l.info("done packing") TF.close()
def pack(data): """ Create an adhoc job """ wd = data['cwd'] if wd[-1] == '/': wd = wd[:-1] options = data['options'] args = data['newargs'] #see if we're calling a pack command if args: if args[0] == 'unpack': return unpack(data) elif args[0] == 'list': return listPacks(data) elif args[0] == 'rm': return rmPack(data) elif args[0] == 'args': return packArgs(data) elif args[0] == 'create': args = args[1:] packName, packPath = _getPackName(data) l.info('start creating pack in %s' % wd) l.info("pack name: %s" % packName) l.info("pack path: %s" % packPath) TF = tarfile.open(name=packPath, mode='w:bz2') if args: l.info("packing with args %s" % args) packArgsFile = os.path.join(wd, 'moa.packargs') with open(packArgsFile, 'w') as F: F.write(" ".join(args)) l.info("adding %s" % packArgsFile) TF.add(packArgsFile, os.path.basename(packArgsFile)) os.unlink(packArgsFile) if options.packRecursive: for dirpath, dirnames, filenames in os.walk(wd): _addPath(wd, dirpath, TF) else: _addPath(wd, wd, TF) l.info("done packing") TF.close()
def templateConvert(data): args = data['newargs'] if len(args) < 1: l.critical("need to specify a target directory") newTemplateDir = args[0] convert_only = args[1:] if not os.path.exists(newTemplateDir): l.warning("creating directory %s" % newTemplateDir) os.makedirs(newTemplateDir) for job in moa.job.list(): if convert_only and not job in convert_only: continue newTemplateConf = os.path.join(newTemplateDir, '%s.moa' % job) newTemplateFile = os.path.join(newTemplateDir, '%s.mk' % job) l.info("start conversion of %s" % job) wd = moa.job.newTestJob(template=job, title='for conversion') inf = moa.info.info(wd) moaId = inf['moa_id'] inf['backend'] = 'gnumake' del inf['parameter_categories'] inf['commands'] = inf['moa_targets'] inf['commands'].remove('all') del inf['moa_targets'] del inf['all_help'] if inf.has_key('title'): del inf['title'] del inf['moa_files'] if not inf['description'] and \ inf['template_description']: inf['description'] = inf['template_description'] del inf['template_file'] for k in inf.keys(): if 'template_' in k: v = inf[k] del inf[k] inf[k.replace('template_', '')] = v inf['help'] = inf.get('help', {}) for c in inf['commands']: inf['help'][c] = inf['%s_help' % c] del inf['%s_help' % c] inf['gnumake_makefile'] = newTemplateFile #del inf['template_file'] for p in inf.get('parameters', []): par = inf['parameters'][p] del inf['parameters'][p] newParName = p.replace('%s_' % moaId, '') if par.has_key('value'): del par['value'] if par.has_key('cardinality'): del par['cardinality'] par['optional'] = True if par.has_key('mandatory'): par['optional'] = not par['mandatory'] del par['mandatory'] #print p, newParName #for k in par: # print ' -', k, par[k] inf['parameters'][newParName]= par #convert gnumake file oldTemplateFile = os.path.join(MOABASE, 'template', '%s.mk' % job) with open(oldTemplateFile) as F: lines = F.readlines() #first parse on a line by line basis for lineNo, line in enumerate(lines): for m in MAKEFILE_CONVERT_LINES: if line.find(m[0]) == 0: lines[lineNo] = m[1] break text = "".join(lines) #print text #see if there are 'input filesets' #$(call moa_fileset_define_opt,$(moa_id)_input,,Input files for $(moa_id)) rere = re.compile(r'\$\(call moa_fileset_define(.*),(.+),(.*),(.*)\)') for x in rere.finditer(text): fsid = x.groups()[1] fsext = x.groups()[2] fsid = fsid.replace('$(moa_id)', moaId).replace(moaId + '_','') fsdesc = x.groups()[3].replace('$(moa_id)', moaId) l.info("found fileset for %s" % fsid) if x.groups()[0] == '_opt': fsopt = True else: fsopt = False fs = { 'type' : 'input', 'category' : 'input', 'help' : fsdesc, 'extension' : fsext, 'optional' : fsopt } if not inf.has_key('filesets'): inf['filesets'] = {} inf['filesets'][fsid] = fs #and now remove all related parameters from the configuration for x in ['_extension', '_glob', '_limit', '_sort', '_dir']: ky = fsid + x if inf['parameters'].has_key(ky): del inf['parameters'][ky] text = rere.sub('', text) # see if there are 'remap' filesets rere = re.compile(r'\$\(call moa_fileset_remap(.*),(.+),(.*),(.*)\)') for x in rere.finditer(text): fsid = x.groups()[2].replace('$(moa_id)', moaId).replace(moaId + '_','') fssrc = x.groups()[1].replace('$(moa_id)', moaId).replace(moaId + '_','') fs_target_extension = x.groups()[3] l.info("found a remap fileset: %s to %s" % (fssrc, fsid)) if x.groups()[0] == '_nodir': fs_target_dir = '.' else: fs_target_dir = os.path.join('.', fs_target_extension) fs = {'type' : 'map', 'category' : 'output', 'source' : fssrc, 'dir' : fs_target_dir, 'extension' : fs_target_extension } if not inf.has_key('filesets'): inf['filesets'] = {} inf['filesets'][fsid] = fs text = rere.sub('', text) #rere = re.compile(r'(\$\(call.*)') #for x in rere.finditer(text): # l.warning("found $call") # l.warning(x.groups()[0]) for suffix in ['title', 'help', 'type', 'define', 'description', 'prerequisites', 'default', 'allowed', 'set', 'simple']: rere = re.compile(r'\S+'+suffix+r' ?\+?=.*?(?<!\\)\n', re.S) text = rere.sub('', text) rere = re.compile(r'([\t ]*\n){2,}', re.S) text = rere.sub("\n\n", text) #save & store the new files thisTargetDir = os.path.dirname(newTemplateConf) if not os.path.exists(thisTargetDir): os.makedirs(thisTargetDir) with open(newTemplateConf, 'w') as F: yaml.dump(inf, F) with open(newTemplateFile, 'w') as G: G.write(text)
def templateConvert(data): args = data['newargs'] if len(args) < 1: l.critical("need to specify a target directory") newTemplateDir = args[0] convert_only = args[1:] if not os.path.exists(newTemplateDir): l.warning("creating directory %s" % newTemplateDir) os.makedirs(newTemplateDir) for job in moa.job.list(): if convert_only and not job in convert_only: continue newTemplateConf = os.path.join(newTemplateDir, '%s.moa' % job) newTemplateFile = os.path.join(newTemplateDir, '%s.mk' % job) l.info("start conversion of %s" % job) wd = moa.job.newTestJob(template=job, title='for conversion') inf = moa.info.info(wd) moaId = inf['moa_id'] inf['backend'] = 'gnumake' del inf['parameter_categories'] inf['commands'] = inf['moa_targets'] inf['commands'].remove('all') del inf['moa_targets'] del inf['all_help'] if inf.has_key('title'): del inf['title'] del inf['moa_files'] if not inf['description'] and \ inf['template_description']: inf['description'] = inf['template_description'] del inf['template_file'] for k in inf.keys(): if 'template_' in k: v = inf[k] del inf[k] inf[k.replace('template_', '')] = v inf['help'] = inf.get('help', {}) for c in inf['commands']: inf['help'][c] = inf['%s_help' % c] del inf['%s_help' % c] inf['gnumake_makefile'] = newTemplateFile #del inf['template_file'] for p in inf.get('parameters', []): par = inf['parameters'][p] del inf['parameters'][p] newParName = p.replace('%s_' % moaId, '') if par.has_key('value'): del par['value'] if par.has_key('cardinality'): del par['cardinality'] par['optional'] = True if par.has_key('mandatory'): par['optional'] = not par['mandatory'] del par['mandatory'] #print p, newParName #for k in par: # print ' -', k, par[k] inf['parameters'][newParName] = par #convert gnumake file oldTemplateFile = os.path.join(MOABASE, 'template', '%s.mk' % job) with open(oldTemplateFile) as F: lines = F.readlines() #first parse on a line by line basis for lineNo, line in enumerate(lines): for m in MAKEFILE_CONVERT_LINES: if line.find(m[0]) == 0: lines[lineNo] = m[1] break text = "".join(lines) #print text #see if there are 'input filesets' #$(call moa_fileset_define_opt,$(moa_id)_input,,Input files for $(moa_id)) rere = re.compile(r'\$\(call moa_fileset_define(.*),(.+),(.*),(.*)\)') for x in rere.finditer(text): fsid = x.groups()[1] fsext = x.groups()[2] fsid = fsid.replace('$(moa_id)', moaId).replace(moaId + '_', '') fsdesc = x.groups()[3].replace('$(moa_id)', moaId) l.info("found fileset for %s" % fsid) if x.groups()[0] == '_opt': fsopt = True else: fsopt = False fs = { 'type': 'input', 'category': 'input', 'help': fsdesc, 'extension': fsext, 'optional': fsopt } if not inf.has_key('filesets'): inf['filesets'] = {} inf['filesets'][fsid] = fs #and now remove all related parameters from the configuration for x in ['_extension', '_glob', '_limit', '_sort', '_dir']: ky = fsid + x if inf['parameters'].has_key(ky): del inf['parameters'][ky] text = rere.sub('', text) # see if there are 'remap' filesets rere = re.compile(r'\$\(call moa_fileset_remap(.*),(.+),(.*),(.*)\)') for x in rere.finditer(text): fsid = x.groups()[2].replace('$(moa_id)', moaId).replace(moaId + '_', '') fssrc = x.groups()[1].replace('$(moa_id)', moaId).replace(moaId + '_', '') fs_target_extension = x.groups()[3] l.info("found a remap fileset: %s to %s" % (fssrc, fsid)) if x.groups()[0] == '_nodir': fs_target_dir = '.' else: fs_target_dir = os.path.join('.', fs_target_extension) fs = { 'type': 'map', 'category': 'output', 'source': fssrc, 'dir': fs_target_dir, 'extension': fs_target_extension } if not inf.has_key('filesets'): inf['filesets'] = {} inf['filesets'][fsid] = fs text = rere.sub('', text) #rere = re.compile(r'(\$\(call.*)') #for x in rere.finditer(text): # l.warning("found $call") # l.warning(x.groups()[0]) for suffix in [ 'title', 'help', 'type', 'define', 'description', 'prerequisites', 'default', 'allowed', 'set', 'simple' ]: rere = re.compile(r'\S+' + suffix + r' ?\+?=.*?(?<!\\)\n', re.S) text = rere.sub('', text) rere = re.compile(r'([\t ]*\n){2,}', re.S) text = rere.sub("\n\n", text) #save & store the new files thisTargetDir = os.path.dirname(newTemplateConf) if not os.path.exists(thisTargetDir): os.makedirs(thisTargetDir) with open(newTemplateConf, 'w') as F: yaml.dump(inf, F) with open(newTemplateFile, 'w') as G: G.write(text)
def renumber(path, fr, to): """ Renumber a moa job >>> import tempfile >>> emptyDir = tempfile.mkdtemp() >>> removeFiles(emptyDir, recursive=True) >>> fromDir = os.path.join(emptyDir, '10.test') >>> problemDir = os.path.join(emptyDir, '20.problem') >>> toDir = os.path.join(emptyDir, '20.test') >>> os.mkdir(os.path.join(emptyDir, '10.test')) >>> os.path.exists(os.path.join(emptyDir, '10.test')) True >>> os.path.exists(toDir) False >>> renumber(emptyDir, '10', '20') >>> os.path.exists(fromDir) False >>> os.path.exists(toDir) True >>> os.mkdir(problemDir) >>> renumber(emptyDir, '20', '30') Traceback (most recent call last): File '/opt/moa/lib/python/moa/utils.py', line 114, in renumber raise MoaFileError(fullDir) MoaFileError: Moa error handling file @param path: the path to operate in @type path: String @param fr: number to rename from @type fr: String representing a number @param to: number to rename to @type to: String representing a number """ frDir = None toDir = None l.debug("moa ren %s %s" % (fr, to)) for x in os.listdir(path): if x[0] == '.' : continue fullDir = os.path.join(path, x) xsplit = x.split('.') if xsplit[0] == fr: if frDir: l.error("more than one directory starting with %s" % fr) raise MoaFileError(fullDir) frDir = fullDir toDir = os.path.join(path, to + "." + ".".join(xsplit[1:])) if xsplit[0] == to: l.error("target directory starting with %s already exists" % to) raise MoaFileError(fullDir) if not frDir: l.error("Cannot find a directory starting with %s" % fr) raise MoaFileError(path) if not toDir: l.error("Cannot find a directory starting with %s" % to) raise MoaFileError(path) if not os.path.isdir(frDir): l.error("%s is not a directory" % frDir) raise MoaFileError(frDir) #if not os.path.isdir(toDir): # l.error("%s is not a directory" % toDir) # raise MoaFileError(toDir) l.info("renaming: %s" % (frDir)) l.info(" to: %s" % (toDir)) os.rename(frDir, toDir)
def checkTemplate(path): """ Run all current template tests - see if any needs updating All job Makefiles should look exactly alike and have only two lines of actual Makefile code: include $(MOABASE)/template/moa/prepare.mk $(call moa_load,emboss/revseq) """ makefilePath = os.path.join(path, 'Makefile') if not os.path.exists(makefilePath): l.error("template check failed - cannot find a Makefile") seenIncludePrepare = False seenIncludeTemplate = False superfluousLines = False template = None makefile = open(makefilePath) i = 0 nmf = [] for line in makefile.readlines(): line = line.strip() i += 1 if not line: continue if line[0] == '#': nmf += line continue if line == 'include $(MOABASE)/template/moa/prepare.mk': nmf += line seenIncludePrepare = True continue if 'include $(MOABASE)/template/' in line: if template: l.error("Duplicate template include??") l.error("Please fix this!") sys.exit(-1) template = line.replace('include $(MOABASE)/template/', '') template = template.replace('.mk', '') l.debug("Discovered template %s" % template) continue if '$(call moa_load,' in line: if template: l.error("Duplicate template include??") l.error("Please fix this!") sys.exit(-1) seenIncludeTemplate = True template = line.replace('call moa_load,', '').replace(')', '').strip() l.debug("Discovered template %s" % template) continue if line in [ '.PHONY: moa_preprocess', 'moa_preprocess:', '.PHONY: moa_postprocess', 'moa_postprocess:', '-include moa.mk', 'MOAMK_INCLUDE=done', '@echo preprocess commands go here', '@echo Postprocess commands go here..', ]: continue superfluousLines = True l.info("extra code: '%s'" % line) if seenIncludePrepare and seenIncludeTemplate and (not superfluousLines): return True if superfluousLines: l.error("Deprecated Makefile, discovered extra code") l.error("Please fix this manually") sys.exit(-1) l.error("Deprecated template, fixing. Please check!") shutil.move(makefilePath, makefilePath + '.old') moa.job.newJob(template=template, wd=path, noInit=True, titleCheck=False) sys.exit(0)
def renumber(path, fr, to): """ Renumber a moa job >>> import tempfile >>> emptyDir = tempfile.mkdtemp() >>> removeFiles(emptyDir, recursive=True) >>> fromDir = os.path.join(emptyDir, '10.test') >>> problemDir = os.path.join(emptyDir, '20.problem') >>> toDir = os.path.join(emptyDir, '20.test') >>> os.mkdir(os.path.join(emptyDir, '10.test')) >>> os.path.exists(os.path.join(emptyDir, '10.test')) True >>> os.path.exists(toDir) False >>> renumber(emptyDir, '10', '20') >>> os.path.exists(fromDir) False >>> os.path.exists(toDir) True >>> os.mkdir(problemDir) >>> renumber(emptyDir, '20', '30') Traceback (most recent call last): File '/opt/moa/lib/python/moa/utils.py', line 114, in renumber raise MoaFileError(fullDir) MoaFileError: Moa error handling file @param path: the path to operate in @type path: String @param fr: number to rename from @type fr: String representing a number @param to: number to rename to @type to: String representing a number """ frDir = None toDir = None l.debug("moa ren %s %s" % (fr, to)) for x in os.listdir(path): if x[0] == '.': continue fullDir = os.path.join(path, x) xsplit = x.split('.') if xsplit[0] == fr: if frDir: l.error("more than one directory starting with %s" % fr) raise MoaFileError(fullDir) frDir = fullDir toDir = os.path.join(path, to + "." + ".".join(xsplit[1:])) if xsplit[0] == to: l.error("target directory starting with %s already exists" % to) raise MoaFileError(fullDir) if not frDir: l.error("Cannot find a directory starting with %s" % fr) raise MoaFileError(path) if not toDir: l.error("Cannot find a directory starting with %s" % to) raise MoaFileError(path) if not os.path.isdir(frDir): l.error("%s is not a directory" % frDir) raise MoaFileError(frDir) #if not os.path.isdir(toDir): # l.error("%s is not a directory" % toDir) # raise MoaFileError(toDir) l.info("renaming: %s" % (frDir)) l.info(" to: %s" % (toDir)) os.rename(frDir, toDir)
def checkTemplate(path): """ Run all current template tests - see if any needs updating All job Makefiles should look exactly alike and have only two lines of actual Makefile code: include $(MOABASE)/template/moa/prepare.mk $(call moa_load,emboss/revseq) """ makefilePath = os.path.join(path, 'Makefile') if not os.path.exists(makefilePath): l.error("template check failed - cannot find a Makefile") seenIncludePrepare = False seenIncludeTemplate = False superfluousLines = False template = None makefile = open(makefilePath) i = 0 nmf = [] for line in makefile.readlines(): line = line.strip() i += 1 if not line: continue if line[0] == '#': nmf += line continue if line == 'include $(MOABASE)/template/moa/prepare.mk': nmf += line seenIncludePrepare = True continue if 'include $(MOABASE)/template/' in line: if template: l.error("Duplicate template include??") l.error("Please fix this!") sys.exit(-1) template = line.replace('include $(MOABASE)/template/', '') template = template.replace('.mk', '') l.debug("Discovered template %s" % template) continue if '$(call moa_load,' in line: if template: l.error("Duplicate template include??") l.error("Please fix this!") sys.exit(-1) seenIncludeTemplate = True template = line.replace('call moa_load,', '').replace(')', '').strip() l.debug("Discovered template %s" % template) continue if line in ['.PHONY: moa_preprocess', 'moa_preprocess:', '.PHONY: moa_postprocess', 'moa_postprocess:', '-include moa.mk', 'MOAMK_INCLUDE=done', '@echo preprocess commands go here', '@echo Postprocess commands go here..', ]: continue superfluousLines = True l.info("extra code: '%s'" % line) if seenIncludePrepare and seenIncludeTemplate and (not superfluousLines): return True if superfluousLines: l.error("Deprecated Makefile, discovered extra code") l.error("Please fix this manually") sys.exit(-1) l.error("Deprecated template, fixing. Please check!") shutil.move(makefilePath, makefilePath + '.old') moa.job.newJob(template=template, wd=path, noInit=True, titleCheck=False) sys.exit(0)
def createAdhoc(data): """ Create an adhoc job """ wd = data['cwd'] options = data['options'] args = data['newargs'] command = " ".join(args).strip() if not command: command=moa.utils.askUser('adhoc_command:\n>', '') l.info('Parsing command: %s' % command) params = [] mode = None searchGlobs = True if options.mode: mode = options.mode if options.mode == 'simple': searchGlobs = False if not options.mode in ['seq', 'par', 'all', 'simple']: l.critical("Unknown adhoc mode: %s" % options.mode) sys.exit(-1) elif '$<' in command: mode = 'seq' searchGlobs = False elif ('$^' in command) or ('$?' in command): mode = 'all' searchGlobs = False l.warning("Observed '$^' or '$?', setting mode to 'all'") l.warning("Processing all files in one go") #see if we have to look for file globs if not searchGlobs: l.info("No recognizable globs found") else: #it appears to make sense to see if there is a glob in the command refindGlob = re.compile( r"([^ *]+" \ + os.sep \ + ")?([^ *]*\*[^ *]*?)((?:\.[^ .*]+)?)") globs = [] for g in refindGlob.finditer(command): globs.append(g) if globs: globReplace = '$<', '$t' mode = 'seq' if len(globs) > 2: raise Exception("Too many globs ?? I not understand :(") if len(globs) == 2: st1 = _sourceOrTarget(globs[0]) st2 = _sourceOrTarget(globs[1]) if st1 == st2: l.warn("Unsure wich is the source & target glob, assuming:") inGlob,outGlob = globs if st1 == 'source': inGlob,outGlob = globs else: outGlob,inGlob = globs globReplace = '$t', '$<' l.info("Input glob: %s" % inGlob.group()) l.info("Output glob: %s" % outGlob.group()) else: l.info("Input glob: %s" % globs[0].group()) inGlob, outGlob = globs[0], None inD, inG, inE = inGlob.groups() if not inD: inD = "" if not inE: inE = "" l.info(" - set input dir : %s" % inD) l.info(" - set input glob : %s" % inG) l.info(" - set input extension : %s" % inE[1:]) params += ['adhoc_input_dir=%s' % inD] params += ['adhoc_input_glob=%s' % inG] params += ['adhoc_input_extension=%s' % inE[1:]] if outGlob: ouD, ouG, ouE = outGlob.groups() if not ouD: ouD = "" if not ouE: ouE = "" ogg = outGlob.groups() ouG1, ouG2 = ouG.split('*') sed = r"s^\(.*\)%s^%s%s\1%s%s^g" % ( inE.replace('.', '\.'), ouD.replace('/', '/'), ouG.split('*')[0], ouG.split('*')[1], ouE ) l.info(" - set name_sed : %s " % sed) l.info(" - set output dir : %s " % ouD) params += ['adhoc_output_dir=%s' % ouD] params += ['adhoc_name_sed=%s' % sed] #hack the commandline for i in range(len(globs)-1, -1, -1): g = globs[i] command = command[:g.start()] + globReplace[i] + command[g.end():] if not mode: mode = 'simple' if command: l.info(" - set command : %s" % command) params.append('adhoc_process=%s' % command) params.append('adhoc_mode=%s' % mode) l.info(" - set mode : %s" % mode) if mode == 'seq': l.warning("Note: adhoc is running in sequential ('seq') mode. If ") l.warning("you are confident that the individual jobs do not interfere, you might ") l.warning("consider setting adhoc to parallel operation:") l.warning("$ moa set adhoc_mode=par") l.debug('setting parameters %s' % params) job = moa.job.newJob(wd, template='adhoc', title = options.title, force = options.force, parameters=params)
def run(options, args): #remove the first argument from args (should be 'test') args.pop(0) os.putenv('MOA_UNITTESTS', "yes") if args: l.info("Testing '%s'" % " ".join(args)) if not args: l.info("Start running python doctests") setSilent() testModule(moa.utils) testModule(moa.lock) testModule(moa.conf) testModule(moa.project) testModule(moa.template) testModule(moa.job) if options.verbose: setVerbose() else: setInfo() l.info("Finished running of python unittests") l.info("Ran %d test, %d failed" % (tests, failures)) l.info("Start running basic template tests") testTemplates() l.info("Ran %d template test, %d failed" % ( templateTests, templateFailures)) l.info("Start running plugin tests") testPlugins() l.info("Ran %d plugin test, %d failed" % ( pluginTests, pluginFailures)) l.info("Finished running plugin tests") sys.exit() elif args[0] == 'plugins': l.info("Start running plugin tests") testPlugins(args[1:]) l.info("Ran %d plugin test, %d failed" % ( pluginTests, pluginFailures)) l.info("Finished running plugin tests") elif args[0] == 'plugin': l.info("Start running plugin tests") testPlugins(args[1:]) elif args[0] == 'templates': l.info("Start running basic template tests") testTemplates() l.info("Ran %d template test, %d failed" % ( templateTests, templateFailures)) l.info("Finished running basic template tests") elif args[0][:4] == 'moa.': l.info("testing moa python module %s" % args[0]) setSilent() eval("testModule(%s)" % args[0]) if options.verbose: setVerbose() else: setInfo() l.info("Finished running unittests for %s" % args[0]) l.info("Ran %d test, %d failed" % (tests, failures)) else: #Assume it is a templat testTemplates(args[0], verbose=options.verbose) testTemplateExtensive(args[0], verbose=options.verbose)