def flogger(*args, **kwargs): l.critical("Executing %s" % func.__name__) for a in args: l.error(" - calling with arg %s" % a) for k in kwargs.keys(): l.error(" - calling with kwargs %s=%s" % (k, kwargs[k])) return func(*args, **kwargs)
def moamv(data): wd = data['cwd'] options = data['options'] args = data['newargs'] fr = args[0] if fr[-1] == '/': fr = fr[:-1] if len(args) > 1: to = args[1] else: to = '.' #see if fr is a number if re.match('\d+', fr): newfr = glob.glob('%s*' % fr) if len(newfr) != 1: l.critical("Cannot resolve %s (%s)" % (fr, newfr)) sys.exit(1) fr = newfr[0] if re.match('\d+', to): if re.search('^\d+', fr): to = re.sub('^\d+', to, fr) else: to = '%s.%s' % (to, fr) shutil.move(fr, to)
def _gitInit(data): """ Initialize a git repository """ global GITREPO global GITROOT if GITROOT: l.critical( "trying to initialize a git repository wihtin a git repository") sys.exit(-1) wd = data['cwd'] GITROOT = wd GITREPO = git.Repo.init(GITROOT) l.info("created a git repository at %s" % GITROOT) info = moa.info.info(wd) index = GITREPO.index index.add(info['moa_files'].split()) index.commit('Settin up project "%s"' % info['title']) #write some data to .gitignore with open(os.path.join(wd, '.gitignore'), 'w') as F: F.write(".*\n") F.write("moa.success\n") F.write("moa.out\n") F.write("moa.err\n") F.write("moa.failed\n") F.write("moa.runlock\n")
def _gitInit(data): """ Initialize a git repository """ global GITREPO global GITROOT if GITROOT: l.critical("trying to initialize a git repository wihtin a git repository") sys.exit(-1) wd = data['cwd'] GITROOT = wd GITREPO = git.Repo.init(GITROOT) l.info("created a git repository at %s" % GITROOT) info = moa.info.info(wd) index = GITREPO.index index.add(info['moa_files'].split()) index.commit('Settin up project "%s"' % info['title']) #write some data to .gitignore with open(os.path.join(wd, '.gitignore'), 'w') as F: F.write(".*\n") F.write("moa.success\n") F.write("moa.out\n") F.write("moa.err\n") F.write("moa.failed\n") F.write("moa.runlock\n")
def moakill(data): """ kill a running job """ cwd = data['cwd'] if not moa.info.status(cwd) == 'running': l.warning("Moa does not seem to be running!") sys.exit(-1) pid = int(open(os.path.join(cwd, 'moa.runlock')).read()) l.critical("killing job %d" % pid) os.kill(pid, 9)
def fixOld(wd): makefile = os.path.join(wd, 'Makefile') if not os.path.exists(makefile): return False with open(makefile) as F: t = F.read() if not 'include $(MOABASE)/template/moa/prepare.mk' in t: return False import shutil l.warning("Old style Makefile trying to upgrade!") t = t.replace('/template/moa/', '/lib/gnumake/') if os.path.exists('%s.old' % makefile): os.unlink('%s.old' % makefile) shutil.move(makefile, '%s.old' % makefile) with open('%s' % makefile, 'w') as F: F.write(t) templateName = t.split('moa_load,')[1].replace(')', '').strip() l.warning("Found a makefile with template %s" % templateName) cdir = os.path.join(wd, '.moa') if not os.path.exists(cdir): os.mkdir(cdir) l.warning("wrote an updated Makefile - please check!") with open(os.path.join(cdir, 'template'), 'w') as F: F.write(templateName) moamk = os.path.join(wd, 'moa.mk') if not os.path.exists(moamk): return #create a regular job job = moa.job.getJob(wd) #convert moamk with open(moamk) as F: for line in F.readlines(): line = line.strip() if '+=' in line: l.critical("Cannot autoconvert: %s" % line) k, v = line.split('=', 1) job.conf.set(k, v) job.conf.save() l.warning("converted moa.mk - please check %s/config" % cdir) sys.exit()
def fixOld(wd): makefile = os.path.join(wd,'Makefile') if not os.path.exists(makefile): return False with open(makefile) as F: t = F.read() if not 'include $(MOABASE)/template/moa/prepare.mk' in t: return False import shutil l.warning("Old style Makefile trying to upgrade!") t = t.replace('/template/moa/', '/lib/gnumake/') if os.path.exists('%s.old' % makefile): os.unlink('%s.old' % makefile) shutil.move(makefile,'%s.old' % makefile) with open('%s' % makefile, 'w') as F: F.write(t) templateName = t.split('moa_load,')[1].replace(')','').strip() l.warning("Found a makefile with template %s" % templateName) cdir = os.path.join(wd, '.moa') if not os.path.exists(cdir): os.mkdir(cdir) l.warning("wrote an updated Makefile - please check!") with open(os.path.join(cdir, 'template'), 'w') as F: F.write(templateName) moamk = os.path.join(wd, 'moa.mk') if not os.path.exists(moamk): return #create a regular job job = moa.job.getJob(wd) #convert moamk with open(moamk) as F: for line in F.readlines(): line = line.strip() if '+=' in line: l.critical("Cannot autoconvert: %s" % line) k,v = line.split('=',1) job.conf.set(k,v) job.conf.save() l.warning("converted moa.mk - please check %s/config" % cdir) sys.exit()
def loadBackend(self): """ load the backend """ backendName = self.template.backend.value l.debug("attempt to load backend %s" % backendName) try: _moduleName = 'moa.backend.%s' % backendName _module = __import__( _moduleName, globals(), locals(), [_moduleName], -1) l.debug("Successfully Loaded module %s" % _moduleName) except ImportError, e: if str(e) == "No module named %s" % _moduleName: l.critical("Backend %s does not exists" % backendName) l.critical("Error loading backend %s" % backendName) raise sys.exit(-1)
def loadBackend(self): """ load the backend """ backendName = self.template.backend.value l.debug("attempt to load backend %s" % backendName) try: _moduleName = 'moa.backend.%s' % backendName _module = __import__(_moduleName, globals(), locals(), [_moduleName], -1) l.debug("Successfully Loaded module %s" % _moduleName) except ImportError, e: if str(e) == "No module named %s" % _moduleName: l.critical("Backend %s does not exists" % backendName) l.critical("Error loading backend %s" % backendName) raise sys.exit(-1)
def refresh(self): """ refresh / initialize this moajob object """ self._makefile = os.path.join(self.wd, 'Makefile') if not os.path.exists(self._makefile): self.isMoa = False return with open(self._makefile) as F: self._makefileText = F.read() reft = self._reFindTemplate.search(self._makefileText) if not reft: l.critical("Invalidly formatted Makefile - exitting") self.isMoa = True self.template = reft.groups()[0]
def testPlugins(args=[]): global pluginFailures global pluginTests #new style plugin tests plugins = moa.plugins.Plugins() for plugin, testCode in plugins.getAttr('TESTSCRIPT'): #if asking for a single plugin, test only that plugin if args and plugin not in args: continue l.info("Starting new style test of %s" % plugin) testDir = tempfile.mkdtemp() testScript = os.path.join(testDir, 'test.sh') with open(testScript, 'w') as F: F.write(TESTSCRIPTHEADER) F.write(testCode) l.debug("executing test.sh in %s" % testScript) p = subprocess.Popen('bash %s' % testScript, shell=True, cwd = testDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) out, err = p.communicate() rc = p.returncode if rc != 0: l.critical("Errors in plugin test %s (rc %d)" % (plugin, rc)) if out: l.critical("Stdout:\n" + out) if err: l.critical("Stderr:\n" + err) pluginFailures += 1 else: if out: l.debug("Stdout:\n" + out) if err: l.info("Stderr:\n" + err) l.info("Success testing %s (%d lines)" % ( plugin, len(testCode.strip().split("\n")))) pluginTests += 1
def templateConvert(data): args = data['newargs'] if len(args) < 1: l.critical("need to specify a target directory") newTemplateDir = args[0] convert_only = args[1:] if not os.path.exists(newTemplateDir): l.warning("creating directory %s" % newTemplateDir) os.makedirs(newTemplateDir) for job in moa.job.list(): if convert_only and not job in convert_only: continue newTemplateConf = os.path.join(newTemplateDir, '%s.moa' % job) newTemplateFile = os.path.join(newTemplateDir, '%s.mk' % job) l.info("start conversion of %s" % job) wd = moa.job.newTestJob(template=job, title='for conversion') inf = moa.info.info(wd) moaId = inf['moa_id'] inf['backend'] = 'gnumake' del inf['parameter_categories'] inf['commands'] = inf['moa_targets'] inf['commands'].remove('all') del inf['moa_targets'] del inf['all_help'] if inf.has_key('title'): del inf['title'] del inf['moa_files'] if not inf['description'] and \ inf['template_description']: inf['description'] = inf['template_description'] del inf['template_file'] for k in inf.keys(): if 'template_' in k: v = inf[k] del inf[k] inf[k.replace('template_', '')] = v inf['help'] = inf.get('help', {}) for c in inf['commands']: inf['help'][c] = inf['%s_help' % c] del inf['%s_help' % c] inf['gnumake_makefile'] = newTemplateFile #del inf['template_file'] for p in inf.get('parameters', []): par = inf['parameters'][p] del inf['parameters'][p] newParName = p.replace('%s_' % moaId, '') if par.has_key('value'): del par['value'] if par.has_key('cardinality'): del par['cardinality'] par['optional'] = True if par.has_key('mandatory'): par['optional'] = not par['mandatory'] del par['mandatory'] #print p, newParName #for k in par: # print ' -', k, par[k] inf['parameters'][newParName] = par #convert gnumake file oldTemplateFile = os.path.join(MOABASE, 'template', '%s.mk' % job) with open(oldTemplateFile) as F: lines = F.readlines() #first parse on a line by line basis for lineNo, line in enumerate(lines): for m in MAKEFILE_CONVERT_LINES: if line.find(m[0]) == 0: lines[lineNo] = m[1] break text = "".join(lines) #print text #see if there are 'input filesets' #$(call moa_fileset_define_opt,$(moa_id)_input,,Input files for $(moa_id)) rere = re.compile(r'\$\(call moa_fileset_define(.*),(.+),(.*),(.*)\)') for x in rere.finditer(text): fsid = x.groups()[1] fsext = x.groups()[2] fsid = fsid.replace('$(moa_id)', moaId).replace(moaId + '_', '') fsdesc = x.groups()[3].replace('$(moa_id)', moaId) l.info("found fileset for %s" % fsid) if x.groups()[0] == '_opt': fsopt = True else: fsopt = False fs = { 'type': 'input', 'category': 'input', 'help': fsdesc, 'extension': fsext, 'optional': fsopt } if not inf.has_key('filesets'): inf['filesets'] = {} inf['filesets'][fsid] = fs #and now remove all related parameters from the configuration for x in ['_extension', '_glob', '_limit', '_sort', '_dir']: ky = fsid + x if inf['parameters'].has_key(ky): del inf['parameters'][ky] text = rere.sub('', text) # see if there are 'remap' filesets rere = re.compile(r'\$\(call moa_fileset_remap(.*),(.+),(.*),(.*)\)') for x in rere.finditer(text): fsid = x.groups()[2].replace('$(moa_id)', moaId).replace(moaId + '_', '') fssrc = x.groups()[1].replace('$(moa_id)', moaId).replace(moaId + '_', '') fs_target_extension = x.groups()[3] l.info("found a remap fileset: %s to %s" % (fssrc, fsid)) if x.groups()[0] == '_nodir': fs_target_dir = '.' else: fs_target_dir = os.path.join('.', fs_target_extension) fs = { 'type': 'map', 'category': 'output', 'source': fssrc, 'dir': fs_target_dir, 'extension': fs_target_extension } if not inf.has_key('filesets'): inf['filesets'] = {} inf['filesets'][fsid] = fs text = rere.sub('', text) #rere = re.compile(r'(\$\(call.*)') #for x in rere.finditer(text): # l.warning("found $call") # l.warning(x.groups()[0]) for suffix in [ 'title', 'help', 'type', 'define', 'description', 'prerequisites', 'default', 'allowed', 'set', 'simple' ]: rere = re.compile(r'\S+' + suffix + r' ?\+?=.*?(?<!\\)\n', re.S) text = rere.sub('', text) rere = re.compile(r'([\t ]*\n){2,}', re.S) text = rere.sub("\n\n", text) #save & store the new files thisTargetDir = os.path.dirname(newTemplateConf) if not os.path.exists(thisTargetDir): os.makedirs(thisTargetDir) with open(newTemplateConf, 'w') as F: yaml.dump(inf, F) with open(newTemplateFile, 'w') as G: G.write(text)
def createAdhoc(data): """ Create an adhoc job """ wd = data['cwd'] options = data['options'] args = data['newargs'] command = " ".join(args).strip() if not command: command=moa.utils.askUser('adhoc_command:\n>', '') l.info('Parsing command: %s' % command) params = [] mode = None searchGlobs = True if options.mode: mode = options.mode if options.mode == 'simple': searchGlobs = False if not options.mode in ['seq', 'par', 'all', 'simple']: l.critical("Unknown adhoc mode: %s" % options.mode) sys.exit(-1) elif '$<' in command: mode = 'seq' searchGlobs = False elif ('$^' in command) or ('$?' in command): mode = 'all' searchGlobs = False l.warning("Observed '$^' or '$?', setting mode to 'all'") l.warning("Processing all files in one go") #see if we have to look for file globs if not searchGlobs: l.info("No recognizable globs found") else: #it appears to make sense to see if there is a glob in the command refindGlob = re.compile( r"([^ *]+" \ + os.sep \ + ")?([^ *]*\*[^ *]*?)((?:\.[^ .*]+)?)") globs = [] for g in refindGlob.finditer(command): globs.append(g) if globs: globReplace = '$<', '$t' mode = 'seq' if len(globs) > 2: raise Exception("Too many globs ?? I not understand :(") if len(globs) == 2: st1 = _sourceOrTarget(globs[0]) st2 = _sourceOrTarget(globs[1]) if st1 == st2: l.warn("Unsure wich is the source & target glob, assuming:") inGlob,outGlob = globs if st1 == 'source': inGlob,outGlob = globs else: outGlob,inGlob = globs globReplace = '$t', '$<' l.info("Input glob: %s" % inGlob.group()) l.info("Output glob: %s" % outGlob.group()) else: l.info("Input glob: %s" % globs[0].group()) inGlob, outGlob = globs[0], None inD, inG, inE = inGlob.groups() if not inD: inD = "" if not inE: inE = "" l.info(" - set input dir : %s" % inD) l.info(" - set input glob : %s" % inG) l.info(" - set input extension : %s" % inE[1:]) params += ['adhoc_input_dir=%s' % inD] params += ['adhoc_input_glob=%s' % inG] params += ['adhoc_input_extension=%s' % inE[1:]] if outGlob: ouD, ouG, ouE = outGlob.groups() if not ouD: ouD = "" if not ouE: ouE = "" ogg = outGlob.groups() ouG1, ouG2 = ouG.split('*') sed = r"s^\(.*\)%s^%s%s\1%s%s^g" % ( inE.replace('.', '\.'), ouD.replace('/', '/'), ouG.split('*')[0], ouG.split('*')[1], ouE ) l.info(" - set name_sed : %s " % sed) l.info(" - set output dir : %s " % ouD) params += ['adhoc_output_dir=%s' % ouD] params += ['adhoc_name_sed=%s' % sed] #hack the commandline for i in range(len(globs)-1, -1, -1): g = globs[i] command = command[:g.start()] + globReplace[i] + command[g.end():] if not mode: mode = 'simple' if command: l.info(" - set command : %s" % command) params.append('adhoc_process=%s' % command) params.append('adhoc_mode=%s' % mode) l.info(" - set mode : %s" % mode) if mode == 'seq': l.warning("Note: adhoc is running in sequential ('seq') mode. If ") l.warning("you are confident that the individual jobs do not interfere, you might ") l.warning("consider setting adhoc to parallel operation:") l.warning("$ moa set adhoc_mode=par") l.debug('setting parameters %s' % params) job = moa.job.newJob(wd, template='adhoc', title = options.title, force = options.force, parameters=params)
def _func(*args, **kargs): start = time.time() res = func(*args, **kargs) l.critical("executed %s %s" % (func, time.time() - start)) return res
def depfunc(*args, **kwargs): l.critical('Calling deprecated function %s' % func.__name__) l.critical("\n" + "\n".join(traceback.format_stack())) func(*args, **kwargs)
def templateConvert(data): args = data['newargs'] if len(args) < 1: l.critical("need to specify a target directory") newTemplateDir = args[0] convert_only = args[1:] if not os.path.exists(newTemplateDir): l.warning("creating directory %s" % newTemplateDir) os.makedirs(newTemplateDir) for job in moa.job.list(): if convert_only and not job in convert_only: continue newTemplateConf = os.path.join(newTemplateDir, '%s.moa' % job) newTemplateFile = os.path.join(newTemplateDir, '%s.mk' % job) l.info("start conversion of %s" % job) wd = moa.job.newTestJob(template=job, title='for conversion') inf = moa.info.info(wd) moaId = inf['moa_id'] inf['backend'] = 'gnumake' del inf['parameter_categories'] inf['commands'] = inf['moa_targets'] inf['commands'].remove('all') del inf['moa_targets'] del inf['all_help'] if inf.has_key('title'): del inf['title'] del inf['moa_files'] if not inf['description'] and \ inf['template_description']: inf['description'] = inf['template_description'] del inf['template_file'] for k in inf.keys(): if 'template_' in k: v = inf[k] del inf[k] inf[k.replace('template_', '')] = v inf['help'] = inf.get('help', {}) for c in inf['commands']: inf['help'][c] = inf['%s_help' % c] del inf['%s_help' % c] inf['gnumake_makefile'] = newTemplateFile #del inf['template_file'] for p in inf.get('parameters', []): par = inf['parameters'][p] del inf['parameters'][p] newParName = p.replace('%s_' % moaId, '') if par.has_key('value'): del par['value'] if par.has_key('cardinality'): del par['cardinality'] par['optional'] = True if par.has_key('mandatory'): par['optional'] = not par['mandatory'] del par['mandatory'] #print p, newParName #for k in par: # print ' -', k, par[k] inf['parameters'][newParName]= par #convert gnumake file oldTemplateFile = os.path.join(MOABASE, 'template', '%s.mk' % job) with open(oldTemplateFile) as F: lines = F.readlines() #first parse on a line by line basis for lineNo, line in enumerate(lines): for m in MAKEFILE_CONVERT_LINES: if line.find(m[0]) == 0: lines[lineNo] = m[1] break text = "".join(lines) #print text #see if there are 'input filesets' #$(call moa_fileset_define_opt,$(moa_id)_input,,Input files for $(moa_id)) rere = re.compile(r'\$\(call moa_fileset_define(.*),(.+),(.*),(.*)\)') for x in rere.finditer(text): fsid = x.groups()[1] fsext = x.groups()[2] fsid = fsid.replace('$(moa_id)', moaId).replace(moaId + '_','') fsdesc = x.groups()[3].replace('$(moa_id)', moaId) l.info("found fileset for %s" % fsid) if x.groups()[0] == '_opt': fsopt = True else: fsopt = False fs = { 'type' : 'input', 'category' : 'input', 'help' : fsdesc, 'extension' : fsext, 'optional' : fsopt } if not inf.has_key('filesets'): inf['filesets'] = {} inf['filesets'][fsid] = fs #and now remove all related parameters from the configuration for x in ['_extension', '_glob', '_limit', '_sort', '_dir']: ky = fsid + x if inf['parameters'].has_key(ky): del inf['parameters'][ky] text = rere.sub('', text) # see if there are 'remap' filesets rere = re.compile(r'\$\(call moa_fileset_remap(.*),(.+),(.*),(.*)\)') for x in rere.finditer(text): fsid = x.groups()[2].replace('$(moa_id)', moaId).replace(moaId + '_','') fssrc = x.groups()[1].replace('$(moa_id)', moaId).replace(moaId + '_','') fs_target_extension = x.groups()[3] l.info("found a remap fileset: %s to %s" % (fssrc, fsid)) if x.groups()[0] == '_nodir': fs_target_dir = '.' else: fs_target_dir = os.path.join('.', fs_target_extension) fs = {'type' : 'map', 'category' : 'output', 'source' : fssrc, 'dir' : fs_target_dir, 'extension' : fs_target_extension } if not inf.has_key('filesets'): inf['filesets'] = {} inf['filesets'][fsid] = fs text = rere.sub('', text) #rere = re.compile(r'(\$\(call.*)') #for x in rere.finditer(text): # l.warning("found $call") # l.warning(x.groups()[0]) for suffix in ['title', 'help', 'type', 'define', 'description', 'prerequisites', 'default', 'allowed', 'set', 'simple']: rere = re.compile(r'\S+'+suffix+r' ?\+?=.*?(?<!\\)\n', re.S) text = rere.sub('', text) rere = re.compile(r'([\t ]*\n){2,}', re.S) text = rere.sub("\n\n", text) #save & store the new files thisTargetDir = os.path.dirname(newTemplateConf) if not os.path.exists(thisTargetDir): os.makedirs(thisTargetDir) with open(newTemplateConf, 'w') as F: yaml.dump(inf, F) with open(newTemplateFile, 'w') as G: G.write(text)