def flogger(*args, **kwargs):
l.critical("Executing %s" % func.__name__)
for a in args:
l.error(" - calling with arg %s" % a)
for k in kwargs.keys():
l.error(" - calling with kwargs %s=%s" % (k, kwargs[k]))
return func(*args, **kwargs)
def flogger(*args, **kwargs):
l.critical("Executing %s" % func.__name__)
for a in args:
l.error(" - calling with arg %s" % a)
for k in kwargs.keys():
l.error(" - calling with kwargs %s=%s" % (k, kwargs[k]))
return func(*args, **kwargs)
def moamv(data):
wd = data['cwd']
options = data['options']
args = data['newargs']
fr = args[0]
if fr[-1] == '/':
fr = fr[:-1]
if len(args) > 1: to = args[1]
else: to = '.'
#see if fr is a number
if re.match('\d+', fr):
newfr = glob.glob('%s*' % fr)
if len(newfr) != 1:
l.critical("Cannot resolve %s (%s)" % (fr, newfr))
sys.exit(1)
fr = newfr[0]
if re.match('\d+', to):
if re.search('^\d+', fr):
to = re.sub('^\d+', to, fr)
else:
to = '%s.%s' % (to, fr)
shutil.move(fr, to)
def _gitInit(data):
"""
Initialize a git repository
"""
global GITREPO
global GITROOT
if GITROOT:
l.critical(
"trying to initialize a git repository wihtin a git repository")
sys.exit(-1)
wd = data['cwd']
GITROOT = wd
GITREPO = git.Repo.init(GITROOT)
l.info("created a git repository at %s" % GITROOT)
info = moa.info.info(wd)
index = GITREPO.index
index.add(info['moa_files'].split())
index.commit('Settin up project "%s"' % info['title'])
#write some data to .gitignore
with open(os.path.join(wd, '.gitignore'), 'w') as F:
F.write(".*\n")
F.write("moa.success\n")
F.write("moa.out\n")
F.write("moa.err\n")
F.write("moa.failed\n")
F.write("moa.runlock\n")
def _gitInit(data):
"""
Initialize a git repository
"""
global GITREPO
global GITROOT
if GITROOT:
l.critical("trying to initialize a git repository wihtin a git repository")
sys.exit(-1)
wd = data['cwd']
GITROOT = wd
GITREPO = git.Repo.init(GITROOT)
l.info("created a git repository at %s" % GITROOT)
info = moa.info.info(wd)
index = GITREPO.index
index.add(info['moa_files'].split())
index.commit('Settin up project "%s"' % info['title'])
#write some data to .gitignore
with open(os.path.join(wd, '.gitignore'), 'w') as F:
F.write(".*\n")
F.write("moa.success\n")
F.write("moa.out\n")
F.write("moa.err\n")
F.write("moa.failed\n")
F.write("moa.runlock\n")
def moakill(data):
"""
kill a running job
"""
cwd = data['cwd']
if not moa.info.status(cwd) == 'running':
l.warning("Moa does not seem to be running!")
sys.exit(-1)
pid = int(open(os.path.join(cwd, 'moa.runlock')).read())
l.critical("killing job %d" % pid)
os.kill(pid, 9)
def fixOld(wd):
makefile = os.path.join(wd, 'Makefile')
if not os.path.exists(makefile): return False
with open(makefile) as F:
t = F.read()
if not 'include $(MOABASE)/template/moa/prepare.mk' in t:
return False
import shutil
l.warning("Old style Makefile trying to upgrade!")
t = t.replace('/template/moa/', '/lib/gnumake/')
if os.path.exists('%s.old' % makefile):
os.unlink('%s.old' % makefile)
shutil.move(makefile, '%s.old' % makefile)
with open('%s' % makefile, 'w') as F:
F.write(t)
templateName = t.split('moa_load,')[1].replace(')', '').strip()
l.warning("Found a makefile with template %s" % templateName)
cdir = os.path.join(wd, '.moa')
if not os.path.exists(cdir): os.mkdir(cdir)
l.warning("wrote an updated Makefile - please check!")
with open(os.path.join(cdir, 'template'), 'w') as F:
F.write(templateName)
moamk = os.path.join(wd, 'moa.mk')
if not os.path.exists(moamk): return
#create a regular job
job = moa.job.getJob(wd)
#convert moamk
with open(moamk) as F:
for line in F.readlines():
line = line.strip()
if '+=' in line:
l.critical("Cannot autoconvert: %s" % line)
k, v = line.split('=', 1)
job.conf.set(k, v)
job.conf.save()
l.warning("converted moa.mk - please check %s/config" % cdir)
sys.exit()
def fixOld(wd):
makefile = os.path.join(wd,'Makefile')
if not os.path.exists(makefile): return False
with open(makefile) as F:
t = F.read()
if not 'include $(MOABASE)/template/moa/prepare.mk' in t:
return False
import shutil
l.warning("Old style Makefile trying to upgrade!")
t = t.replace('/template/moa/', '/lib/gnumake/')
if os.path.exists('%s.old' % makefile):
os.unlink('%s.old' % makefile)
shutil.move(makefile,'%s.old' % makefile)
with open('%s' % makefile, 'w') as F:
F.write(t)
templateName = t.split('moa_load,')[1].replace(')','').strip()
l.warning("Found a makefile with template %s" % templateName)
cdir = os.path.join(wd, '.moa')
if not os.path.exists(cdir): os.mkdir(cdir)
l.warning("wrote an updated Makefile - please check!")
with open(os.path.join(cdir, 'template'), 'w') as F:
F.write(templateName)
moamk = os.path.join(wd, 'moa.mk')
if not os.path.exists(moamk): return
#create a regular job
job = moa.job.getJob(wd)
#convert moamk
with open(moamk) as F:
for line in F.readlines():
line = line.strip()
if '+=' in line:
l.critical("Cannot autoconvert: %s" % line)
k,v = line.split('=',1)
job.conf.set(k,v)
job.conf.save()
l.warning("converted moa.mk - please check %s/config" % cdir)
sys.exit()
def loadBackend(self):
"""
load the backend
"""
backendName = self.template.backend.value
l.debug("attempt to load backend %s" % backendName)
try:
_moduleName = 'moa.backend.%s' % backendName
_module = __import__( _moduleName, globals(),
locals(), [_moduleName], -1)
l.debug("Successfully Loaded module %s" % _moduleName)
except ImportError, e:
if str(e) == "No module named %s" % _moduleName:
l.critical("Backend %s does not exists" % backendName)
l.critical("Error loading backend %s" % backendName)
raise
sys.exit(-1)
def loadBackend(self):
"""
load the backend
"""
backendName = self.template.backend.value
l.debug("attempt to load backend %s" % backendName)
try:
_moduleName = 'moa.backend.%s' % backendName
_module = __import__(_moduleName, globals(), locals(),
[_moduleName], -1)
l.debug("Successfully Loaded module %s" % _moduleName)
except ImportError, e:
if str(e) == "No module named %s" % _moduleName:
l.critical("Backend %s does not exists" % backendName)
l.critical("Error loading backend %s" % backendName)
raise
sys.exit(-1)
def refresh(self):
"""
refresh / initialize this moajob object
"""
self._makefile = os.path.join(self.wd, 'Makefile')
if not os.path.exists(self._makefile):
self.isMoa = False
return
with open(self._makefile) as F:
self._makefileText = F.read()
reft = self._reFindTemplate.search(self._makefileText)
if not reft:
l.critical("Invalidly formatted Makefile - exitting")
self.isMoa = True
self.template = reft.groups()[0]
def testPlugins(args=[]):
global pluginFailures
global pluginTests
#new style plugin tests
plugins = moa.plugins.Plugins()
for plugin, testCode in plugins.getAttr('TESTSCRIPT'):
#if asking for a single plugin, test only that plugin
if args and plugin not in args: continue
l.info("Starting new style test of %s" % plugin)
testDir = tempfile.mkdtemp()
testScript = os.path.join(testDir, 'test.sh')
with open(testScript, 'w') as F:
F.write(TESTSCRIPTHEADER)
F.write(testCode)
l.debug("executing test.sh in %s" % testScript)
p = subprocess.Popen('bash %s' % testScript,
shell=True,
cwd = testDir,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True)
out, err = p.communicate()
rc = p.returncode
if rc != 0:
l.critical("Errors in plugin test %s (rc %d)" % (plugin, rc))
if out: l.critical("Stdout:\n" + out)
if err: l.critical("Stderr:\n" + err)
pluginFailures += 1
else:
if out: l.debug("Stdout:\n" + out)
if err: l.info("Stderr:\n" + err)
l.info("Success testing %s (%d lines)" % (
plugin, len(testCode.strip().split("\n"))))
pluginTests += 1
def templateConvert(data):
args = data['newargs']
if len(args) < 1:
l.critical("need to specify a target directory")
newTemplateDir = args[0]
convert_only = args[1:]
if not os.path.exists(newTemplateDir):
l.warning("creating directory %s" % newTemplateDir)
os.makedirs(newTemplateDir)
for job in moa.job.list():
if convert_only and not job in convert_only: continue
newTemplateConf = os.path.join(newTemplateDir, '%s.moa' % job)
newTemplateFile = os.path.join(newTemplateDir, '%s.mk' % job)
l.info("start conversion of %s" % job)
wd = moa.job.newTestJob(template=job, title='for conversion')
inf = moa.info.info(wd)
moaId = inf['moa_id']
inf['backend'] = 'gnumake'
del inf['parameter_categories']
inf['commands'] = inf['moa_targets']
inf['commands'].remove('all')
del inf['moa_targets']
del inf['all_help']
if inf.has_key('title'): del inf['title']
del inf['moa_files']
if not inf['description'] and \
inf['template_description']:
inf['description'] = inf['template_description']
del inf['template_file']
for k in inf.keys():
if 'template_' in k:
v = inf[k]
del inf[k]
inf[k.replace('template_', '')] = v
inf['help'] = inf.get('help', {})
for c in inf['commands']:
inf['help'][c] = inf['%s_help' % c]
del inf['%s_help' % c]
inf['gnumake_makefile'] = newTemplateFile
#del inf['template_file']
for p in inf.get('parameters', []):
par = inf['parameters'][p]
del inf['parameters'][p]
newParName = p.replace('%s_' % moaId, '')
if par.has_key('value'): del par['value']
if par.has_key('cardinality'): del par['cardinality']
par['optional'] = True
if par.has_key('mandatory'):
par['optional'] = not par['mandatory']
del par['mandatory']
#print p, newParName
#for k in par:
# print ' -', k, par[k]
inf['parameters'][newParName] = par
#convert gnumake file
oldTemplateFile = os.path.join(MOABASE, 'template', '%s.mk' % job)
with open(oldTemplateFile) as F:
lines = F.readlines()
#first parse on a line by line basis
for lineNo, line in enumerate(lines):
for m in MAKEFILE_CONVERT_LINES:
if line.find(m[0]) == 0:
lines[lineNo] = m[1]
break
text = "".join(lines)
#print text
#see if there are 'input filesets'
#$(call moa_fileset_define_opt,$(moa_id)_input,,Input files for $(moa_id))
rere = re.compile(r'\$\(call moa_fileset_define(.*),(.+),(.*),(.*)\)')
for x in rere.finditer(text):
fsid = x.groups()[1]
fsext = x.groups()[2]
fsid = fsid.replace('$(moa_id)', moaId).replace(moaId + '_', '')
fsdesc = x.groups()[3].replace('$(moa_id)', moaId)
l.info("found fileset for %s" % fsid)
if x.groups()[0] == '_opt':
fsopt = True
else:
fsopt = False
fs = {
'type': 'input',
'category': 'input',
'help': fsdesc,
'extension': fsext,
'optional': fsopt
}
if not inf.has_key('filesets'):
inf['filesets'] = {}
inf['filesets'][fsid] = fs
#and now remove all related parameters from the configuration
for x in ['_extension', '_glob', '_limit', '_sort', '_dir']:
ky = fsid + x
if inf['parameters'].has_key(ky):
del inf['parameters'][ky]
text = rere.sub('', text)
# see if there are 'remap' filesets
rere = re.compile(r'\$\(call moa_fileset_remap(.*),(.+),(.*),(.*)\)')
for x in rere.finditer(text):
fsid = x.groups()[2].replace('$(moa_id)',
moaId).replace(moaId + '_', '')
fssrc = x.groups()[1].replace('$(moa_id)',
moaId).replace(moaId + '_', '')
fs_target_extension = x.groups()[3]
l.info("found a remap fileset: %s to %s" % (fssrc, fsid))
if x.groups()[0] == '_nodir':
fs_target_dir = '.'
else:
fs_target_dir = os.path.join('.', fs_target_extension)
fs = {
'type': 'map',
'category': 'output',
'source': fssrc,
'dir': fs_target_dir,
'extension': fs_target_extension
}
if not inf.has_key('filesets'):
inf['filesets'] = {}
inf['filesets'][fsid] = fs
text = rere.sub('', text)
#rere = re.compile(r'(\$\(call.*)')
#for x in rere.finditer(text):
# l.warning("found $call")
# l.warning(x.groups()[0])
for suffix in [
'title', 'help', 'type', 'define', 'description',
'prerequisites', 'default', 'allowed', 'set', 'simple'
]:
rere = re.compile(r'\S+' + suffix + r' ?\+?=.*?(?<!\\)\n', re.S)
text = rere.sub('', text)
rere = re.compile(r'([\t ]*\n){2,}', re.S)
text = rere.sub("\n\n", text)
#save & store the new files
thisTargetDir = os.path.dirname(newTemplateConf)
if not os.path.exists(thisTargetDir):
os.makedirs(thisTargetDir)
with open(newTemplateConf, 'w') as F:
yaml.dump(inf, F)
with open(newTemplateFile, 'w') as G:
G.write(text)
def createAdhoc(data):
"""
Create an adhoc job
"""
wd = data['cwd']
options = data['options']
args = data['newargs']
command = " ".join(args).strip()
if not command:
command=moa.utils.askUser('adhoc_command:\n>', '')
l.info('Parsing command: %s' % command)
params = []
mode = None
searchGlobs = True
if options.mode:
mode = options.mode
if options.mode == 'simple': searchGlobs = False
if not options.mode in ['seq', 'par', 'all', 'simple']:
l.critical("Unknown adhoc mode: %s" % options.mode)
sys.exit(-1)
elif '$<' in command:
mode = 'seq'
searchGlobs = False
elif ('$^' in command) or ('$?' in command):
mode = 'all'
searchGlobs = False
l.warning("Observed '$^' or '$?', setting mode to 'all'")
l.warning("Processing all files in one go")
#see if we have to look for file globs
if not searchGlobs:
l.info("No recognizable globs found")
else:
#it appears to make sense to see if there is a glob in the command
refindGlob = re.compile(
r"([^ *]+" \
+ os.sep \
+ ")?([^ *]*\*[^ *]*?)((?:\.[^ .*]+)?)")
globs = []
for g in refindGlob.finditer(command):
globs.append(g)
if globs:
globReplace = '$<', '$t'
mode = 'seq'
if len(globs) > 2:
raise Exception("Too many globs ?? I not understand :(")
if len(globs) == 2:
st1 = _sourceOrTarget(globs[0])
st2 = _sourceOrTarget(globs[1])
if st1 == st2:
l.warn("Unsure wich is the source & target glob, assuming:")
inGlob,outGlob = globs
if st1 == 'source': inGlob,outGlob = globs
else:
outGlob,inGlob = globs
globReplace = '$t', '$<'
l.info("Input glob: %s" % inGlob.group())
l.info("Output glob: %s" % outGlob.group())
else:
l.info("Input glob: %s" % globs[0].group())
inGlob, outGlob = globs[0], None
inD, inG, inE = inGlob.groups()
if not inD: inD = ""
if not inE: inE = ""
l.info(" - set input dir : %s" % inD)
l.info(" - set input glob : %s" % inG)
l.info(" - set input extension : %s" % inE[1:])
params += ['adhoc_input_dir=%s' % inD]
params += ['adhoc_input_glob=%s' % inG]
params += ['adhoc_input_extension=%s' % inE[1:]]
if outGlob:
ouD, ouG, ouE = outGlob.groups()
if not ouD: ouD = ""
if not ouE: ouE = ""
ogg = outGlob.groups()
ouG1, ouG2 = ouG.split('*')
sed = r"s^\(.*\)%s^%s%s\1%s%s^g" % (
inE.replace('.', '\.'),
ouD.replace('/', '/'),
ouG.split('*')[0],
ouG.split('*')[1],
ouE
)
l.info(" - set name_sed : %s " % sed)
l.info(" - set output dir : %s " % ouD)
params += ['adhoc_output_dir=%s' % ouD]
params += ['adhoc_name_sed=%s' % sed]
#hack the commandline
for i in range(len(globs)-1, -1, -1):
g = globs[i]
command = command[:g.start()] + globReplace[i] + command[g.end():]
if not mode:
mode = 'simple'
if command:
l.info(" - set command : %s" % command)
params.append('adhoc_process=%s' % command)
params.append('adhoc_mode=%s' % mode)
l.info(" - set mode : %s" % mode)
if mode == 'seq':
l.warning("Note: adhoc is running in sequential ('seq') mode. If ")
l.warning("you are confident that the individual jobs do not interfere, you might ")
l.warning("consider setting adhoc to parallel operation:")
l.warning("$ moa set adhoc_mode=par")
l.debug('setting parameters %s' % params)
job = moa.job.newJob(wd,
template='adhoc',
title = options.title,
force = options.force,
parameters=params)
def _func(*args, **kargs):
start = time.time()
res = func(*args, **kargs)
l.critical("executed %s %s" % (func, time.time() - start))
return res
def depfunc(*args, **kwargs):
l.critical('Calling deprecated function %s' % func.__name__)
l.critical("\n" + "\n".join(traceback.format_stack()))
func(*args, **kwargs)
def templateConvert(data):
args = data['newargs']
if len(args) < 1:
l.critical("need to specify a target directory")
newTemplateDir = args[0]
convert_only = args[1:]
if not os.path.exists(newTemplateDir):
l.warning("creating directory %s" % newTemplateDir)
os.makedirs(newTemplateDir)
for job in moa.job.list():
if convert_only and not job in convert_only: continue
newTemplateConf = os.path.join(newTemplateDir, '%s.moa' % job)
newTemplateFile = os.path.join(newTemplateDir, '%s.mk' % job)
l.info("start conversion of %s" % job)
wd = moa.job.newTestJob(template=job, title='for conversion')
inf = moa.info.info(wd)
moaId = inf['moa_id']
inf['backend'] = 'gnumake'
del inf['parameter_categories']
inf['commands'] = inf['moa_targets']
inf['commands'].remove('all')
del inf['moa_targets']
del inf['all_help']
if inf.has_key('title'): del inf['title']
del inf['moa_files']
if not inf['description'] and \
inf['template_description']:
inf['description'] = inf['template_description']
del inf['template_file']
for k in inf.keys():
if 'template_' in k:
v = inf[k]
del inf[k]
inf[k.replace('template_', '')] = v
inf['help'] = inf.get('help', {})
for c in inf['commands']:
inf['help'][c] = inf['%s_help' % c]
del inf['%s_help' % c]
inf['gnumake_makefile'] = newTemplateFile
#del inf['template_file']
for p in inf.get('parameters', []):
par = inf['parameters'][p]
del inf['parameters'][p]
newParName = p.replace('%s_' % moaId, '')
if par.has_key('value'): del par['value']
if par.has_key('cardinality'): del par['cardinality']
par['optional'] = True
if par.has_key('mandatory'):
par['optional'] = not par['mandatory']
del par['mandatory']
#print p, newParName
#for k in par:
# print ' -', k, par[k]
inf['parameters'][newParName]= par
#convert gnumake file
oldTemplateFile = os.path.join(MOABASE, 'template', '%s.mk' % job)
with open(oldTemplateFile) as F:
lines = F.readlines()
#first parse on a line by line basis
for lineNo, line in enumerate(lines):
for m in MAKEFILE_CONVERT_LINES:
if line.find(m[0]) == 0:
lines[lineNo] = m[1]
break
text = "".join(lines)
#print text
#see if there are 'input filesets'
#$(call moa_fileset_define_opt,$(moa_id)_input,,Input files for $(moa_id))
rere = re.compile(r'\$\(call moa_fileset_define(.*),(.+),(.*),(.*)\)')
for x in rere.finditer(text):
fsid = x.groups()[1]
fsext = x.groups()[2]
fsid = fsid.replace('$(moa_id)', moaId).replace(moaId + '_','')
fsdesc = x.groups()[3].replace('$(moa_id)', moaId)
l.info("found fileset for %s" % fsid)
if x.groups()[0] == '_opt':
fsopt = True
else:
fsopt = False
fs = { 'type' : 'input',
'category' : 'input',
'help' : fsdesc,
'extension' : fsext,
'optional' : fsopt
}
if not inf.has_key('filesets'):
inf['filesets'] = {}
inf['filesets'][fsid] = fs
#and now remove all related parameters from the configuration
for x in ['_extension', '_glob', '_limit', '_sort', '_dir']:
ky = fsid + x
if inf['parameters'].has_key(ky):
del inf['parameters'][ky]
text = rere.sub('', text)
# see if there are 'remap' filesets
rere = re.compile(r'\$\(call moa_fileset_remap(.*),(.+),(.*),(.*)\)')
for x in rere.finditer(text):
fsid = x.groups()[2].replace('$(moa_id)', moaId).replace(moaId + '_','')
fssrc = x.groups()[1].replace('$(moa_id)', moaId).replace(moaId + '_','')
fs_target_extension = x.groups()[3]
l.info("found a remap fileset: %s to %s" % (fssrc, fsid))
if x.groups()[0] == '_nodir':
fs_target_dir = '.'
else:
fs_target_dir = os.path.join('.', fs_target_extension)
fs = {'type' : 'map',
'category' : 'output',
'source' : fssrc,
'dir' : fs_target_dir,
'extension' : fs_target_extension
}
if not inf.has_key('filesets'):
inf['filesets'] = {}
inf['filesets'][fsid] = fs
text = rere.sub('', text)
#rere = re.compile(r'(\$\(call.*)')
#for x in rere.finditer(text):
# l.warning("found $call")
# l.warning(x.groups()[0])
for suffix in ['title', 'help', 'type', 'define', 'description',
'prerequisites', 'default', 'allowed', 'set', 'simple']:
rere = re.compile(r'\S+'+suffix+r' ?\+?=.*?(?<!\\)\n', re.S)
text = rere.sub('', text)
rere = re.compile(r'([\t ]*\n){2,}', re.S)
text = rere.sub("\n\n", text)
#save & store the new files
thisTargetDir = os.path.dirname(newTemplateConf)
if not os.path.exists(thisTargetDir):
os.makedirs(thisTargetDir)
with open(newTemplateConf, 'w') as F:
yaml.dump(inf, F)
with open(newTemplateFile, 'w') as G:
G.write(text)
def _func(*args, **kargs):
start = time.time()
res = func(*args, **kargs)
l.critical("executed %s %s" % (func, time.time() - start))
return res
def depfunc(*args, **kwargs):
l.critical('Calling deprecated function %s' % func.__name__)
l.critical("\n" + "\n".join(traceback.format_stack()))
func(*args, **kwargs)
