def templateEdit(data):
"""
Create a Tk interaface to edit template x
"""
args = data['args']
l.warning("start editing %s" % args)
tj = moa.job.newTestJob()
def moaresume(data):
"""
resume a paused job -
"""
cwd = data['cwd']
if not moa.info.status(cwd) == 'paused':
l.warning("Moa process does not seem to be paused!")
sys.exit(-1)
pid = int(open(os.path.join(cwd, 'moa.runlock')).read())
l.warning("Resming job %d" % pid)
os.kill(pid, 18)
def moapause(data):
"""
pause a running job
"""
cwd = data['cwd']
if not moa.info.status(cwd) == 'running':
l.warning("Moa process does not seem to be active!")
sys.exit(-1)
pid = int(open(os.path.join(cwd, 'moa.runlock')).read())
l.warning("Pausing job %d" % pid)
os.kill(pid, 19)
def moakill(data):
"""
kill a running job
"""
cwd = data['cwd']
if not moa.info.status(cwd) == 'running':
l.warning("Moa does not seem to be running!")
sys.exit(-1)
pid = int(open(os.path.join(cwd, 'moa.runlock')).read())
l.critical("killing job %d" % pid)
os.kill(pid, 9)
def fixOld(wd):
makefile = os.path.join(wd,'Makefile')
if not os.path.exists(makefile): return False
with open(makefile) as F:
t = F.read()
if not 'include $(MOABASE)/template/moa/prepare.mk' in t:
return False
import shutil
l.warning("Old style Makefile trying to upgrade!")
t = t.replace('/template/moa/', '/lib/gnumake/')
if os.path.exists('%s.old' % makefile):
os.unlink('%s.old' % makefile)
shutil.move(makefile,'%s.old' % makefile)
with open('%s' % makefile, 'w') as F:
F.write(t)
templateName = t.split('moa_load,')[1].replace(')','').strip()
l.warning("Found a makefile with template %s" % templateName)
cdir = os.path.join(wd, '.moa')
if not os.path.exists(cdir): os.mkdir(cdir)
l.warning("wrote an updated Makefile - please check!")
with open(os.path.join(cdir, 'template'), 'w') as F:
F.write(templateName)
moamk = os.path.join(wd, 'moa.mk')
if not os.path.exists(moamk): return
#create a regular job
job = moa.job.getJob(wd)
#convert moamk
with open(moamk) as F:
for line in F.readlines():
line = line.strip()
if '+=' in line:
l.critical("Cannot autoconvert: %s" % line)
k,v = line.split('=',1)
job.conf.set(k,v)
job.conf.save()
l.warning("converted moa.mk - please check %s/config" % cdir)
sys.exit()
def fixOld(wd):
makefile = os.path.join(wd, 'Makefile')
if not os.path.exists(makefile): return False
with open(makefile) as F:
t = F.read()
if not 'include $(MOABASE)/template/moa/prepare.mk' in t:
return False
import shutil
l.warning("Old style Makefile trying to upgrade!")
t = t.replace('/template/moa/', '/lib/gnumake/')
if os.path.exists('%s.old' % makefile):
os.unlink('%s.old' % makefile)
shutil.move(makefile, '%s.old' % makefile)
with open('%s' % makefile, 'w') as F:
F.write(t)
templateName = t.split('moa_load,')[1].replace(')', '').strip()
l.warning("Found a makefile with template %s" % templateName)
cdir = os.path.join(wd, '.moa')
if not os.path.exists(cdir): os.mkdir(cdir)
l.warning("wrote an updated Makefile - please check!")
with open(os.path.join(cdir, 'template'), 'w') as F:
F.write(templateName)
moamk = os.path.join(wd, 'moa.mk')
if not os.path.exists(moamk): return
#create a regular job
job = moa.job.getJob(wd)
#convert moamk
with open(moamk) as F:
for line in F.readlines():
line = line.strip()
if '+=' in line:
l.critical("Cannot autoconvert: %s" % line)
k, v = line.split('=', 1)
job.conf.set(k, v)
job.conf.save()
l.warning("converted moa.mk - please check %s/config" % cdir)
sys.exit()
def createAdhoc(data):
"""
Create an adhoc job
"""
wd = data['cwd']
options = data['options']
args = data['newargs']
command = " ".join(args).strip()
if not command:
command=moa.utils.askUser('adhoc_command:\n>', '')
l.info('Parsing command: %s' % command)
params = []
mode = None
searchGlobs = True
if options.mode:
mode = options.mode
if options.mode == 'simple': searchGlobs = False
if not options.mode in ['seq', 'par', 'all', 'simple']:
l.critical("Unknown adhoc mode: %s" % options.mode)
sys.exit(-1)
elif '$<' in command:
mode = 'seq'
searchGlobs = False
elif ('$^' in command) or ('$?' in command):
mode = 'all'
searchGlobs = False
l.warning("Observed '$^' or '$?', setting mode to 'all'")
l.warning("Processing all files in one go")
#see if we have to look for file globs
if not searchGlobs:
l.info("No recognizable globs found")
else:
#it appears to make sense to see if there is a glob in the command
refindGlob = re.compile(
r"([^ *]+" \
+ os.sep \
+ ")?([^ *]*\*[^ *]*?)((?:\.[^ .*]+)?)")
globs = []
for g in refindGlob.finditer(command):
globs.append(g)
if globs:
globReplace = '$<', '$t'
mode = 'seq'
if len(globs) > 2:
raise Exception("Too many globs ?? I not understand :(")
if len(globs) == 2:
st1 = _sourceOrTarget(globs[0])
st2 = _sourceOrTarget(globs[1])
if st1 == st2:
l.warn("Unsure wich is the source & target glob, assuming:")
inGlob,outGlob = globs
if st1 == 'source': inGlob,outGlob = globs
else:
outGlob,inGlob = globs
globReplace = '$t', '$<'
l.info("Input glob: %s" % inGlob.group())
l.info("Output glob: %s" % outGlob.group())
else:
l.info("Input glob: %s" % globs[0].group())
inGlob, outGlob = globs[0], None
inD, inG, inE = inGlob.groups()
if not inD: inD = ""
if not inE: inE = ""
l.info(" - set input dir : %s" % inD)
l.info(" - set input glob : %s" % inG)
l.info(" - set input extension : %s" % inE[1:])
params += ['adhoc_input_dir=%s' % inD]
params += ['adhoc_input_glob=%s' % inG]
params += ['adhoc_input_extension=%s' % inE[1:]]
if outGlob:
ouD, ouG, ouE = outGlob.groups()
if not ouD: ouD = ""
if not ouE: ouE = ""
ogg = outGlob.groups()
ouG1, ouG2 = ouG.split('*')
sed = r"s^\(.*\)%s^%s%s\1%s%s^g" % (
inE.replace('.', '\.'),
ouD.replace('/', '/'),
ouG.split('*')[0],
ouG.split('*')[1],
ouE
)
l.info(" - set name_sed : %s " % sed)
l.info(" - set output dir : %s " % ouD)
params += ['adhoc_output_dir=%s' % ouD]
params += ['adhoc_name_sed=%s' % sed]
#hack the commandline
for i in range(len(globs)-1, -1, -1):
g = globs[i]
command = command[:g.start()] + globReplace[i] + command[g.end():]
if not mode:
mode = 'simple'
if command:
l.info(" - set command : %s" % command)
params.append('adhoc_process=%s' % command)
params.append('adhoc_mode=%s' % mode)
l.info(" - set mode : %s" % mode)
if mode == 'seq':
l.warning("Note: adhoc is running in sequential ('seq') mode. If ")
l.warning("you are confident that the individual jobs do not interfere, you might ")
l.warning("consider setting adhoc to parallel operation:")
l.warning("$ moa set adhoc_mode=par")
l.debug('setting parameters %s' % params)
job = moa.job.newJob(wd,
template='adhoc',
title = options.title,
force = options.force,
parameters=params)
def templateConvert(data):
args = data['newargs']
if len(args) < 1:
l.critical("need to specify a target directory")
newTemplateDir = args[0]
convert_only = args[1:]
if not os.path.exists(newTemplateDir):
l.warning("creating directory %s" % newTemplateDir)
os.makedirs(newTemplateDir)
for job in moa.job.list():
if convert_only and not job in convert_only: continue
newTemplateConf = os.path.join(newTemplateDir, '%s.moa' % job)
newTemplateFile = os.path.join(newTemplateDir, '%s.mk' % job)
l.info("start conversion of %s" % job)
wd = moa.job.newTestJob(template=job, title='for conversion')
inf = moa.info.info(wd)
moaId = inf['moa_id']
inf['backend'] = 'gnumake'
del inf['parameter_categories']
inf['commands'] = inf['moa_targets']
inf['commands'].remove('all')
del inf['moa_targets']
del inf['all_help']
if inf.has_key('title'): del inf['title']
del inf['moa_files']
if not inf['description'] and \
inf['template_description']:
inf['description'] = inf['template_description']
del inf['template_file']
for k in inf.keys():
if 'template_' in k:
v = inf[k]
del inf[k]
inf[k.replace('template_', '')] = v
inf['help'] = inf.get('help', {})
for c in inf['commands']:
inf['help'][c] = inf['%s_help' % c]
del inf['%s_help' % c]
inf['gnumake_makefile'] = newTemplateFile
#del inf['template_file']
for p in inf.get('parameters', []):
par = inf['parameters'][p]
del inf['parameters'][p]
newParName = p.replace('%s_' % moaId, '')
if par.has_key('value'): del par['value']
if par.has_key('cardinality'): del par['cardinality']
par['optional'] = True
if par.has_key('mandatory'):
par['optional'] = not par['mandatory']
del par['mandatory']
#print p, newParName
#for k in par:
# print ' -', k, par[k]
inf['parameters'][newParName]= par
#convert gnumake file
oldTemplateFile = os.path.join(MOABASE, 'template', '%s.mk' % job)
with open(oldTemplateFile) as F:
lines = F.readlines()
#first parse on a line by line basis
for lineNo, line in enumerate(lines):
for m in MAKEFILE_CONVERT_LINES:
if line.find(m[0]) == 0:
lines[lineNo] = m[1]
break
text = "".join(lines)
#print text
#see if there are 'input filesets'
#$(call moa_fileset_define_opt,$(moa_id)_input,,Input files for $(moa_id))
rere = re.compile(r'\$\(call moa_fileset_define(.*),(.+),(.*),(.*)\)')
for x in rere.finditer(text):
fsid = x.groups()[1]
fsext = x.groups()[2]
fsid = fsid.replace('$(moa_id)', moaId).replace(moaId + '_','')
fsdesc = x.groups()[3].replace('$(moa_id)', moaId)
l.info("found fileset for %s" % fsid)
if x.groups()[0] == '_opt':
fsopt = True
else:
fsopt = False
fs = { 'type' : 'input',
'category' : 'input',
'help' : fsdesc,
'extension' : fsext,
'optional' : fsopt
}
if not inf.has_key('filesets'):
inf['filesets'] = {}
inf['filesets'][fsid] = fs
#and now remove all related parameters from the configuration
for x in ['_extension', '_glob', '_limit', '_sort', '_dir']:
ky = fsid + x
if inf['parameters'].has_key(ky):
del inf['parameters'][ky]
text = rere.sub('', text)
# see if there are 'remap' filesets
rere = re.compile(r'\$\(call moa_fileset_remap(.*),(.+),(.*),(.*)\)')
for x in rere.finditer(text):
fsid = x.groups()[2].replace('$(moa_id)', moaId).replace(moaId + '_','')
fssrc = x.groups()[1].replace('$(moa_id)', moaId).replace(moaId + '_','')
fs_target_extension = x.groups()[3]
l.info("found a remap fileset: %s to %s" % (fssrc, fsid))
if x.groups()[0] == '_nodir':
fs_target_dir = '.'
else:
fs_target_dir = os.path.join('.', fs_target_extension)
fs = {'type' : 'map',
'category' : 'output',
'source' : fssrc,
'dir' : fs_target_dir,
'extension' : fs_target_extension
}
if not inf.has_key('filesets'):
inf['filesets'] = {}
inf['filesets'][fsid] = fs
text = rere.sub('', text)
#rere = re.compile(r'(\$\(call.*)')
#for x in rere.finditer(text):
# l.warning("found $call")
# l.warning(x.groups()[0])
for suffix in ['title', 'help', 'type', 'define', 'description',
'prerequisites', 'default', 'allowed', 'set', 'simple']:
rere = re.compile(r'\S+'+suffix+r' ?\+?=.*?(?<!\\)\n', re.S)
text = rere.sub('', text)
rere = re.compile(r'([\t ]*\n){2,}', re.S)
text = rere.sub("\n\n", text)
#save & store the new files
thisTargetDir = os.path.dirname(newTemplateConf)
if not os.path.exists(thisTargetDir):
os.makedirs(thisTargetDir)
with open(newTemplateConf, 'w') as F:
yaml.dump(inf, F)
with open(newTemplateFile, 'w') as G:
G.write(text)
def new(self,
template,
title=None,
parameters=[],
force=False,
titleCheck=True,
noInit=False):
"""
Create a new template in the `wd`
"""
l.debug("Creating a new job from template '%s'" % template)
l.debug("- in wd %s" % self.wd)
if not os.path.exists(self.wd):
l.debug("creating folder for %s" % self.wd)
os.makedirs(self.wd)
#TODO: do something with the results of this check
if not moa.template.check(template):
l.error("Invalid template")
if os.path.exists(self.makefile):
l.debug("Makefile exists!")
if not force:
l.error("makefile exists, use -f (--force) to overwrite")
sys.exit(-1)
l.debug("Start writing %s" % self.makefile)
with open(self.makefile, 'w') as F:
F.write(NEW_MAKEFILE_HEADER)
F.write("$(call moa_load,%s)\n" % template)
if title:
self.conf.add('title', title)
params = []
for par in parameters:
if not '=' in par: continue
self.conf.add(par)
self.conf.save()
if noInit: return
l.debug("Running moa initialization")
job = moa.runMake.MOAMAKE(wd=self.wd,
target='initialize',
captureOut=False,
captureErr=False,
stealth=True,
verbose=False)
job.run()
job.finish()
l.debug("Written %s, try: moa help" % self.makefile)
# check if a title is defined as 'title=something' on the
# commandline, as opposed to using the -t option
if not title:
for p in parameters:
if p.find('title=') == 0:
title = p.split('=', 1)[1].strip()
parameters.remove(p)
break
if (not title) and titleCheck and (not template == 'traverse'):
l.warning("You *must* specify a job title")
l.warning("You can still do so by running: ")
l.warning(" moa set title='something descriptive'")
title = ""
if title:
l.debug('creating a new moa makefile with title "%s" in %s' %
(title, self.wd))
else:
l.debug('creating a new moa makefile in %s' % (self.wd))
def templateConvert(data):
args = data['newargs']
if len(args) < 1:
l.critical("need to specify a target directory")
newTemplateDir = args[0]
convert_only = args[1:]
if not os.path.exists(newTemplateDir):
l.warning("creating directory %s" % newTemplateDir)
os.makedirs(newTemplateDir)
for job in moa.job.list():
if convert_only and not job in convert_only: continue
newTemplateConf = os.path.join(newTemplateDir, '%s.moa' % job)
newTemplateFile = os.path.join(newTemplateDir, '%s.mk' % job)
l.info("start conversion of %s" % job)
wd = moa.job.newTestJob(template=job, title='for conversion')
inf = moa.info.info(wd)
moaId = inf['moa_id']
inf['backend'] = 'gnumake'
del inf['parameter_categories']
inf['commands'] = inf['moa_targets']
inf['commands'].remove('all')
del inf['moa_targets']
del inf['all_help']
if inf.has_key('title'): del inf['title']
del inf['moa_files']
if not inf['description'] and \
inf['template_description']:
inf['description'] = inf['template_description']
del inf['template_file']
for k in inf.keys():
if 'template_' in k:
v = inf[k]
del inf[k]
inf[k.replace('template_', '')] = v
inf['help'] = inf.get('help', {})
for c in inf['commands']:
inf['help'][c] = inf['%s_help' % c]
del inf['%s_help' % c]
inf['gnumake_makefile'] = newTemplateFile
#del inf['template_file']
for p in inf.get('parameters', []):
par = inf['parameters'][p]
del inf['parameters'][p]
newParName = p.replace('%s_' % moaId, '')
if par.has_key('value'): del par['value']
if par.has_key('cardinality'): del par['cardinality']
par['optional'] = True
if par.has_key('mandatory'):
par['optional'] = not par['mandatory']
del par['mandatory']
#print p, newParName
#for k in par:
# print ' -', k, par[k]
inf['parameters'][newParName] = par
#convert gnumake file
oldTemplateFile = os.path.join(MOABASE, 'template', '%s.mk' % job)
with open(oldTemplateFile) as F:
lines = F.readlines()
#first parse on a line by line basis
for lineNo, line in enumerate(lines):
for m in MAKEFILE_CONVERT_LINES:
if line.find(m[0]) == 0:
lines[lineNo] = m[1]
break
text = "".join(lines)
#print text
#see if there are 'input filesets'
#$(call moa_fileset_define_opt,$(moa_id)_input,,Input files for $(moa_id))
rere = re.compile(r'\$\(call moa_fileset_define(.*),(.+),(.*),(.*)\)')
for x in rere.finditer(text):
fsid = x.groups()[1]
fsext = x.groups()[2]
fsid = fsid.replace('$(moa_id)', moaId).replace(moaId + '_', '')
fsdesc = x.groups()[3].replace('$(moa_id)', moaId)
l.info("found fileset for %s" % fsid)
if x.groups()[0] == '_opt':
fsopt = True
else:
fsopt = False
fs = {
'type': 'input',
'category': 'input',
'help': fsdesc,
'extension': fsext,
'optional': fsopt
}
if not inf.has_key('filesets'):
inf['filesets'] = {}
inf['filesets'][fsid] = fs
#and now remove all related parameters from the configuration
for x in ['_extension', '_glob', '_limit', '_sort', '_dir']:
ky = fsid + x
if inf['parameters'].has_key(ky):
del inf['parameters'][ky]
text = rere.sub('', text)
# see if there are 'remap' filesets
rere = re.compile(r'\$\(call moa_fileset_remap(.*),(.+),(.*),(.*)\)')
for x in rere.finditer(text):
fsid = x.groups()[2].replace('$(moa_id)',
moaId).replace(moaId + '_', '')
fssrc = x.groups()[1].replace('$(moa_id)',
moaId).replace(moaId + '_', '')
fs_target_extension = x.groups()[3]
l.info("found a remap fileset: %s to %s" % (fssrc, fsid))
if x.groups()[0] == '_nodir':
fs_target_dir = '.'
else:
fs_target_dir = os.path.join('.', fs_target_extension)
fs = {
'type': 'map',
'category': 'output',
'source': fssrc,
'dir': fs_target_dir,
'extension': fs_target_extension
}
if not inf.has_key('filesets'):
inf['filesets'] = {}
inf['filesets'][fsid] = fs
text = rere.sub('', text)
#rere = re.compile(r'(\$\(call.*)')
#for x in rere.finditer(text):
# l.warning("found $call")
# l.warning(x.groups()[0])
for suffix in [
'title', 'help', 'type', 'define', 'description',
'prerequisites', 'default', 'allowed', 'set', 'simple'
]:
rere = re.compile(r'\S+' + suffix + r' ?\+?=.*?(?<!\\)\n', re.S)
text = rere.sub('', text)
rere = re.compile(r'([\t ]*\n){2,}', re.S)
text = rere.sub("\n\n", text)
#save & store the new files
thisTargetDir = os.path.dirname(newTemplateConf)
if not os.path.exists(thisTargetDir):
os.makedirs(thisTargetDir)
with open(newTemplateConf, 'w') as F:
yaml.dump(inf, F)
with open(newTemplateFile, 'w') as G:
G.write(text)
def new(self,
template,
title = None,
parameters = [],
force = False,
titleCheck = True,
noInit = False):
"""
Create a new template in the `wd`
"""
l.debug("Creating a new job from template '%s'" % template)
l.debug("- in wd %s" % self.wd)
if not os.path.exists(self.wd):
l.debug("creating folder for %s" % self.wd)
os.makedirs(self.wd)
#TODO: do something with the results of this check
if not moa.template.check(template):
l.error("Invalid template")
if os.path.exists(self.makefile):
l.debug("Makefile exists!")
if not force:
l.error("makefile exists, use -f (--force) to overwrite")
sys.exit(-1)
l.debug("Start writing %s" % self.makefile)
with open(self.makefile, 'w') as F:
F.write(NEW_MAKEFILE_HEADER)
F.write("$(call moa_load,%s)\n" % template)
if title:
self.conf.add('title', title)
params = []
for par in parameters:
if not '=' in par: continue
self.conf.add(par)
self.conf.save()
if noInit: return
l.debug("Running moa initialization")
job = moa.runMake.MOAMAKE(wd = self.wd,
target='initialize',
captureOut = False,
captureErr = False,
stealth = True,
verbose=False)
job.run()
job.finish()
l.debug("Written %s, try: moa help" % self.makefile)
# check if a title is defined as 'title=something' on the
# commandline, as opposed to using the -t option
if not title:
for p in parameters:
if p.find('title=') == 0:
title = p.split('=',1)[1].strip()
parameters.remove(p)
break
if (not title) and titleCheck and (not template == 'traverse'):
l.warning("You *must* specify a job title")
l.warning("You can still do so by running: ")
l.warning(" moa set title='something descriptive'")
title = ""
if title:
l.debug('creating a new moa makefile with title "%s" in %s' % (
title, self.wd))
else:
l.debug('creating a new moa makefile in %s' % ( self.wd))
