def flogger(*args, **kwargs):
l.critical("Executing %s" % func.__name__)
for a in args:
l.error(" - calling with arg %s" % a)
for k in kwargs.keys():
l.error(" - calling with kwargs %s=%s" % (k, kwargs[k]))
return func(*args, **kwargs)
def flogger(*args, **kwargs):
l.critical("Executing %s" % func.__name__)
for a in args:
l.error(" - calling with arg %s" % a)
for k in kwargs.keys():
l.error(" - calling with kwargs %s=%s" % (k, kwargs[k]))
return func(*args, **kwargs)
def moaDirOrExit(job):
"""
Check if the job resides in a moa directory, if not, exit
with an error message
"""
if not job.isMoa():
l.error("This command must be executed in a Moa job directory")
sys.exit(-1)
def _getPackName(data):
packName, packPath = _getPackName2(data)
if (not data['options'].force) and os.path.exists(packPath):
l.error(('Pack %s exists, use a different name (--pn) or ' %
packName) + 'specify -f to overwrite')
sys.exit(-1)
return packName, packPath
def _getPackName(data):
packName, packPath = _getPackName2(data)
if (not data['options'].force) and os.path.exists(packPath):
l.error(('Pack %s exists, use a different name (--pn) or ' % packName) +
'specify -f to overwrite')
sys.exit(-1)
return packName, packPath
def moaDirOrExit(job):
"""
Check if the job resides in a moa directory, if not, exit
with an error message
"""
if not job.isMoa():
l.error("This command must be executed in a Moa job directory")
sys.exit(-1)
def rmPack(data):
packDir = os.path.join(os.path.expanduser('~'), '.moa', 'packs')
for pack in data['newargs']:
if pack == 'rm': continue
packFile = os.path.join(packDir, '%s.tar.bz2' % pack)
if not os.path.exists(packFile):
l.error("cannot find a pack file for %s" % pack)
l.error("Checked: %s" % packFile)
sys.exit(-1)
os.unlink(packFile)
def execute(self, command):
"""
"""
l.debug("executing %s" % command)
if self.backend:
self.backend.execute(command,
verbose=self.options.verbose,
background=self.options.background)
else:
l.error("No backend loaded - cannot execute %s" % command)
def execute(self, command):
"""
"""
l.debug("executing %s" % command)
if self.backend:
self.backend.execute(command,
verbose = self.options.verbose,
background = self.options.background)
else:
l.error("No backend loaded - cannot execute %s" % command)
def rmPack(data):
packDir = os.path.join(
os.path.expanduser('~'), '.moa', 'packs')
for pack in data['newargs']:
if pack == 'rm': continue
packFile = os.path.join(packDir, '%s.tar.bz2' % pack)
if not os.path.exists(packFile):
l.error("cannot find a pack file for %s" % pack)
l.error("Checked: %s" % packFile)
sys.exit(-1)
os.unlink(packFile)
def configUnset(data):
"""
remove variables from the configuration
"""
job = data["job"]
for a in data["newargs"]:
if "=" in a:
l.error("Invalid argument to unset %s" % a)
else:
l.debug("Unsetting %s" % a)
job.conf.unset(a)
job.conf.save()
def configUnset(data):
"""
remove variables from the configuration
"""
job = data['job']
for a in data['newargs']:
if '=' in a:
l.error("Invalid argument to unset %s" % a)
else:
l.debug("Unsetting %s" % a)
job.conf.unset(a)
job.conf.save()
def finish(self):
if self.rc == 0:
l.debug("Succesfully finished make in %s" % (self.wd))
else:
if self.verbose:
l.error("Error running make in %s. Return code %s" % (
self.wd, self.rc))
else:
l.debug("Error running make in %s. Return code %s" % (
self.wd, self.rc))
if self.captureName:
if self.captureOut: os.unlink(self.captureOutName)
if self.captureErr: os.unlink(self.captureErrName)
def finish(self):
if self.rc == 0:
l.debug("Succesfully finished make in %s" % (self.wd))
else:
if self.verbose:
l.error("Error running make in %s. Return code %s" %
(self.wd, self.rc))
else:
l.debug("Error running make in %s. Return code %s" %
(self.wd, self.rc))
if self.captureName:
if self.captureOut: os.unlink(self.captureOutName)
if self.captureErr: os.unlink(self.captureErrName)
def _func(*args, **kargs):
ch = []
l.error("### CALLING %s" % func.__name__)
for c in traceback.extract_stack()[-8:]:
l.error("### TB %s %s %s" % (c[0].split('/')[-1], c[2], c[1]))
l.error("### ARGS %s " % " ".join(args))
l.error("### KWRG %s" % str(kargs))
res = func(*args, **kargs)
return res
def _func(*args, **kargs):
ch = []
l.error("### CALLING %s" % func.__name__)
for c in traceback.extract_stack()[-8:]:
l.error("### TB %s %s %s" % (
c[0].split('/')[-1],
c[2], c[1]))
l.error("### ARGS %s " % " ".join(args))
l.error("### KWRG %s" % str(kargs))
res = func(*args, **kargs)
return res
def initialize(self, force=False, **kwargs):
"""
Initialize a new job in the current wd
"""
if self.isMoa() and not force:
l.error("A job does already exists in this directory")
l.error("specify -f (--force) to override")
return False
#check if a template is defined - if not, use the job template
if kwargs.has_key('template'):
if type(kwargs['template']) == type("string"):
self.setTemplate(kwargs['template'])
kwargs['template'] = self.template
else:
kwargs['template'] = self.template
self.backend.initialize(**kwargs)
def initialize(self,
force=False,
**kwargs):
"""
Initialize a new job in the current wd
"""
if self.isMoa() and not force:
l.error("A job does already exists in this directory")
l.error("specify -f (--force) to override")
return False
#check if a template is defined - if not, use the job template
if kwargs.has_key('template'):
if type(kwargs['template']) == type("string"):
self.setTemplate(kwargs['template'])
kwargs['template'] = self.template
else:
kwargs['template'] = self.template
self.backend.initialize(**kwargs)
def initialize(self, template=None):
"""
Create a new GnuMake job in the `wd`
"""
l.debug("Creating a new job from template '%s'" % template)
l.debug("- in wd %s" % self.wd)
if not template:
template = self.job.template
if not template.valid:
l.error("Invalid template")
return False
if not template.backend == 'gnumake':
l.error("template backend mismatch")
return False
l.debug("Start writing %s" % self.makefile)
with open(self.makefile, 'w') as F:
F.write(NEW_MAKEFILE_HEADER)
F.write("$(call moa_load,%s)\n" % template)
def showHelp(data):
wd = data['cwd']
options = data['options']
args = data['newargs']
global JENV
JENV = jinja2.Environment(loader=jinja2.FileSystemLoader(
os.path.join(MOABASE, 'doc')))
if not args:
if moa.info.isMoaDir(wd):
pageTemplateHelp(data)
else:
printWelcome()
else:
if args[0] in chapters:
pager(JENV.get_template('markdown/%s.md' % args[0]),
getRenderData(data))
elif args[0] in moa.job.list():
td = moa.job.newTestJob(template=args[0], title='help')
pageTemplateHelp(td, options,args)
else:
l.error("Unknown help page, try: moa help")
sys.exit()
def initialize(self, template = None):
"""
Create a new GnuMake job in the `wd`
"""
l.debug("Creating a new job from template '%s'" % template)
l.debug("- in wd %s" % self.wd)
if not template:
template = self.job.template
if not template.valid:
l.error("Invalid template")
return False
if not template.backend == 'gnumake':
l.error("template backend mismatch")
return False
l.debug("Start writing %s" % self.makefile)
with open(self.makefile, 'w') as F:
F.write(NEW_MAKEFILE_HEADER)
F.write("$(call moa_load,%s)\n" % template)
def newJob(data):
"""
Create a new job
"""
wd = data['wd']
options = data['options']
args = data['newargs']
params = []
template = 'traverse'
for a in args:
if '=' in a:
params.append(a)
else:
template = a
l.debug("Creating a new '%s' job" % template)
if options.directory:
wd = options.directory
if os.path.exists(os.path.join(
wd, '.moa', 'template')) and \
not options.force:
l.error("Seems that there is already a Moa job in")
l.error(wd)
l.error("")
l.error("use -f to override")
title = options.title
job = moa.job.newJob(wd, template = template)
if title:
job.conf.set('title', title)
for p in params:
k,v = p.split('=', 1)
job.conf.set(k,v)
job.conf.save()
def newJob(data):
"""
Create a new job
"""
wd = data['wd']
options = data['options']
args = data['newargs']
params = []
template = 'traverse'
for a in args:
if '=' in a:
params.append(a)
else:
template = a
l.debug("Creating a new '%s' job" % template)
if options.directory:
wd = options.directory
if os.path.exists(os.path.join(
wd, '.moa', 'template')) and \
not options.force:
l.error("Seems that there is already a Moa job in")
l.error(wd)
l.error("")
l.error("use -f to override")
title = options.title
job = moa.job.newJob(wd, template=template)
if title:
job.conf.set('title', title)
for p in params:
k, v = p.split('=', 1)
job.conf.set(k, v)
job.conf.save()
def checkTemplate(path):
"""
Run all current template tests - see if any needs updating
All job Makefiles should look exactly alike and have only two
lines of actual Makefile code:
include $(MOABASE)/template/moa/prepare.mk
$(call moa_load,emboss/revseq)
"""
makefilePath = os.path.join(path, 'Makefile')
if not os.path.exists(makefilePath):
l.error("template check failed - cannot find a Makefile")
seenIncludePrepare = False
seenIncludeTemplate = False
superfluousLines = False
template = None
makefile = open(makefilePath)
i = 0
nmf = []
for line in makefile.readlines():
line = line.strip()
i += 1
if not line: continue
if line[0] == '#':
nmf += line
continue
if line == 'include $(MOABASE)/template/moa/prepare.mk':
nmf += line
seenIncludePrepare = True
continue
if 'include $(MOABASE)/template/' in line:
if template:
l.error("Duplicate template include??")
l.error("Please fix this!")
sys.exit(-1)
template = line.replace('include $(MOABASE)/template/', '')
template = template.replace('.mk', '')
l.debug("Discovered template %s" % template)
continue
if '$(call moa_load,' in line:
if template:
l.error("Duplicate template include??")
l.error("Please fix this!")
sys.exit(-1)
seenIncludeTemplate = True
template = line.replace('call moa_load,', '').replace(')',
'').strip()
l.debug("Discovered template %s" % template)
continue
if line in [
'.PHONY: moa_preprocess',
'moa_preprocess:',
'.PHONY: moa_postprocess',
'moa_postprocess:',
'-include moa.mk',
'MOAMK_INCLUDE=done',
'@echo preprocess commands go here',
'@echo Postprocess commands go here..',
]:
continue
superfluousLines = True
l.info("extra code: '%s'" % line)
if seenIncludePrepare and seenIncludeTemplate and (not superfluousLines):
return True
if superfluousLines:
l.error("Deprecated Makefile, discovered extra code")
l.error("Please fix this manually")
sys.exit(-1)
l.error("Deprecated template, fixing. Please check!")
shutil.move(makefilePath, makefilePath + '.old')
moa.job.newJob(template=template, wd=path, noInit=True, titleCheck=False)
sys.exit(0)
def new(self,
template,
title = None,
parameters = [],
force = False,
titleCheck = True,
noInit = False):
"""
Create a new template in the `wd`
"""
l.debug("Creating a new job from template '%s'" % template)
l.debug("- in wd %s" % self.wd)
if not os.path.exists(self.wd):
l.debug("creating folder for %s" % self.wd)
os.makedirs(self.wd)
#TODO: do something with the results of this check
if not moa.template.check(template):
l.error("Invalid template")
if os.path.exists(self.makefile):
l.debug("Makefile exists!")
if not force:
l.error("makefile exists, use -f (--force) to overwrite")
sys.exit(-1)
l.debug("Start writing %s" % self.makefile)
with open(self.makefile, 'w') as F:
F.write(NEW_MAKEFILE_HEADER)
F.write("$(call moa_load,%s)\n" % template)
if title:
self.conf.add('title', title)
params = []
for par in parameters:
if not '=' in par: continue
self.conf.add(par)
self.conf.save()
if noInit: return
l.debug("Running moa initialization")
job = moa.runMake.MOAMAKE(wd = self.wd,
target='initialize',
captureOut = False,
captureErr = False,
stealth = True,
verbose=False)
job.run()
job.finish()
l.debug("Written %s, try: moa help" % self.makefile)
# check if a title is defined as 'title=something' on the
# commandline, as opposed to using the -t option
if not title:
for p in parameters:
if p.find('title=') == 0:
title = p.split('=',1)[1].strip()
parameters.remove(p)
break
if (not title) and titleCheck and (not template == 'traverse'):
l.warning("You *must* specify a job title")
l.warning("You can still do so by running: ")
l.warning(" moa set title='something descriptive'")
title = ""
if title:
l.debug('creating a new moa makefile with title "%s" in %s' % (
title, self.wd))
else:
l.debug('creating a new moa makefile in %s' % ( self.wd))
def execute(self, command, **kwargs):
"""
Nojob - cannot execute
"""
l.error("cannot execute 'moa %s'" % command)
l.error("Maybe try this in a directory containing a moa job?")
def unpack(data):
args = data['newargs']
packName = args[0]
originalPackname = packName
packFile = _getPackFile(packName)
if not packFile:
l.error("cannot locate a packfile for %s" % packName)
sys.exit(-1)
options = data['options']
if options.directory:
target = options.directory
if not os.path.exists(target):
os.makedirs(target)
else:
target = '.'
if (not options.force) and (moa.info.isMoaDir(target)):
l.error("There is already a moa job in %s" % target)
l.error("Use force (-f) to unpack it here")
sys.exit(-1)
packArgs = _getPackArgs(packFile)
providedArgs = data['newargs'][1:]
if len(packArgs) != len(providedArgs):
l.error("Provided the wrong amount of arguments")
l.error("This packfile needs to be called with the")
l.error("following arguments:")
l.error(" %s" % " ".join(packArgs))
if not options.force: sys.exit(-1)
else:
l.error("continuing anyway (force)")
for a in range(len(providedArgs)):
l.info("setting %s to %s" % (packArgs[a], args[a]))
moa.conf.setVar(target, packArgs[a], args[a])
def unpack(data):
args = data['newargs']
packName = args[0]
originalPackname = packName
packFile = _getPackFile(packName)
if not packFile:
l.error("cannot locate a packfile for %s" % packName)
sys.exit(-1)
options = data['options']
if options.directory:
target = options.directory
if not os.path.exists(target):
os.makedirs(target)
else: target = '.'
if (not options.force) and (moa.info.isMoaDir(target)):
l.error("There is already a moa job in %s" % target)
l.error("Use force (-f) to unpack it here")
sys.exit(-1)
packArgs = _getPackArgs(packFile)
providedArgs = data['newargs'][1:]
if len(packArgs) != len(providedArgs):
l.error("Provided the wrong amount of arguments")
l.error("This packfile needs to be called with the")
l.error("following arguments:")
l.error(" %s" % " ".join(packArgs))
if not options.force: sys.exit(-1)
else:
l.error("continuing anyway (force)")
for a in range(len(providedArgs)):
l.info("setting %s to %s" % (packArgs[a], args[a]))
moa.conf.setVar(target, packArgs[a], args[a])
def checkTemplate(path):
"""
Run all current template tests - see if any needs updating
All job Makefiles should look exactly alike and have only two
lines of actual Makefile code:
include $(MOABASE)/template/moa/prepare.mk
$(call moa_load,emboss/revseq)
"""
makefilePath = os.path.join(path, 'Makefile')
if not os.path.exists(makefilePath):
l.error("template check failed - cannot find a Makefile")
seenIncludePrepare = False
seenIncludeTemplate = False
superfluousLines = False
template = None
makefile = open(makefilePath)
i = 0
nmf = []
for line in makefile.readlines():
line = line.strip()
i += 1
if not line: continue
if line[0] == '#':
nmf += line
continue
if line == 'include $(MOABASE)/template/moa/prepare.mk':
nmf += line
seenIncludePrepare = True
continue
if 'include $(MOABASE)/template/' in line:
if template:
l.error("Duplicate template include??")
l.error("Please fix this!")
sys.exit(-1)
template = line.replace('include $(MOABASE)/template/', '')
template = template.replace('.mk', '')
l.debug("Discovered template %s" % template)
continue
if '$(call moa_load,' in line:
if template:
l.error("Duplicate template include??")
l.error("Please fix this!")
sys.exit(-1)
seenIncludeTemplate = True
template = line.replace('call moa_load,', '').replace(')', '').strip()
l.debug("Discovered template %s" % template)
continue
if line in ['.PHONY: moa_preprocess', 'moa_preprocess:',
'.PHONY: moa_postprocess', 'moa_postprocess:',
'-include moa.mk', 'MOAMK_INCLUDE=done',
'@echo preprocess commands go here',
'@echo Postprocess commands go here..',
]:
continue
superfluousLines = True
l.info("extra code: '%s'" % line)
if seenIncludePrepare and seenIncludeTemplate and (not superfluousLines):
return True
if superfluousLines:
l.error("Deprecated Makefile, discovered extra code")
l.error("Please fix this manually")
sys.exit(-1)
l.error("Deprecated template, fixing. Please check!")
shutil.move(makefilePath, makefilePath + '.old')
moa.job.newJob(template=template, wd=path,
noInit=True, titleCheck=False)
sys.exit(0)
def new(self,
template,
title=None,
parameters=[],
force=False,
titleCheck=True,
noInit=False):
"""
Create a new template in the `wd`
"""
l.debug("Creating a new job from template '%s'" % template)
l.debug("- in wd %s" % self.wd)
if not os.path.exists(self.wd):
l.debug("creating folder for %s" % self.wd)
os.makedirs(self.wd)
#TODO: do something with the results of this check
if not moa.template.check(template):
l.error("Invalid template")
if os.path.exists(self.makefile):
l.debug("Makefile exists!")
if not force:
l.error("makefile exists, use -f (--force) to overwrite")
sys.exit(-1)
l.debug("Start writing %s" % self.makefile)
with open(self.makefile, 'w') as F:
F.write(NEW_MAKEFILE_HEADER)
F.write("$(call moa_load,%s)\n" % template)
if title:
self.conf.add('title', title)
params = []
for par in parameters:
if not '=' in par: continue
self.conf.add(par)
self.conf.save()
if noInit: return
l.debug("Running moa initialization")
job = moa.runMake.MOAMAKE(wd=self.wd,
target='initialize',
captureOut=False,
captureErr=False,
stealth=True,
verbose=False)
job.run()
job.finish()
l.debug("Written %s, try: moa help" % self.makefile)
# check if a title is defined as 'title=something' on the
# commandline, as opposed to using the -t option
if not title:
for p in parameters:
if p.find('title=') == 0:
title = p.split('=', 1)[1].strip()
parameters.remove(p)
break
if (not title) and titleCheck and (not template == 'traverse'):
l.warning("You *must* specify a job title")
l.warning("You can still do so by running: ")
l.warning(" moa set title='something descriptive'")
title = ""
if title:
l.debug('creating a new moa makefile with title "%s" in %s' %
(title, self.wd))
else:
l.debug('creating a new moa makefile in %s' % (self.wd))
def renumber(path, fr, to):
"""
Renumber a moa job
>>> import tempfile
>>> emptyDir = tempfile.mkdtemp()
>>> removeFiles(emptyDir, recursive=True)
>>> fromDir = os.path.join(emptyDir, '10.test')
>>> problemDir = os.path.join(emptyDir, '20.problem')
>>> toDir = os.path.join(emptyDir, '20.test')
>>> os.mkdir(os.path.join(emptyDir, '10.test'))
>>> os.path.exists(os.path.join(emptyDir, '10.test'))
True
>>> os.path.exists(toDir)
False
>>> renumber(emptyDir, '10', '20')
>>> os.path.exists(fromDir)
False
>>> os.path.exists(toDir)
True
>>> os.mkdir(problemDir)
>>> renumber(emptyDir, '20', '30')
Traceback (most recent call last):
File '/opt/moa/lib/python/moa/utils.py', line 114, in renumber
raise MoaFileError(fullDir)
MoaFileError: Moa error handling file
@param path: the path to operate in
@type path: String
@param fr: number to rename from
@type fr: String representing a number
@param to: number to rename to
@type to: String representing a number
"""
frDir = None
toDir = None
l.debug("moa ren %s %s" % (fr, to))
for x in os.listdir(path):
if x[0] == '.': continue
fullDir = os.path.join(path, x)
xsplit = x.split('.')
if xsplit[0] == fr:
if frDir:
l.error("more than one directory starting with %s" % fr)
raise MoaFileError(fullDir)
frDir = fullDir
toDir = os.path.join(path, to + "." + ".".join(xsplit[1:]))
if xsplit[0] == to:
l.error("target directory starting with %s already exists" % to)
raise MoaFileError(fullDir)
if not frDir:
l.error("Cannot find a directory starting with %s" % fr)
raise MoaFileError(path)
if not toDir:
l.error("Cannot find a directory starting with %s" % to)
raise MoaFileError(path)
if not os.path.isdir(frDir):
l.error("%s is not a directory" % frDir)
raise MoaFileError(frDir)
#if not os.path.isdir(toDir):
# l.error("%s is not a directory" % toDir)
# raise MoaFileError(toDir)
l.info("renaming: %s" % (frDir))
l.info(" to: %s" % (toDir))
os.rename(frDir, toDir)
def renumber(path, fr, to):
"""
Renumber a moa job
>>> import tempfile
>>> emptyDir = tempfile.mkdtemp()
>>> removeFiles(emptyDir, recursive=True)
>>> fromDir = os.path.join(emptyDir, '10.test')
>>> problemDir = os.path.join(emptyDir, '20.problem')
>>> toDir = os.path.join(emptyDir, '20.test')
>>> os.mkdir(os.path.join(emptyDir, '10.test'))
>>> os.path.exists(os.path.join(emptyDir, '10.test'))
True
>>> os.path.exists(toDir)
False
>>> renumber(emptyDir, '10', '20')
>>> os.path.exists(fromDir)
False
>>> os.path.exists(toDir)
True
>>> os.mkdir(problemDir)
>>> renumber(emptyDir, '20', '30')
Traceback (most recent call last):
File '/opt/moa/lib/python/moa/utils.py', line 114, in renumber
raise MoaFileError(fullDir)
MoaFileError: Moa error handling file
@param path: the path to operate in
@type path: String
@param fr: number to rename from
@type fr: String representing a number
@param to: number to rename to
@type to: String representing a number
"""
frDir = None
toDir = None
l.debug("moa ren %s %s" % (fr, to))
for x in os.listdir(path):
if x[0] == '.' : continue
fullDir = os.path.join(path, x)
xsplit = x.split('.')
if xsplit[0] == fr:
if frDir:
l.error("more than one directory starting with %s" % fr)
raise MoaFileError(fullDir)
frDir = fullDir
toDir = os.path.join(path, to + "." + ".".join(xsplit[1:]))
if xsplit[0] == to:
l.error("target directory starting with %s already exists" % to)
raise MoaFileError(fullDir)
if not frDir:
l.error("Cannot find a directory starting with %s" % fr)
raise MoaFileError(path)
if not toDir:
l.error("Cannot find a directory starting with %s" % to)
raise MoaFileError(path)
if not os.path.isdir(frDir):
l.error("%s is not a directory" % frDir)
raise MoaFileError(frDir)
#if not os.path.isdir(toDir):
# l.error("%s is not a directory" % toDir)
# raise MoaFileError(toDir)
l.info("renaming: %s" % (frDir))
l.info(" to: %s" % (toDir))
os.rename(frDir, toDir)
