def execute(self, command, verbose=False, background=False):
## make sure the MOA_THREADS env var is set - this is used from inside
## the Makefiles later threads need to be treated different from the
## other parameters. multi-threaded operation is only allowed in the
## second phase of execution.
self.job.env['MOA_THREADS'] = "%s" % self.job.options.threads
#if moa is not verbose, make should be silent
if not self.job.options.verbose:
self.makeArgs.append('-s')
background = self.job.options.background
l.debug("Calling make for command %s" % command)
actor = self.job.getActor()
#dump the job env into the actor environment (sys env)
actor.setEnv(self.job.env)
#and the job configuration
actor.setEnv(self.job.conf)
cl = ['make', command] + self.makeArgs
l.debug("executing %s" % " ".join(cl))
actor.run(cl, background = background)
def __init__(self, templateName = None):
"""
Initialze the template object, which means:
* Check if the template exists, if not raise an Exception
* Load template info
"""
super(Template, self).__init__()
self.name = 'nojob'
self.backend = 'nojob'
self.valid = False
self.parameters = {}
if templateName:
self.meta.name = templateName
self.meta.templateDataFile = os.path.join(TEMPLATEDIR, '%s.moa' % templateName)
if not os.path.exists(self.meta.templateDataFile):
raise InvalidTemplate()
self.meta.valid = True
self.load(self.meta.templateDataFile)
l.debug("set template to %s, backend %s" % (self.name, self.backend))
def __init__(self, templateName=None):
"""
Initialze the template object, which means:
* Check if the template exists, if not raise an Exception
* Load template info
"""
super(Template, self).__init__()
self.name = "nojob"
self.backend = "nojob"
self.valid = False
self.parameters = {}
if templateName:
self.meta.name = templateName
self.meta.templateDataFile = os.path.join(TEMPLATEDIR, "%s.moa" % templateName)
if not os.path.exists(self.meta.templateDataFile):
raise InvalidTemplate()
self.meta.valid = True
self.load(self.meta.templateDataFile)
l.debug("set template to %s, backend %s" % (self.name, self.backend))
def execute(self, command, verbose=False, background=False):
## make sure the MOA_THREADS env var is set - this is used from inside
## the Makefiles later threads need to be treated different from the
## other parameters. multi-threaded operation is only allowed in the
## second phase of execution.
self.job.env['MOA_THREADS'] = "%s" % self.job.options.threads
#if moa is not verbose, make should be silent
if not self.job.options.verbose:
self.makeArgs.append('-s')
background = self.job.options.background
l.debug("Calling make for command %s" % command)
actor = self.job.getActor()
#dump the job env into the actor environment (sys env)
actor.setEnv(self.job.env)
#and the job configuration
actor.setEnv(self.job.conf)
cl = ['make', command] + self.makeArgs
l.debug("executing %s" % " ".join(cl))
actor.run(cl, background=background)
def postNew(data):
"""
To be executed just after the 'moa new' command
"""
wd = data['cwd']
global GITROOT
global GITREPO
l.debug("running git post new hook")
l.debug('GITROOT %s' % GITROOT)
template = 'traverse'
for a in data['newargs']:
if '=' in a: continue
template = a
if template == 'project':
# creating new project - see if we need to create a
# new repository
return _gitInit(data)
#try to commit this project
if not GITROOT:
#no git repository - not registering any files
return
info = moa.info.info(wd)
index = GITREPO.index
index.add(info['moa_files'].split())
index.commit('Inital commit of job "%s"' % info['title'])
def load(self):
""" Load configuration from disk """
for f in self.meta.configFiles.keys():
fileName = self.meta.configFiles[f]
l.debug("Considering config file %s / %s" % (f, fileName))
if os.path.exists(fileName):
super(Config, self).load(fileName, set_name=f)
def load(self):
""" Load configuration from disk """
for f in self.meta.configFiles.keys():
fileName = self.meta.configFiles[f]
l.debug("Considering config file %s / %s" % (f, fileName))
if os.path.exists(fileName):
super(Config, self).load(fileName, set_name=f)
def postNew(data):
"""
To be executed just after the 'moa new' command
"""
wd = data['cwd']
global GITROOT
global GITREPO
l.debug("running git post new hook")
l.debug('GITROOT %s' % GITROOT)
template = 'traverse'
for a in data['newargs']:
if '=' in a: continue
template = a
if template == 'project':
# creating new project - see if we need to create a
# new repository
return _gitInit(data)
#try to commit this project
if not GITROOT:
#no git repository - not registering any files
return
info = moa.info.info(wd)
index = GITREPO.index
index.add(info['moa_files'].split())
index.commit('Inital commit of job "%s"' % info['title'])
def getOutput(self):
"""
Get the output from a moa run
"""
if not os.path.exists(self.captureOutName):
return ""
l.debug("reading output from %s" % self.captureOutName)
return open(self.captureOutName).read()
def getOutput(self):
"""
Get the output from a moa run
"""
if not os.path.exists(self.captureOutName):
return ""
l.debug("reading output from %s" % self.captureOutName)
return open(self.captureOutName).read()
def setTemplate(self, name):
"""
Set a new template for this job
"""
self.checkConfDir()
self.template = moa.template.Template(name)
with open(os.path.join(self.confDir, 'template'), 'w') as F:
F.write(name)
l.debug('set job in %s to template %s' % (self.wd, name))
def setTemplate(self, name):
"""
Set a new template for this job
"""
self.checkConfDir()
self.template = moa.template.Template(name)
with open(os.path.join(self.confDir, 'template'), 'w') as F:
F.write(name)
l.debug('set job in %s to template %s' % (self.wd, name))
def execute(self, command):
"""
"""
l.debug("executing %s" % command)
if self.backend:
self.backend.execute(command,
verbose=self.options.verbose,
background=self.options.background)
else:
l.error("No backend loaded - cannot execute %s" % command)
def execute(self, command):
"""
"""
l.debug("executing %s" % command)
if self.backend:
self.backend.execute(command,
verbose = self.options.verbose,
background = self.options.background)
else:
l.error("No backend loaded - cannot execute %s" % command)
def getError(self):
"""
Get the stderr of a moa run
@returns: stderr output of this job (if captured)
@rtype: string
"""
if not os.path.exists(self.captureErrName):
return ""
l.debug("reading error from %s" % self.captureErrName)
return open(self.captureErrName).read()
def getError(self):
"""
Get the stderr of a moa run
@returns: stderr output of this job (if captured)
@rtype: string
"""
if not os.path.exists(self.captureErrName):
return ""
l.debug("reading error from %s" % self.captureErrName)
return open(self.captureErrName).read()
def __init__(self, job):
self.job = job
## Determine what plugins are loaded
self.plugins = {}
self.data = {}
self.allPlugins = self.job.conf.getPlugins()
l.debug("Plugins %s" % ", ".join(self.allPlugins))
## load the plugins as seperate modules. A plugin does not need to
self.initialize()
def configUnset(data):
"""
remove variables from the configuration
"""
job = data['job']
for a in data['newargs']:
if '=' in a:
l.error("Invalid argument to unset %s" % a)
else:
l.debug("Unsetting %s" % a)
job.conf.unset(a)
job.conf.save()
def configUnset(data):
"""
remove variables from the configuration
"""
job = data["job"]
for a in data["newargs"]:
if "=" in a:
l.error("Invalid argument to unset %s" % a)
else:
l.debug("Unsetting %s" % a)
job.conf.unset(a)
job.conf.save()
def run2(self, cl):
"""
Really run
"""
self.runStartTime = time.time()
l.debug("Starting actor now in %s" % self.wd)
l.debug(" - with command line: '%s'" % " ".join(cl))
if self.captureOut:
self.FOUT = open(self.captureOutName, 'w')
if self.captureErr:
self.FERR = open(self.captureErrName, 'w')
self.p = subprocess.Popen(cl,
shell=False,
cwd=self.wd,
stdout=self.FOUT,
stderr=self.FERR)
l.debug("execution has started with pid %d" % self.p.pid)
self.pid = self.p.pid
self.out, self.err = self.p.communicate()
self.rc = self.p.returncode
self.runStopTime = time.time()
if self.captureOut:
self.FOUT.close()
if self.captureErr:
self.FERR.close()
l.debug("Finished execution with rc %d " % self.rc)
return self.rc
def run2(self, cl):
"""
Really run
"""
self.runStartTime = time.time()
l.debug("Starting actor now in %s" % self.wd)
l.debug(" - with command line: '%s'" % " ".join(cl))
if self.captureOut:
self.FOUT = open(self.captureOutName, 'w')
if self.captureErr:
self.FERR = open(self.captureErrName, 'w')
self.p = subprocess.Popen(
cl, shell=False, cwd = self.wd,
stdout = self.FOUT, stderr = self.FERR)
l.debug("execution has started with pid %d" % self.p.pid)
self.pid = self.p.pid
self.out, self.err = self.p.communicate()
self.rc = self.p.returncode
self.runStopTime = time.time()
if self.captureOut:
self.FOUT.close()
if self.captureErr:
self.FERR.close()
l.debug("Finished execution with rc %d " % self.rc)
return self.rc
def finish(self):
if self.rc == 0:
l.debug("Succesfully finished make in %s" % (self.wd))
else:
if self.verbose:
l.error("Error running make in %s. Return code %s" %
(self.wd, self.rc))
else:
l.debug("Error running make in %s. Return code %s" %
(self.wd, self.rc))
if self.captureName:
if self.captureOut: os.unlink(self.captureOutName)
if self.captureErr: os.unlink(self.captureErrName)
def postSet(data):
"""
Execute just after setting a parameter
"""
wd = data['cwd']
global GITROOT
global GITREPO
#try to commit this project
if not GITROOT:
#no git repository - not registering any files
return
l.debug("committing postset")
index = GITREPO.index
index.commit('Set %s ' % " ".join(data['newargs']))
def finish(self):
if self.rc == 0:
l.debug("Succesfully finished make in %s" % (self.wd))
else:
if self.verbose:
l.error("Error running make in %s. Return code %s" % (
self.wd, self.rc))
else:
l.debug("Error running make in %s. Return code %s" % (
self.wd, self.rc))
if self.captureName:
if self.captureOut: os.unlink(self.captureOutName)
if self.captureErr: os.unlink(self.captureErrName)
def postSet(data):
"""
Execute just after setting a parameter
"""
wd = data['cwd']
global GITROOT
global GITREPO
#try to commit this project
if not GITROOT:
#no git repository - not registering any files
return
l.debug("committing postset")
index = GITREPO.index
index.commit('Set %s ' % " ".join(data['newargs']))
def __init__(self, wd=None):
"""
Constructor
"""
l.debug("Initializing a GNU Make job")
if not wd:
self.wd = os.getcwd()
else:
self.wd = wd
#This is most probably a makefile job
self.isMoa = False
self.template = None
self.makeArgs = []
self.refresh()
def prepare(data):
#check if we're inside a git repository
if not data.has_key('projectRoot'):
return
wd = data['cwd']
projectRoot = data['projectRoot']
l.debug("seeing if %s is a git repos" % projectRoot)
repo = git.Repo(projectRoot)
global GITROOT
global GITREPO
GITROOT = projectRoot
GITREPO = repo
l.debug("found a git root at %s " % GITROOT)
def loadBackend(self):
"""
load the backend
"""
backendName = self.template.backend.value
l.debug("attempt to load backend %s" % backendName)
try:
_moduleName = 'moa.backend.%s' % backendName
_module = __import__( _moduleName, globals(),
locals(), [_moduleName], -1)
l.debug("Successfully Loaded module %s" % _moduleName)
except ImportError, e:
if str(e) == "No module named %s" % _moduleName:
l.critical("Backend %s does not exists" % backendName)
l.critical("Error loading backend %s" % backendName)
raise
sys.exit(-1)
def prepare(data):
#check if we're inside a git repository
if not data.has_key('projectRoot'):
return
wd = data['cwd']
projectRoot = data['projectRoot']
l.debug("seeing if %s is a git repos" % projectRoot)
repo = git.Repo(projectRoot)
global GITROOT
global GITREPO
GITROOT = projectRoot
GITREPO = repo
l.debug("found a git root at %s " % GITROOT)
def loadBackend(self):
"""
load the backend
"""
backendName = self.template.backend.value
l.debug("attempt to load backend %s" % backendName)
try:
_moduleName = 'moa.backend.%s' % backendName
_module = __import__(_moduleName, globals(), locals(),
[_moduleName], -1)
l.debug("Successfully Loaded module %s" % _moduleName)
except ImportError, e:
if str(e) == "No module named %s" % _moduleName:
l.critical("Backend %s does not exists" % backendName)
l.critical("Error loading backend %s" % backendName)
raise
sys.exit(-1)
def findProjectRoot(job):
"""
Find the project root of a certain moa job
>>> wd = moa.job.newTestJob(template='project',
... title = 'test job creation')
>>> subdir = os.path.join(wd, 'test')
>>> os.mkdir(subdir)
>>> job = moa.job.newJob(subdir, template='traverse', title='test')
>>> result = findProjectRoot(subdir)
>>> result == wd
True
>>> None == findProjectRoot('/usr')
True
@param path: The path for which we're looking for the project root, if
omitted, use the cwd
@type path: string
@returns: a tuple, first value indicates if there is a parent project
the second is the path to the project, the third value is the project
title
@rtype: tuple of a boolean and two strings
"""
l.debug("Finding project root for job in %s" % job.wd)
# start walking up through the tree until we discover
#start walking through the parent tree
lookAt = job
while True:
res = _projectRoot(lookAt)
if res == 'project':
return lookAt
newPath = os.path.split(lookAt.wd)[0]
if newPath == '/':
return None
lookAt = moa.job.getJob(newPath)
# assuming that we're not creating moa jobs in the system root
# if you do want to, I'm not going to cooperate.
if cwd == '/': return None
def findProjectRoot(job):
"""
Find the project root of a certain moa job
>>> wd = moa.job.newTestJob(template='project',
... title = 'test job creation')
>>> subdir = os.path.join(wd, 'test')
>>> os.mkdir(subdir)
>>> job = moa.job.newJob(subdir, template='traverse', title='test')
>>> result = findProjectRoot(subdir)
>>> result == wd
True
>>> None == findProjectRoot('/usr')
True
@param path: The path for which we're looking for the project root, if
omitted, use the cwd
@type path: string
@returns: a tuple, first value indicates if there is a parent project
the second is the path to the project, the third value is the project
title
@rtype: tuple of a boolean and two strings
"""
l.debug("Finding project root for job in %s" % job.wd)
# start walking up through the tree until we discover
#start walking through the parent tree
lookAt = job
while True:
res = _projectRoot(lookAt)
if res == 'project':
return lookAt
newPath = os.path.split(lookAt.wd)[0]
if newPath == '/':
return None
lookAt = moa.job.getJob(newPath)
# assuming that we're not creating moa jobs in the system root
# if you do want to, I'm not going to cooperate.
if cwd == '/': return None
def run(self, cl, background=False):
"""
Start a run
"""
if background:
# try to fork
child = os.fork()
if child != 0:
# This is the parent thread - return
return child
l.debug("Child thread - start executing - start run %s" % child)
#no questions asked - we're saving the output
self.captureOut = True
self.captureErr = True
return self.run2(cl)
def run(self, cl, background = False):
"""
Start a run
"""
if background:
# try to fork
child = os.fork()
if child != 0:
# This is the parent thread - return
return child
l.debug("Child thread - start executing - start run %s" % child)
#no questions asked - we're saving the output
self.captureOut = True
self.captureErr = True
return self.run2(cl)
def isMoa(self):
"""
Is the job directory a Moa directory
currently we assume if there is a moa.mk file
>>> job = Job('/')
>>> job.isMoa()
False
>>> import moa.job
>>> jobdir = moa.job.newTestJob('traverse')
>>> job = Job(jobdir)
>>> job.isMoa()
True
"""
makefile = os.path.join(self.wd, 'Makefile')
if not os.access(makefile, os.R_OK):
#ok, this might be a moa directory, but
#you do not have sufficient permissions
return False
l.debug('isMoaDir: checking %s' % makefile)
if not os.path.exists(makefile):
return False
#we could run make, but that is rather slow just to check if the
#Makefile is a proper Moa Makefile - so, we' quickly read the
#Makefile to get a quick indication
isMoa = False
F = open(makefile)
for line in F.readlines():
if 'MOABASE' in line:
isMoa = True
break
if '$(call moa_load' in line:
isMoa = True
break
F.close()
return isMoa
def isMoa(self):
"""
Is the job directory a Moa directory
currently we assume if there is a moa.mk file
>>> job = Job('/')
>>> job.isMoa()
False
>>> import moa.job
>>> jobdir = moa.job.newTestJob('traverse')
>>> job = Job(jobdir)
>>> job.isMoa()
True
"""
makefile = os.path.join(self.wd, 'Makefile')
if not os.access(makefile, os.R_OK):
#ok, this might be a moa directory, but
#you do not have sufficient permissions
return False
l.debug('isMoaDir: checking %s' % makefile)
if not os.path.exists(makefile):
return False
#we could run make, but that is rather slow just to check if the
#Makefile is a proper Moa Makefile - so, we' quickly read the
#Makefile to get a quick indication
isMoa = False
F = open(makefile)
for line in F.readlines():
if 'MOABASE' in line:
isMoa = True
break
if '$(call moa_load' in line:
isMoa = True
break
F.close()
return isMoa
def __init__(self, wd, options, args):
self.wd = wd
self.options = options
self.args = args
self.job = None
### determine what the command is
self.command = ""
if len(args) > 0:
self.command = args[0]
self.args = args[1:]
l.info('Starting moa command "%s"' % self.command)
### Determine project root (if there is one)
projectRoot = moa.project.findProjectRoot(self.wd)
if projectRoot:
l.debug('Project root is %s' % projectRoot)
os.putenv('MOAPROJECTROOT', projectRoot)
self.refresh()
def __init__(self, wd, options, args):
self.wd = wd
self.options = options
self.args = args
self.job = None
### determine what the command is
self.command = ""
if len(args) > 0:
self.command = args[0]
self.args = args[1:]
l.info('Starting moa command "%s"' % self.command)
### Determine project root (if there is one)
projectRoot = moa.project.findProjectRoot(self.wd)
if projectRoot:
l.debug('Project root is %s' % projectRoot)
os.putenv('MOAPROJECTROOT', projectRoot)
self.refresh()
def get():
"""
Initialize the Moa job
Determine if there is a moa job installed here and if so, what
kind. This is done so to keep the possibility open to have
multiple backends (i.e. possibly replacing make at a certain point
"""
global moaJob
if moaJob:
return moaJob
l.debug("Initalizing Moa job")
if os.path.exists('Makefile'):
moaJob = MOAMAKEFILEJOB()
else:
moaJob = None
return moaJob
def _addPath(wd, pth, tar):
l.info("Checking %s" % pth)
if not moa.info.isMoaDir(pth):
l.info("not Moa: ignoring %s" % pth)
return
if not wd in pth:
raise Exception("Odd path?? %s, its not part of %s" % (pth, wd))
rpth = pth.replace(wd, '.')
l.info("Adding directory %s" % rpth)
jobInfo = moa.info.info(rpth)
l.debug("files: %s" % str(jobInfo['moa_files'].split()))
for f in jobInfo['moa_files'].split():
addFile = os.path.join(rpth, f)
if not os.path.exists(addFile):
continue
l.info("Adding file %s" % addFile)
tar.add(addFile)
def newJob(data):
"""
Create a new job
"""
wd = data['wd']
options = data['options']
args = data['newargs']
params = []
template = 'traverse'
for a in args:
if '=' in a:
params.append(a)
else:
template = a
l.debug("Creating a new '%s' job" % template)
if options.directory:
wd = options.directory
if os.path.exists(os.path.join(
wd, '.moa', 'template')) and \
not options.force:
l.error("Seems that there is already a Moa job in")
l.error(wd)
l.error("")
l.error("use -f to override")
title = options.title
job = moa.job.newJob(wd, template=template)
if title:
job.conf.set('title', title)
for p in params:
k, v = p.split('=', 1)
job.conf.set(k, v)
job.conf.save()
def newJob(data):
"""
Create a new job
"""
wd = data['wd']
options = data['options']
args = data['newargs']
params = []
template = 'traverse'
for a in args:
if '=' in a:
params.append(a)
else:
template = a
l.debug("Creating a new '%s' job" % template)
if options.directory:
wd = options.directory
if os.path.exists(os.path.join(
wd, '.moa', 'template')) and \
not options.force:
l.error("Seems that there is already a Moa job in")
l.error(wd)
l.error("")
l.error("use -f to override")
title = options.title
job = moa.job.newJob(wd, template = template)
if title:
job.conf.set('title', title)
for p in params:
k,v = p.split('=', 1)
job.conf.set(k,v)
job.conf.save()
def _addPath(wd, pth, tar):
l.info("Checking %s" % pth)
if not moa.info.isMoaDir(pth):
l.info("not Moa: ignoring %s" % pth)
return
if not wd in pth:
raise Exception("Odd path?? %s, its not part of %s" %(
pth, wd))
rpth = pth.replace(wd, '.')
l.info("Adding directory %s" % rpth)
jobInfo = moa.info.info(rpth)
l.debug("files: %s" % str(jobInfo['moa_files'].split()))
for f in jobInfo['moa_files'].split():
addFile = os.path.join(rpth, f)
if not os.path.exists(addFile):
continue
l.info("Adding file %s" % addFile)
tar.add(addFile)
def testPlugins(args=[]):
global pluginFailures
global pluginTests
#new style plugin tests
plugins = moa.plugins.Plugins()
for plugin, testCode in plugins.getAttr('TESTSCRIPT'):
#if asking for a single plugin, test only that plugin
if args and plugin not in args: continue
l.info("Starting new style test of %s" % plugin)
testDir = tempfile.mkdtemp()
testScript = os.path.join(testDir, 'test.sh')
with open(testScript, 'w') as F:
F.write(TESTSCRIPTHEADER)
F.write(testCode)
l.debug("executing test.sh in %s" % testScript)
p = subprocess.Popen('bash %s' % testScript,
shell=True,
cwd = testDir,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True)
out, err = p.communicate()
rc = p.returncode
if rc != 0:
l.critical("Errors in plugin test %s (rc %d)" % (plugin, rc))
if out: l.critical("Stdout:\n" + out)
if err: l.critical("Stderr:\n" + err)
pluginFailures += 1
else:
if out: l.debug("Stdout:\n" + out)
if err: l.info("Stderr:\n" + err)
l.info("Success testing %s (%d lines)" % (
plugin, len(testCode.strip().split("\n"))))
pluginTests += 1
def prepareInvocation(self, invocation):
self.invocation = invocation
# Prepare the arguments for a moaMake call
if invocation.options.remake:
self.makeArgs.append('-B')
if invocation.options.makedebug:
self.makeArgs.append('-d')
if invocation.options.threads > 1:
l.debug("Running make with %d threads" % options.threads)
self.makeArgs.append('-j %d' % invocation.options.threads)
## make sure the MOA_THREADS env var is set - this is used from inside
## the Makefiles later threads need to be treated different from the
## other parameters. multi-threaded operation is only allowed in the
## second phase of execution.
os.putenv('MOA_THREADS', "%s" % options.threads)
## Store all arguments in the environment - we might want to have a
## look at them from the Makefiles
os.putenv("MOAARGS", "%s" %
" ".join(["'" + str(x) + "'" for x in args]))
def initialize(self):
"""
attempt to load the python part of the plugins
"""
## do we have a python module??
l.debug('Start plugin init')
for plugin in self.allPlugins:
pyModule = 'moa.plugin.%s' % plugin
try:
_m = __import__( pyModule, globals(), locals(), ['git'], -1)
self.plugins[plugin] = _m
l.debug("Successfully Loaded module %s" % pyModule)
except ImportError, e:
if not str(e) == "No module named %s" % plugin:
raise
l.debug("No python plugin module found for %s" % plugin)
def initialize(self, template=None):
"""
Create a new GnuMake job in the `wd`
"""
l.debug("Creating a new job from template '%s'" % template)
l.debug("- in wd %s" % self.wd)
if not template:
template = self.job.template
if not template.valid:
l.error("Invalid template")
return False
if not template.backend == 'gnumake':
l.error("template backend mismatch")
return False
l.debug("Start writing %s" % self.makefile)
with open(self.makefile, 'w') as F:
F.write(NEW_MAKEFILE_HEADER)
F.write("$(call moa_load,%s)\n" % template)
def initialize(self, template = None):
"""
Create a new GnuMake job in the `wd`
"""
l.debug("Creating a new job from template '%s'" % template)
l.debug("- in wd %s" % self.wd)
if not template:
template = self.job.template
if not template.valid:
l.error("Invalid template")
return False
if not template.backend == 'gnumake':
l.error("template backend mismatch")
return False
l.debug("Start writing %s" % self.makefile)
with open(self.makefile, 'w') as F:
F.write(NEW_MAKEFILE_HEADER)
F.write("$(call moa_load,%s)\n" % template)
def checkTemplate(path):
"""
Run all current template tests - see if any needs updating
All job Makefiles should look exactly alike and have only two
lines of actual Makefile code:
include $(MOABASE)/template/moa/prepare.mk
$(call moa_load,emboss/revseq)
"""
makefilePath = os.path.join(path, 'Makefile')
if not os.path.exists(makefilePath):
l.error("template check failed - cannot find a Makefile")
seenIncludePrepare = False
seenIncludeTemplate = False
superfluousLines = False
template = None
makefile = open(makefilePath)
i = 0
nmf = []
for line in makefile.readlines():
line = line.strip()
i += 1
if not line: continue
if line[0] == '#':
nmf += line
continue
if line == 'include $(MOABASE)/template/moa/prepare.mk':
nmf += line
seenIncludePrepare = True
continue
if 'include $(MOABASE)/template/' in line:
if template:
l.error("Duplicate template include??")
l.error("Please fix this!")
sys.exit(-1)
template = line.replace('include $(MOABASE)/template/', '')
template = template.replace('.mk', '')
l.debug("Discovered template %s" % template)
continue
if '$(call moa_load,' in line:
if template:
l.error("Duplicate template include??")
l.error("Please fix this!")
sys.exit(-1)
seenIncludeTemplate = True
template = line.replace('call moa_load,', '').replace(')',
'').strip()
l.debug("Discovered template %s" % template)
continue
if line in [
'.PHONY: moa_preprocess',
'moa_preprocess:',
'.PHONY: moa_postprocess',
'moa_postprocess:',
'-include moa.mk',
'MOAMK_INCLUDE=done',
'@echo preprocess commands go here',
'@echo Postprocess commands go here..',
]:
continue
superfluousLines = True
l.info("extra code: '%s'" % line)
if seenIncludePrepare and seenIncludeTemplate and (not superfluousLines):
return True
if superfluousLines:
l.error("Deprecated Makefile, discovered extra code")
l.error("Please fix this manually")
sys.exit(-1)
l.error("Deprecated template, fixing. Please check!")
shutil.move(makefilePath, makefilePath + '.old')
moa.job.newJob(template=template, wd=path, noInit=True, titleCheck=False)
sys.exit(0)
def run(self, command):
for p in self.keys():
if not command in dir(self[p]):
continue
l.debug("plugin executing hook %s for %s" % (command, p))
getattr(self[p], command)(self.data)
def renumber(path, fr, to):
"""
Renumber a moa job
>>> import tempfile
>>> emptyDir = tempfile.mkdtemp()
>>> removeFiles(emptyDir, recursive=True)
>>> fromDir = os.path.join(emptyDir, '10.test')
>>> problemDir = os.path.join(emptyDir, '20.problem')
>>> toDir = os.path.join(emptyDir, '20.test')
>>> os.mkdir(os.path.join(emptyDir, '10.test'))
>>> os.path.exists(os.path.join(emptyDir, '10.test'))
True
>>> os.path.exists(toDir)
False
>>> renumber(emptyDir, '10', '20')
>>> os.path.exists(fromDir)
False
>>> os.path.exists(toDir)
True
>>> os.mkdir(problemDir)
>>> renumber(emptyDir, '20', '30')
Traceback (most recent call last):
File '/opt/moa/lib/python/moa/utils.py', line 114, in renumber
raise MoaFileError(fullDir)
MoaFileError: Moa error handling file
@param path: the path to operate in
@type path: String
@param fr: number to rename from
@type fr: String representing a number
@param to: number to rename to
@type to: String representing a number
"""
frDir = None
toDir = None
l.debug("moa ren %s %s" % (fr, to))
for x in os.listdir(path):
if x[0] == '.' : continue
fullDir = os.path.join(path, x)
xsplit = x.split('.')
if xsplit[0] == fr:
if frDir:
l.error("more than one directory starting with %s" % fr)
raise MoaFileError(fullDir)
frDir = fullDir
toDir = os.path.join(path, to + "." + ".".join(xsplit[1:]))
if xsplit[0] == to:
l.error("target directory starting with %s already exists" % to)
raise MoaFileError(fullDir)
if not frDir:
l.error("Cannot find a directory starting with %s" % fr)
raise MoaFileError(path)
if not toDir:
l.error("Cannot find a directory starting with %s" % to)
raise MoaFileError(path)
if not os.path.isdir(frDir):
l.error("%s is not a directory" % frDir)
raise MoaFileError(frDir)
#if not os.path.isdir(toDir):
# l.error("%s is not a directory" % toDir)
# raise MoaFileError(toDir)
l.info("renaming: %s" % (frDir))
l.info(" to: %s" % (toDir))
os.rename(frDir, toDir)
def checkTemplate(path):
"""
Run all current template tests - see if any needs updating
All job Makefiles should look exactly alike and have only two
lines of actual Makefile code:
include $(MOABASE)/template/moa/prepare.mk
$(call moa_load,emboss/revseq)
"""
makefilePath = os.path.join(path, 'Makefile')
if not os.path.exists(makefilePath):
l.error("template check failed - cannot find a Makefile")
seenIncludePrepare = False
seenIncludeTemplate = False
superfluousLines = False
template = None
makefile = open(makefilePath)
i = 0
nmf = []
for line in makefile.readlines():
line = line.strip()
i += 1
if not line: continue
if line[0] == '#':
nmf += line
continue
if line == 'include $(MOABASE)/template/moa/prepare.mk':
nmf += line
seenIncludePrepare = True
continue
if 'include $(MOABASE)/template/' in line:
if template:
l.error("Duplicate template include??")
l.error("Please fix this!")
sys.exit(-1)
template = line.replace('include $(MOABASE)/template/', '')
template = template.replace('.mk', '')
l.debug("Discovered template %s" % template)
continue
if '$(call moa_load,' in line:
if template:
l.error("Duplicate template include??")
l.error("Please fix this!")
sys.exit(-1)
seenIncludeTemplate = True
template = line.replace('call moa_load,', '').replace(')', '').strip()
l.debug("Discovered template %s" % template)
continue
if line in ['.PHONY: moa_preprocess', 'moa_preprocess:',
'.PHONY: moa_postprocess', 'moa_postprocess:',
'-include moa.mk', 'MOAMK_INCLUDE=done',
'@echo preprocess commands go here',
'@echo Postprocess commands go here..',
]:
continue
superfluousLines = True
l.info("extra code: '%s'" % line)
if seenIncludePrepare and seenIncludeTemplate and (not superfluousLines):
return True
if superfluousLines:
l.error("Deprecated Makefile, discovered extra code")
l.error("Please fix this manually")
sys.exit(-1)
l.error("Deprecated template, fixing. Please check!")
shutil.move(makefilePath, makefilePath + '.old')
moa.job.newJob(template=template, wd=path,
noInit=True, titleCheck=False)
sys.exit(0)
def new(self,
template,
title=None,
parameters=[],
force=False,
titleCheck=True,
noInit=False):
"""
Create a new template in the `wd`
"""
l.debug("Creating a new job from template '%s'" % template)
l.debug("- in wd %s" % self.wd)
if not os.path.exists(self.wd):
l.debug("creating folder for %s" % self.wd)
os.makedirs(self.wd)
#TODO: do something with the results of this check
if not moa.template.check(template):
l.error("Invalid template")
if os.path.exists(self.makefile):
l.debug("Makefile exists!")
if not force:
l.error("makefile exists, use -f (--force) to overwrite")
sys.exit(-1)
l.debug("Start writing %s" % self.makefile)
with open(self.makefile, 'w') as F:
F.write(NEW_MAKEFILE_HEADER)
F.write("$(call moa_load,%s)\n" % template)
if title:
self.conf.add('title', title)
params = []
for par in parameters:
if not '=' in par: continue
self.conf.add(par)
self.conf.save()
if noInit: return
l.debug("Running moa initialization")
job = moa.runMake.MOAMAKE(wd=self.wd,
target='initialize',
captureOut=False,
captureErr=False,
stealth=True,
verbose=False)
job.run()
job.finish()
l.debug("Written %s, try: moa help" % self.makefile)
# check if a title is defined as 'title=something' on the
# commandline, as opposed to using the -t option
if not title:
for p in parameters:
if p.find('title=') == 0:
title = p.split('=', 1)[1].strip()
parameters.remove(p)
break
if (not title) and titleCheck and (not template == 'traverse'):
l.warning("You *must* specify a job title")
l.warning("You can still do so by running: ")
l.warning(" moa set title='something descriptive'")
title = ""
if title:
l.debug('creating a new moa makefile with title "%s" in %s' %
(title, self.wd))
else:
l.debug('creating a new moa makefile in %s' % (self.wd))
def createAdhoc(data):
"""
Create an adhoc job
"""
wd = data['cwd']
options = data['options']
args = data['newargs']
command = " ".join(args).strip()
if not command:
command=moa.utils.askUser('adhoc_command:\n>', '')
l.info('Parsing command: %s' % command)
params = []
mode = None
searchGlobs = True
if options.mode:
mode = options.mode
if options.mode == 'simple': searchGlobs = False
if not options.mode in ['seq', 'par', 'all', 'simple']:
l.critical("Unknown adhoc mode: %s" % options.mode)
sys.exit(-1)
elif '$<' in command:
mode = 'seq'
searchGlobs = False
elif ('$^' in command) or ('$?' in command):
mode = 'all'
searchGlobs = False
l.warning("Observed '$^' or '$?', setting mode to 'all'")
l.warning("Processing all files in one go")
#see if we have to look for file globs
if not searchGlobs:
l.info("No recognizable globs found")
else:
#it appears to make sense to see if there is a glob in the command
refindGlob = re.compile(
r"([^ *]+" \
+ os.sep \
+ ")?([^ *]*\*[^ *]*?)((?:\.[^ .*]+)?)")
globs = []
for g in refindGlob.finditer(command):
globs.append(g)
if globs:
globReplace = '$<', '$t'
mode = 'seq'
if len(globs) > 2:
raise Exception("Too many globs ?? I not understand :(")
if len(globs) == 2:
st1 = _sourceOrTarget(globs[0])
st2 = _sourceOrTarget(globs[1])
if st1 == st2:
l.warn("Unsure wich is the source & target glob, assuming:")
inGlob,outGlob = globs
if st1 == 'source': inGlob,outGlob = globs
else:
outGlob,inGlob = globs
globReplace = '$t', '$<'
l.info("Input glob: %s" % inGlob.group())
l.info("Output glob: %s" % outGlob.group())
else:
l.info("Input glob: %s" % globs[0].group())
inGlob, outGlob = globs[0], None
inD, inG, inE = inGlob.groups()
if not inD: inD = ""
if not inE: inE = ""
l.info(" - set input dir : %s" % inD)
l.info(" - set input glob : %s" % inG)
l.info(" - set input extension : %s" % inE[1:])
params += ['adhoc_input_dir=%s' % inD]
params += ['adhoc_input_glob=%s' % inG]
params += ['adhoc_input_extension=%s' % inE[1:]]
if outGlob:
ouD, ouG, ouE = outGlob.groups()
if not ouD: ouD = ""
if not ouE: ouE = ""
ogg = outGlob.groups()
ouG1, ouG2 = ouG.split('*')
sed = r"s^\(.*\)%s^%s%s\1%s%s^g" % (
inE.replace('.', '\.'),
ouD.replace('/', '/'),
ouG.split('*')[0],
ouG.split('*')[1],
ouE
)
l.info(" - set name_sed : %s " % sed)
l.info(" - set output dir : %s " % ouD)
params += ['adhoc_output_dir=%s' % ouD]
params += ['adhoc_name_sed=%s' % sed]
#hack the commandline
for i in range(len(globs)-1, -1, -1):
g = globs[i]
command = command[:g.start()] + globReplace[i] + command[g.end():]
if not mode:
mode = 'simple'
if command:
l.info(" - set command : %s" % command)
params.append('adhoc_process=%s' % command)
params.append('adhoc_mode=%s' % mode)
l.info(" - set mode : %s" % mode)
if mode == 'seq':
l.warning("Note: adhoc is running in sequential ('seq') mode. If ")
l.warning("you are confident that the individual jobs do not interfere, you might ")
l.warning("consider setting adhoc to parallel operation:")
l.warning("$ moa set adhoc_mode=par")
l.debug('setting parameters %s' % params)
job = moa.job.newJob(wd,
template='adhoc',
title = options.title,
force = options.force,
parameters=params)
def renumber(path, fr, to):
"""
Renumber a moa job
>>> import tempfile
>>> emptyDir = tempfile.mkdtemp()
>>> removeFiles(emptyDir, recursive=True)
>>> fromDir = os.path.join(emptyDir, '10.test')
>>> problemDir = os.path.join(emptyDir, '20.problem')
>>> toDir = os.path.join(emptyDir, '20.test')
>>> os.mkdir(os.path.join(emptyDir, '10.test'))
>>> os.path.exists(os.path.join(emptyDir, '10.test'))
True
>>> os.path.exists(toDir)
False
>>> renumber(emptyDir, '10', '20')
>>> os.path.exists(fromDir)
False
>>> os.path.exists(toDir)
True
>>> os.mkdir(problemDir)
>>> renumber(emptyDir, '20', '30')
Traceback (most recent call last):
File '/opt/moa/lib/python/moa/utils.py', line 114, in renumber
raise MoaFileError(fullDir)
MoaFileError: Moa error handling file
@param path: the path to operate in
@type path: String
@param fr: number to rename from
@type fr: String representing a number
@param to: number to rename to
@type to: String representing a number
"""
frDir = None
toDir = None
l.debug("moa ren %s %s" % (fr, to))
for x in os.listdir(path):
if x[0] == '.': continue
fullDir = os.path.join(path, x)
xsplit = x.split('.')
if xsplit[0] == fr:
if frDir:
l.error("more than one directory starting with %s" % fr)
raise MoaFileError(fullDir)
frDir = fullDir
toDir = os.path.join(path, to + "." + ".".join(xsplit[1:]))
if xsplit[0] == to:
l.error("target directory starting with %s already exists" % to)
raise MoaFileError(fullDir)
if not frDir:
l.error("Cannot find a directory starting with %s" % fr)
raise MoaFileError(path)
if not toDir:
l.error("Cannot find a directory starting with %s" % to)
raise MoaFileError(path)
if not os.path.isdir(frDir):
l.error("%s is not a directory" % frDir)
raise MoaFileError(frDir)
#if not os.path.isdir(toDir):
# l.error("%s is not a directory" % toDir)
# raise MoaFileError(toDir)
l.info("renaming: %s" % (frDir))
l.info(" to: %s" % (toDir))
os.rename(frDir, toDir)
