def _gitInit(data):
"""
Initialize a git repository
"""
global GITREPO
global GITROOT
if GITROOT:
l.critical("trying to initialize a git repository wihtin a git repository")
sys.exit(-1)
wd = data['cwd']
GITROOT = wd
GITREPO = git.Repo.init(GITROOT)
l.info("created a git repository at %s" % GITROOT)
info = moa.info.info(wd)
index = GITREPO.index
index.add(info['moa_files'].split())
index.commit('Settin up project "%s"' % info['title'])
#write some data to .gitignore
with open(os.path.join(wd, '.gitignore'), 'w') as F:
F.write(".*\n")
F.write("moa.success\n")
F.write("moa.out\n")
F.write("moa.err\n")
F.write("moa.failed\n")
F.write("moa.runlock\n")
def gitlog(data):
"""
Print a log to screen
"""
global GITREPO
global GITROOT
if not GITREPO:
l.info("noting to report - no repo")
return
tags = {}
for t in GITREPO.tags:
print t.commit
tags[t.commit] = t
for c in GITREPO.iter_commits():
#if str(c) in tags.keys()
t = time.strftime("%d %b %Y %H:%M", time.localtime(c.authored_date))
tag = None
if c in tags.keys():
print " tag| %s" % tags[c]
print "%3s | %s | %s" % (c.count(), t, c.message)
def gitlog(data):
"""
Print a log to screen
"""
global GITREPO
global GITROOT
if not GITREPO:
l.info("noting to report - no repo")
return
tags = {}
for t in GITREPO.tags:
print t.commit
tags[t.commit] = t
for c in GITREPO.iter_commits():
#if str(c) in tags.keys()
t = time.strftime("%d %b %Y %H:%M", time.localtime(c.authored_date))
tag = None
if c in tags.keys():
print " tag| %s" % tags[c]
print "%3s | %s | %s" % (c.count(), t, c.message)
def _gitInit(data):
"""
Initialize a git repository
"""
global GITREPO
global GITROOT
if GITROOT:
l.critical(
"trying to initialize a git repository wihtin a git repository")
sys.exit(-1)
wd = data['cwd']
GITROOT = wd
GITREPO = git.Repo.init(GITROOT)
l.info("created a git repository at %s" % GITROOT)
info = moa.info.info(wd)
index = GITREPO.index
index.add(info['moa_files'].split())
index.commit('Settin up project "%s"' % info['title'])
#write some data to .gitignore
with open(os.path.join(wd, '.gitignore'), 'w') as F:
F.write(".*\n")
F.write("moa.success\n")
F.write("moa.out\n")
F.write("moa.err\n")
F.write("moa.failed\n")
F.write("moa.runlock\n")
def tag(data):
global GITREPO
if not GITREPO:
l.info("no repository is initialized")
return
tagname = data['args'][1]
message = data['options'].gitMessage
l.info('tagging with "%s"' % tagname)
GITREPO.create_tag(tagname, message=message)
def tag(data):
global GITREPO
if not GITREPO:
l.info("no repository is initialized")
return
tagname = data['args'][1]
message = data['options'].gitMessage
l.info('tagging with "%s"' % tagname)
GITREPO.create_tag(tagname, message=message)
def removeDirectory(path):
"""
Remove a directory with contents - no questions asked
>>> import tempfile
>>> tempdir = tempfile.mkdtemp()
>>> os.path.exists(tempdir)
True
>>> os.path.isdir(tempdir)
True
>>> removeDirectory(tempdir)
>>> os.path.exists(tempdir)
False
"""
l.info("Removing %s" % path)
shutil.rmtree(path)
l.info("Finished removing %s" % path)
def removeDirectory(path):
"""
Remove a directory with contents - no questions asked
>>> import tempfile
>>> tempdir = tempfile.mkdtemp()
>>> os.path.exists(tempdir)
True
>>> os.path.isdir(tempdir)
True
>>> removeDirectory(tempdir)
>>> os.path.exists(tempdir)
False
"""
l.info("Removing %s" % path)
shutil.rmtree(path)
l.info("Finished removing %s" % path)
def __init__(self, wd, options, args):
self.wd = wd
self.options = options
self.args = args
self.job = None
### determine what the command is
self.command = ""
if len(args) > 0:
self.command = args[0]
self.args = args[1:]
l.info('Starting moa command "%s"' % self.command)
### Determine project root (if there is one)
projectRoot = moa.project.findProjectRoot(self.wd)
if projectRoot:
l.debug('Project root is %s' % projectRoot)
os.putenv('MOAPROJECTROOT', projectRoot)
self.refresh()
def __init__(self, wd, options, args):
self.wd = wd
self.options = options
self.args = args
self.job = None
### determine what the command is
self.command = ""
if len(args) > 0:
self.command = args[0]
self.args = args[1:]
l.info('Starting moa command "%s"' % self.command)
### Determine project root (if there is one)
projectRoot = moa.project.findProjectRoot(self.wd)
if projectRoot:
l.debug('Project root is %s' % projectRoot)
os.putenv('MOAPROJECTROOT', projectRoot)
self.refresh()
def unpack(data):
args = data['newargs']
packName = args[0]
originalPackname = packName
packFile = _getPackFile(packName)
if not packFile:
l.error("cannot locate a packfile for %s" % packName)
sys.exit(-1)
options = data['options']
if options.directory:
target = options.directory
if not os.path.exists(target):
os.makedirs(target)
else:
target = '.'
if (not options.force) and (moa.info.isMoaDir(target)):
l.error("There is already a moa job in %s" % target)
l.error("Use force (-f) to unpack it here")
sys.exit(-1)
packArgs = _getPackArgs(packFile)
providedArgs = data['newargs'][1:]
if len(packArgs) != len(providedArgs):
l.error("Provided the wrong amount of arguments")
l.error("This packfile needs to be called with the")
l.error("following arguments:")
l.error(" %s" % " ".join(packArgs))
if not options.force: sys.exit(-1)
else:
l.error("continuing anyway (force)")
for a in range(len(providedArgs)):
l.info("setting %s to %s" % (packArgs[a], args[a]))
moa.conf.setVar(target, packArgs[a], args[a])
def unpack(data):
args = data['newargs']
packName = args[0]
originalPackname = packName
packFile = _getPackFile(packName)
if not packFile:
l.error("cannot locate a packfile for %s" % packName)
sys.exit(-1)
options = data['options']
if options.directory:
target = options.directory
if not os.path.exists(target):
os.makedirs(target)
else: target = '.'
if (not options.force) and (moa.info.isMoaDir(target)):
l.error("There is already a moa job in %s" % target)
l.error("Use force (-f) to unpack it here")
sys.exit(-1)
packArgs = _getPackArgs(packFile)
providedArgs = data['newargs'][1:]
if len(packArgs) != len(providedArgs):
l.error("Provided the wrong amount of arguments")
l.error("This packfile needs to be called with the")
l.error("following arguments:")
l.error(" %s" % " ".join(packArgs))
if not options.force: sys.exit(-1)
else:
l.error("continuing anyway (force)")
for a in range(len(providedArgs)):
l.info("setting %s to %s" % (packArgs[a], args[a]))
moa.conf.setVar(target, packArgs[a], args[a])
def _addPath(wd, pth, tar):
l.info("Checking %s" % pth)
if not moa.info.isMoaDir(pth):
l.info("not Moa: ignoring %s" % pth)
return
if not wd in pth:
raise Exception("Odd path?? %s, its not part of %s" % (pth, wd))
rpth = pth.replace(wd, '.')
l.info("Adding directory %s" % rpth)
jobInfo = moa.info.info(rpth)
l.debug("files: %s" % str(jobInfo['moa_files'].split()))
for f in jobInfo['moa_files'].split():
addFile = os.path.join(rpth, f)
if not os.path.exists(addFile):
continue
l.info("Adding file %s" % addFile)
tar.add(addFile)
def _addPath(wd, pth, tar):
l.info("Checking %s" % pth)
if not moa.info.isMoaDir(pth):
l.info("not Moa: ignoring %s" % pth)
return
if not wd in pth:
raise Exception("Odd path?? %s, its not part of %s" %(
pth, wd))
rpth = pth.replace(wd, '.')
l.info("Adding directory %s" % rpth)
jobInfo = moa.info.info(rpth)
l.debug("files: %s" % str(jobInfo['moa_files'].split()))
for f in jobInfo['moa_files'].split():
addFile = os.path.join(rpth, f)
if not os.path.exists(addFile):
continue
l.info("Adding file %s" % addFile)
tar.add(addFile)
def moacp(data):
"""
Copy a moa job -
0 create a new directory
1 copy the configuration
TODO: adapt file & dir links
"""
wd = data['cwd']
options = data['options']
args = data['newargs']
if len(args) > 1: dirto = args[1]
else: dirto = '.'
dirfrom = args[0]
#remove trailing slash & determine basename
if dirfrom[-1] == '/': dirfrom = dirfrom[:-1]
fromBase = os.path.basename(dirfrom)
# trick - the second argument is a number
# renumber the target directory
if re.match("^[0-9]+$", dirto):
dirto = re.sub("^[0-9]*\.", dirto + '.', fromBase)
if not os.path.exists(dirto):
l.info("creating directory %s" % dirto)
os.makedirs(dirto)
else:
dirto = os.path.join(dirto, fromBase)
os.makedirs(dirto)
l.info("Copying from %s to %s" % (dirfrom, dirto))
for f in ['Makefile', 'moa.mk']:
cfr = os.path.join(dirfrom, f)
cto = os.path.join(dirto, f)
l.info("copy %s to %s" % (cfr, cto))
shutil.copyfile(cfr, cto)
def testPlugins(args=[]):
global pluginFailures
global pluginTests
#new style plugin tests
plugins = moa.plugins.Plugins()
for plugin, testCode in plugins.getAttr('TESTSCRIPT'):
#if asking for a single plugin, test only that plugin
if args and plugin not in args: continue
l.info("Starting new style test of %s" % plugin)
testDir = tempfile.mkdtemp()
testScript = os.path.join(testDir, 'test.sh')
with open(testScript, 'w') as F:
F.write(TESTSCRIPTHEADER)
F.write(testCode)
l.debug("executing test.sh in %s" % testScript)
p = subprocess.Popen('bash %s' % testScript,
shell=True,
cwd = testDir,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True)
out, err = p.communicate()
rc = p.returncode
if rc != 0:
l.critical("Errors in plugin test %s (rc %d)" % (plugin, rc))
if out: l.critical("Stdout:\n" + out)
if err: l.critical("Stderr:\n" + err)
pluginFailures += 1
else:
if out: l.debug("Stdout:\n" + out)
if err: l.info("Stderr:\n" + err)
l.info("Success testing %s (%d lines)" % (
plugin, len(testCode.strip().split("\n"))))
pluginTests += 1
def pack(data):
"""
Create an adhoc job
"""
wd = data['cwd']
if wd[-1] == '/': wd = wd[:-1]
options = data['options']
args = data['newargs']
#see if we're calling a pack command
if args:
if args[0] == 'unpack': return unpack(data)
elif args[0] == 'list' : return listPacks(data)
elif args[0] == 'rm' : return rmPack(data)
elif args[0] == 'args' : return packArgs(data)
elif args[0] == 'create': args = args[1:]
packName, packPath = _getPackName(data)
l.info('start creating pack in %s' % wd)
l.info("pack name: %s" % packName)
l.info("pack path: %s" % packPath)
TF = tarfile.open(
name = packPath,
mode = 'w:bz2')
if args:
l.info("packing with args %s" % args)
packArgsFile = os.path.join(wd, 'moa.packargs')
with open(packArgsFile, 'w') as F:
F.write(" ".join(args))
l.info("adding %s" % packArgsFile)
TF.add(packArgsFile, os.path.basename(packArgsFile))
os.unlink(packArgsFile)
if options.packRecursive:
for dirpath, dirnames, filenames in os.walk(wd):
_addPath(wd, dirpath, TF)
else:
_addPath(wd, wd, TF)
l.info("done packing")
TF.close()
def pack(data):
"""
Create an adhoc job
"""
wd = data['cwd']
if wd[-1] == '/': wd = wd[:-1]
options = data['options']
args = data['newargs']
#see if we're calling a pack command
if args:
if args[0] == 'unpack': return unpack(data)
elif args[0] == 'list': return listPacks(data)
elif args[0] == 'rm': return rmPack(data)
elif args[0] == 'args': return packArgs(data)
elif args[0] == 'create': args = args[1:]
packName, packPath = _getPackName(data)
l.info('start creating pack in %s' % wd)
l.info("pack name: %s" % packName)
l.info("pack path: %s" % packPath)
TF = tarfile.open(name=packPath, mode='w:bz2')
if args:
l.info("packing with args %s" % args)
packArgsFile = os.path.join(wd, 'moa.packargs')
with open(packArgsFile, 'w') as F:
F.write(" ".join(args))
l.info("adding %s" % packArgsFile)
TF.add(packArgsFile, os.path.basename(packArgsFile))
os.unlink(packArgsFile)
if options.packRecursive:
for dirpath, dirnames, filenames in os.walk(wd):
_addPath(wd, dirpath, TF)
else:
_addPath(wd, wd, TF)
l.info("done packing")
TF.close()
def templateConvert(data):
args = data['newargs']
if len(args) < 1:
l.critical("need to specify a target directory")
newTemplateDir = args[0]
convert_only = args[1:]
if not os.path.exists(newTemplateDir):
l.warning("creating directory %s" % newTemplateDir)
os.makedirs(newTemplateDir)
for job in moa.job.list():
if convert_only and not job in convert_only: continue
newTemplateConf = os.path.join(newTemplateDir, '%s.moa' % job)
newTemplateFile = os.path.join(newTemplateDir, '%s.mk' % job)
l.info("start conversion of %s" % job)
wd = moa.job.newTestJob(template=job, title='for conversion')
inf = moa.info.info(wd)
moaId = inf['moa_id']
inf['backend'] = 'gnumake'
del inf['parameter_categories']
inf['commands'] = inf['moa_targets']
inf['commands'].remove('all')
del inf['moa_targets']
del inf['all_help']
if inf.has_key('title'): del inf['title']
del inf['moa_files']
if not inf['description'] and \
inf['template_description']:
inf['description'] = inf['template_description']
del inf['template_file']
for k in inf.keys():
if 'template_' in k:
v = inf[k]
del inf[k]
inf[k.replace('template_', '')] = v
inf['help'] = inf.get('help', {})
for c in inf['commands']:
inf['help'][c] = inf['%s_help' % c]
del inf['%s_help' % c]
inf['gnumake_makefile'] = newTemplateFile
#del inf['template_file']
for p in inf.get('parameters', []):
par = inf['parameters'][p]
del inf['parameters'][p]
newParName = p.replace('%s_' % moaId, '')
if par.has_key('value'): del par['value']
if par.has_key('cardinality'): del par['cardinality']
par['optional'] = True
if par.has_key('mandatory'):
par['optional'] = not par['mandatory']
del par['mandatory']
#print p, newParName
#for k in par:
# print ' -', k, par[k]
inf['parameters'][newParName]= par
#convert gnumake file
oldTemplateFile = os.path.join(MOABASE, 'template', '%s.mk' % job)
with open(oldTemplateFile) as F:
lines = F.readlines()
#first parse on a line by line basis
for lineNo, line in enumerate(lines):
for m in MAKEFILE_CONVERT_LINES:
if line.find(m[0]) == 0:
lines[lineNo] = m[1]
break
text = "".join(lines)
#print text
#see if there are 'input filesets'
#$(call moa_fileset_define_opt,$(moa_id)_input,,Input files for $(moa_id))
rere = re.compile(r'\$\(call moa_fileset_define(.*),(.+),(.*),(.*)\)')
for x in rere.finditer(text):
fsid = x.groups()[1]
fsext = x.groups()[2]
fsid = fsid.replace('$(moa_id)', moaId).replace(moaId + '_','')
fsdesc = x.groups()[3].replace('$(moa_id)', moaId)
l.info("found fileset for %s" % fsid)
if x.groups()[0] == '_opt':
fsopt = True
else:
fsopt = False
fs = { 'type' : 'input',
'category' : 'input',
'help' : fsdesc,
'extension' : fsext,
'optional' : fsopt
}
if not inf.has_key('filesets'):
inf['filesets'] = {}
inf['filesets'][fsid] = fs
#and now remove all related parameters from the configuration
for x in ['_extension', '_glob', '_limit', '_sort', '_dir']:
ky = fsid + x
if inf['parameters'].has_key(ky):
del inf['parameters'][ky]
text = rere.sub('', text)
# see if there are 'remap' filesets
rere = re.compile(r'\$\(call moa_fileset_remap(.*),(.+),(.*),(.*)\)')
for x in rere.finditer(text):
fsid = x.groups()[2].replace('$(moa_id)', moaId).replace(moaId + '_','')
fssrc = x.groups()[1].replace('$(moa_id)', moaId).replace(moaId + '_','')
fs_target_extension = x.groups()[3]
l.info("found a remap fileset: %s to %s" % (fssrc, fsid))
if x.groups()[0] == '_nodir':
fs_target_dir = '.'
else:
fs_target_dir = os.path.join('.', fs_target_extension)
fs = {'type' : 'map',
'category' : 'output',
'source' : fssrc,
'dir' : fs_target_dir,
'extension' : fs_target_extension
}
if not inf.has_key('filesets'):
inf['filesets'] = {}
inf['filesets'][fsid] = fs
text = rere.sub('', text)
#rere = re.compile(r'(\$\(call.*)')
#for x in rere.finditer(text):
# l.warning("found $call")
# l.warning(x.groups()[0])
for suffix in ['title', 'help', 'type', 'define', 'description',
'prerequisites', 'default', 'allowed', 'set', 'simple']:
rere = re.compile(r'\S+'+suffix+r' ?\+?=.*?(?<!\\)\n', re.S)
text = rere.sub('', text)
rere = re.compile(r'([\t ]*\n){2,}', re.S)
text = rere.sub("\n\n", text)
#save & store the new files
thisTargetDir = os.path.dirname(newTemplateConf)
if not os.path.exists(thisTargetDir):
os.makedirs(thisTargetDir)
with open(newTemplateConf, 'w') as F:
yaml.dump(inf, F)
with open(newTemplateFile, 'w') as G:
G.write(text)
def templateConvert(data):
args = data['newargs']
if len(args) < 1:
l.critical("need to specify a target directory")
newTemplateDir = args[0]
convert_only = args[1:]
if not os.path.exists(newTemplateDir):
l.warning("creating directory %s" % newTemplateDir)
os.makedirs(newTemplateDir)
for job in moa.job.list():
if convert_only and not job in convert_only: continue
newTemplateConf = os.path.join(newTemplateDir, '%s.moa' % job)
newTemplateFile = os.path.join(newTemplateDir, '%s.mk' % job)
l.info("start conversion of %s" % job)
wd = moa.job.newTestJob(template=job, title='for conversion')
inf = moa.info.info(wd)
moaId = inf['moa_id']
inf['backend'] = 'gnumake'
del inf['parameter_categories']
inf['commands'] = inf['moa_targets']
inf['commands'].remove('all')
del inf['moa_targets']
del inf['all_help']
if inf.has_key('title'): del inf['title']
del inf['moa_files']
if not inf['description'] and \
inf['template_description']:
inf['description'] = inf['template_description']
del inf['template_file']
for k in inf.keys():
if 'template_' in k:
v = inf[k]
del inf[k]
inf[k.replace('template_', '')] = v
inf['help'] = inf.get('help', {})
for c in inf['commands']:
inf['help'][c] = inf['%s_help' % c]
del inf['%s_help' % c]
inf['gnumake_makefile'] = newTemplateFile
#del inf['template_file']
for p in inf.get('parameters', []):
par = inf['parameters'][p]
del inf['parameters'][p]
newParName = p.replace('%s_' % moaId, '')
if par.has_key('value'): del par['value']
if par.has_key('cardinality'): del par['cardinality']
par['optional'] = True
if par.has_key('mandatory'):
par['optional'] = not par['mandatory']
del par['mandatory']
#print p, newParName
#for k in par:
# print ' -', k, par[k]
inf['parameters'][newParName] = par
#convert gnumake file
oldTemplateFile = os.path.join(MOABASE, 'template', '%s.mk' % job)
with open(oldTemplateFile) as F:
lines = F.readlines()
#first parse on a line by line basis
for lineNo, line in enumerate(lines):
for m in MAKEFILE_CONVERT_LINES:
if line.find(m[0]) == 0:
lines[lineNo] = m[1]
break
text = "".join(lines)
#print text
#see if there are 'input filesets'
#$(call moa_fileset_define_opt,$(moa_id)_input,,Input files for $(moa_id))
rere = re.compile(r'\$\(call moa_fileset_define(.*),(.+),(.*),(.*)\)')
for x in rere.finditer(text):
fsid = x.groups()[1]
fsext = x.groups()[2]
fsid = fsid.replace('$(moa_id)', moaId).replace(moaId + '_', '')
fsdesc = x.groups()[3].replace('$(moa_id)', moaId)
l.info("found fileset for %s" % fsid)
if x.groups()[0] == '_opt':
fsopt = True
else:
fsopt = False
fs = {
'type': 'input',
'category': 'input',
'help': fsdesc,
'extension': fsext,
'optional': fsopt
}
if not inf.has_key('filesets'):
inf['filesets'] = {}
inf['filesets'][fsid] = fs
#and now remove all related parameters from the configuration
for x in ['_extension', '_glob', '_limit', '_sort', '_dir']:
ky = fsid + x
if inf['parameters'].has_key(ky):
del inf['parameters'][ky]
text = rere.sub('', text)
# see if there are 'remap' filesets
rere = re.compile(r'\$\(call moa_fileset_remap(.*),(.+),(.*),(.*)\)')
for x in rere.finditer(text):
fsid = x.groups()[2].replace('$(moa_id)',
moaId).replace(moaId + '_', '')
fssrc = x.groups()[1].replace('$(moa_id)',
moaId).replace(moaId + '_', '')
fs_target_extension = x.groups()[3]
l.info("found a remap fileset: %s to %s" % (fssrc, fsid))
if x.groups()[0] == '_nodir':
fs_target_dir = '.'
else:
fs_target_dir = os.path.join('.', fs_target_extension)
fs = {
'type': 'map',
'category': 'output',
'source': fssrc,
'dir': fs_target_dir,
'extension': fs_target_extension
}
if not inf.has_key('filesets'):
inf['filesets'] = {}
inf['filesets'][fsid] = fs
text = rere.sub('', text)
#rere = re.compile(r'(\$\(call.*)')
#for x in rere.finditer(text):
# l.warning("found $call")
# l.warning(x.groups()[0])
for suffix in [
'title', 'help', 'type', 'define', 'description',
'prerequisites', 'default', 'allowed', 'set', 'simple'
]:
rere = re.compile(r'\S+' + suffix + r' ?\+?=.*?(?<!\\)\n', re.S)
text = rere.sub('', text)
rere = re.compile(r'([\t ]*\n){2,}', re.S)
text = rere.sub("\n\n", text)
#save & store the new files
thisTargetDir = os.path.dirname(newTemplateConf)
if not os.path.exists(thisTargetDir):
os.makedirs(thisTargetDir)
with open(newTemplateConf, 'w') as F:
yaml.dump(inf, F)
with open(newTemplateFile, 'w') as G:
G.write(text)
def renumber(path, fr, to):
"""
Renumber a moa job
>>> import tempfile
>>> emptyDir = tempfile.mkdtemp()
>>> removeFiles(emptyDir, recursive=True)
>>> fromDir = os.path.join(emptyDir, '10.test')
>>> problemDir = os.path.join(emptyDir, '20.problem')
>>> toDir = os.path.join(emptyDir, '20.test')
>>> os.mkdir(os.path.join(emptyDir, '10.test'))
>>> os.path.exists(os.path.join(emptyDir, '10.test'))
True
>>> os.path.exists(toDir)
False
>>> renumber(emptyDir, '10', '20')
>>> os.path.exists(fromDir)
False
>>> os.path.exists(toDir)
True
>>> os.mkdir(problemDir)
>>> renumber(emptyDir, '20', '30')
Traceback (most recent call last):
File '/opt/moa/lib/python/moa/utils.py', line 114, in renumber
raise MoaFileError(fullDir)
MoaFileError: Moa error handling file
@param path: the path to operate in
@type path: String
@param fr: number to rename from
@type fr: String representing a number
@param to: number to rename to
@type to: String representing a number
"""
frDir = None
toDir = None
l.debug("moa ren %s %s" % (fr, to))
for x in os.listdir(path):
if x[0] == '.' : continue
fullDir = os.path.join(path, x)
xsplit = x.split('.')
if xsplit[0] == fr:
if frDir:
l.error("more than one directory starting with %s" % fr)
raise MoaFileError(fullDir)
frDir = fullDir
toDir = os.path.join(path, to + "." + ".".join(xsplit[1:]))
if xsplit[0] == to:
l.error("target directory starting with %s already exists" % to)
raise MoaFileError(fullDir)
if not frDir:
l.error("Cannot find a directory starting with %s" % fr)
raise MoaFileError(path)
if not toDir:
l.error("Cannot find a directory starting with %s" % to)
raise MoaFileError(path)
if not os.path.isdir(frDir):
l.error("%s is not a directory" % frDir)
raise MoaFileError(frDir)
#if not os.path.isdir(toDir):
# l.error("%s is not a directory" % toDir)
# raise MoaFileError(toDir)
l.info("renaming: %s" % (frDir))
l.info(" to: %s" % (toDir))
os.rename(frDir, toDir)
def checkTemplate(path):
"""
Run all current template tests - see if any needs updating
All job Makefiles should look exactly alike and have only two
lines of actual Makefile code:
include $(MOABASE)/template/moa/prepare.mk
$(call moa_load,emboss/revseq)
"""
makefilePath = os.path.join(path, 'Makefile')
if not os.path.exists(makefilePath):
l.error("template check failed - cannot find a Makefile")
seenIncludePrepare = False
seenIncludeTemplate = False
superfluousLines = False
template = None
makefile = open(makefilePath)
i = 0
nmf = []
for line in makefile.readlines():
line = line.strip()
i += 1
if not line: continue
if line[0] == '#':
nmf += line
continue
if line == 'include $(MOABASE)/template/moa/prepare.mk':
nmf += line
seenIncludePrepare = True
continue
if 'include $(MOABASE)/template/' in line:
if template:
l.error("Duplicate template include??")
l.error("Please fix this!")
sys.exit(-1)
template = line.replace('include $(MOABASE)/template/', '')
template = template.replace('.mk', '')
l.debug("Discovered template %s" % template)
continue
if '$(call moa_load,' in line:
if template:
l.error("Duplicate template include??")
l.error("Please fix this!")
sys.exit(-1)
seenIncludeTemplate = True
template = line.replace('call moa_load,', '').replace(')',
'').strip()
l.debug("Discovered template %s" % template)
continue
if line in [
'.PHONY: moa_preprocess',
'moa_preprocess:',
'.PHONY: moa_postprocess',
'moa_postprocess:',
'-include moa.mk',
'MOAMK_INCLUDE=done',
'@echo preprocess commands go here',
'@echo Postprocess commands go here..',
]:
continue
superfluousLines = True
l.info("extra code: '%s'" % line)
if seenIncludePrepare and seenIncludeTemplate and (not superfluousLines):
return True
if superfluousLines:
l.error("Deprecated Makefile, discovered extra code")
l.error("Please fix this manually")
sys.exit(-1)
l.error("Deprecated template, fixing. Please check!")
shutil.move(makefilePath, makefilePath + '.old')
moa.job.newJob(template=template, wd=path, noInit=True, titleCheck=False)
sys.exit(0)
def renumber(path, fr, to):
"""
Renumber a moa job
>>> import tempfile
>>> emptyDir = tempfile.mkdtemp()
>>> removeFiles(emptyDir, recursive=True)
>>> fromDir = os.path.join(emptyDir, '10.test')
>>> problemDir = os.path.join(emptyDir, '20.problem')
>>> toDir = os.path.join(emptyDir, '20.test')
>>> os.mkdir(os.path.join(emptyDir, '10.test'))
>>> os.path.exists(os.path.join(emptyDir, '10.test'))
True
>>> os.path.exists(toDir)
False
>>> renumber(emptyDir, '10', '20')
>>> os.path.exists(fromDir)
False
>>> os.path.exists(toDir)
True
>>> os.mkdir(problemDir)
>>> renumber(emptyDir, '20', '30')
Traceback (most recent call last):
File '/opt/moa/lib/python/moa/utils.py', line 114, in renumber
raise MoaFileError(fullDir)
MoaFileError: Moa error handling file
@param path: the path to operate in
@type path: String
@param fr: number to rename from
@type fr: String representing a number
@param to: number to rename to
@type to: String representing a number
"""
frDir = None
toDir = None
l.debug("moa ren %s %s" % (fr, to))
for x in os.listdir(path):
if x[0] == '.': continue
fullDir = os.path.join(path, x)
xsplit = x.split('.')
if xsplit[0] == fr:
if frDir:
l.error("more than one directory starting with %s" % fr)
raise MoaFileError(fullDir)
frDir = fullDir
toDir = os.path.join(path, to + "." + ".".join(xsplit[1:]))
if xsplit[0] == to:
l.error("target directory starting with %s already exists" % to)
raise MoaFileError(fullDir)
if not frDir:
l.error("Cannot find a directory starting with %s" % fr)
raise MoaFileError(path)
if not toDir:
l.error("Cannot find a directory starting with %s" % to)
raise MoaFileError(path)
if not os.path.isdir(frDir):
l.error("%s is not a directory" % frDir)
raise MoaFileError(frDir)
#if not os.path.isdir(toDir):
# l.error("%s is not a directory" % toDir)
# raise MoaFileError(toDir)
l.info("renaming: %s" % (frDir))
l.info(" to: %s" % (toDir))
os.rename(frDir, toDir)
def checkTemplate(path):
"""
Run all current template tests - see if any needs updating
All job Makefiles should look exactly alike and have only two
lines of actual Makefile code:
include $(MOABASE)/template/moa/prepare.mk
$(call moa_load,emboss/revseq)
"""
makefilePath = os.path.join(path, 'Makefile')
if not os.path.exists(makefilePath):
l.error("template check failed - cannot find a Makefile")
seenIncludePrepare = False
seenIncludeTemplate = False
superfluousLines = False
template = None
makefile = open(makefilePath)
i = 0
nmf = []
for line in makefile.readlines():
line = line.strip()
i += 1
if not line: continue
if line[0] == '#':
nmf += line
continue
if line == 'include $(MOABASE)/template/moa/prepare.mk':
nmf += line
seenIncludePrepare = True
continue
if 'include $(MOABASE)/template/' in line:
if template:
l.error("Duplicate template include??")
l.error("Please fix this!")
sys.exit(-1)
template = line.replace('include $(MOABASE)/template/', '')
template = template.replace('.mk', '')
l.debug("Discovered template %s" % template)
continue
if '$(call moa_load,' in line:
if template:
l.error("Duplicate template include??")
l.error("Please fix this!")
sys.exit(-1)
seenIncludeTemplate = True
template = line.replace('call moa_load,', '').replace(')', '').strip()
l.debug("Discovered template %s" % template)
continue
if line in ['.PHONY: moa_preprocess', 'moa_preprocess:',
'.PHONY: moa_postprocess', 'moa_postprocess:',
'-include moa.mk', 'MOAMK_INCLUDE=done',
'@echo preprocess commands go here',
'@echo Postprocess commands go here..',
]:
continue
superfluousLines = True
l.info("extra code: '%s'" % line)
if seenIncludePrepare and seenIncludeTemplate and (not superfluousLines):
return True
if superfluousLines:
l.error("Deprecated Makefile, discovered extra code")
l.error("Please fix this manually")
sys.exit(-1)
l.error("Deprecated template, fixing. Please check!")
shutil.move(makefilePath, makefilePath + '.old')
moa.job.newJob(template=template, wd=path,
noInit=True, titleCheck=False)
sys.exit(0)
def createAdhoc(data):
"""
Create an adhoc job
"""
wd = data['cwd']
options = data['options']
args = data['newargs']
command = " ".join(args).strip()
if not command:
command=moa.utils.askUser('adhoc_command:\n>', '')
l.info('Parsing command: %s' % command)
params = []
mode = None
searchGlobs = True
if options.mode:
mode = options.mode
if options.mode == 'simple': searchGlobs = False
if not options.mode in ['seq', 'par', 'all', 'simple']:
l.critical("Unknown adhoc mode: %s" % options.mode)
sys.exit(-1)
elif '$<' in command:
mode = 'seq'
searchGlobs = False
elif ('$^' in command) or ('$?' in command):
mode = 'all'
searchGlobs = False
l.warning("Observed '$^' or '$?', setting mode to 'all'")
l.warning("Processing all files in one go")
#see if we have to look for file globs
if not searchGlobs:
l.info("No recognizable globs found")
else:
#it appears to make sense to see if there is a glob in the command
refindGlob = re.compile(
r"([^ *]+" \
+ os.sep \
+ ")?([^ *]*\*[^ *]*?)((?:\.[^ .*]+)?)")
globs = []
for g in refindGlob.finditer(command):
globs.append(g)
if globs:
globReplace = '$<', '$t'
mode = 'seq'
if len(globs) > 2:
raise Exception("Too many globs ?? I not understand :(")
if len(globs) == 2:
st1 = _sourceOrTarget(globs[0])
st2 = _sourceOrTarget(globs[1])
if st1 == st2:
l.warn("Unsure wich is the source & target glob, assuming:")
inGlob,outGlob = globs
if st1 == 'source': inGlob,outGlob = globs
else:
outGlob,inGlob = globs
globReplace = '$t', '$<'
l.info("Input glob: %s" % inGlob.group())
l.info("Output glob: %s" % outGlob.group())
else:
l.info("Input glob: %s" % globs[0].group())
inGlob, outGlob = globs[0], None
inD, inG, inE = inGlob.groups()
if not inD: inD = ""
if not inE: inE = ""
l.info(" - set input dir : %s" % inD)
l.info(" - set input glob : %s" % inG)
l.info(" - set input extension : %s" % inE[1:])
params += ['adhoc_input_dir=%s' % inD]
params += ['adhoc_input_glob=%s' % inG]
params += ['adhoc_input_extension=%s' % inE[1:]]
if outGlob:
ouD, ouG, ouE = outGlob.groups()
if not ouD: ouD = ""
if not ouE: ouE = ""
ogg = outGlob.groups()
ouG1, ouG2 = ouG.split('*')
sed = r"s^\(.*\)%s^%s%s\1%s%s^g" % (
inE.replace('.', '\.'),
ouD.replace('/', '/'),
ouG.split('*')[0],
ouG.split('*')[1],
ouE
)
l.info(" - set name_sed : %s " % sed)
l.info(" - set output dir : %s " % ouD)
params += ['adhoc_output_dir=%s' % ouD]
params += ['adhoc_name_sed=%s' % sed]
#hack the commandline
for i in range(len(globs)-1, -1, -1):
g = globs[i]
command = command[:g.start()] + globReplace[i] + command[g.end():]
if not mode:
mode = 'simple'
if command:
l.info(" - set command : %s" % command)
params.append('adhoc_process=%s' % command)
params.append('adhoc_mode=%s' % mode)
l.info(" - set mode : %s" % mode)
if mode == 'seq':
l.warning("Note: adhoc is running in sequential ('seq') mode. If ")
l.warning("you are confident that the individual jobs do not interfere, you might ")
l.warning("consider setting adhoc to parallel operation:")
l.warning("$ moa set adhoc_mode=par")
l.debug('setting parameters %s' % params)
job = moa.job.newJob(wd,
template='adhoc',
title = options.title,
force = options.force,
parameters=params)
def run(options, args):
#remove the first argument from args (should be 'test')
args.pop(0)
os.putenv('MOA_UNITTESTS', "yes")
if args:
l.info("Testing '%s'" % " ".join(args))
if not args:
l.info("Start running python doctests")
setSilent()
testModule(moa.utils)
testModule(moa.lock)
testModule(moa.conf)
testModule(moa.project)
testModule(moa.template)
testModule(moa.job)
if options.verbose: setVerbose()
else: setInfo()
l.info("Finished running of python unittests")
l.info("Ran %d test, %d failed" % (tests, failures))
l.info("Start running basic template tests")
testTemplates()
l.info("Ran %d template test, %d failed" % (
templateTests, templateFailures))
l.info("Start running plugin tests")
testPlugins()
l.info("Ran %d plugin test, %d failed" % (
pluginTests, pluginFailures))
l.info("Finished running plugin tests")
sys.exit()
elif args[0] == 'plugins':
l.info("Start running plugin tests")
testPlugins(args[1:])
l.info("Ran %d plugin test, %d failed" % (
pluginTests, pluginFailures))
l.info("Finished running plugin tests")
elif args[0] == 'plugin':
l.info("Start running plugin tests")
testPlugins(args[1:])
elif args[0] == 'templates':
l.info("Start running basic template tests")
testTemplates()
l.info("Ran %d template test, %d failed" % (
templateTests, templateFailures))
l.info("Finished running basic template tests")
elif args[0][:4] == 'moa.':
l.info("testing moa python module %s" % args[0])
setSilent()
eval("testModule(%s)" % args[0])
if options.verbose: setVerbose()
else: setInfo()
l.info("Finished running unittests for %s" % args[0])
l.info("Ran %d test, %d failed" % (tests, failures))
else:
#Assume it is a templat
testTemplates(args[0], verbose=options.verbose)
testTemplateExtensive(args[0], verbose=options.verbose)
