def fixLinks(self, searchDir):
runs = self.getRuns()
for prot in runs:
broken = False
if isinstance(prot, em.ProtImport):
for _, attr in prot.iterOutputEM():
fn = attr.getFiles()
for f in attr.getFiles():
if ':' in f:
f = f.split(':')[0]
if not os.path.exists(f):
if not broken:
broken = True
print "Found broken links in run: ", pwutils.magenta(prot.getRunName())
print " Missing: ", pwutils.magenta(f)
if os.path.islink(f):
print " -> ", pwutils.red(os.path.realpath(f))
newFile = pwutils.findFile(os.path.basename(f), searchDir, recursive=True)
if newFile:
print " Found file %s, creating link..." % newFile
print pwutils.green(" %s -> %s" % (f, newFile))
pwutils.createAbsLink(newFile, f)
def _runNewItem(self, itemType, itemName):
if self._match(itemName):
spaces = (itemType * 2) * ' '
scipion = join(os.environ['SCIPION_HOME'], 'scipion')
if (itemName.startswith("xmipp_")):
cmd = "%s %s %s" % (spaces, scipion, itemName)
else:
cmd = "%s %s test %s" % (spaces, scipion, itemName)
run = ((itemType == MODULE and self.mode == 'module')
or (itemType == CLASS and self.mode == 'classes')
or (itemType == TEST and self.mode == 'all'))
if run:
if self.log:
logFile = join(self.testsDir, '%s.txt' % itemName)
cmdFull = cmd + " > %s 2>&1" % logFile
else:
logFile = ''
cmdFull = cmd
print pwutils.green(cmdFull)
t = pwutils.Timer()
t.tic()
self.testCount += 1
result = os.system(cmdFull)
if self.log:
self._logTest(cmd.replace(scipion, 'scipion'), t.getToc(),
result, logFile)
def _runNewItem(self, itemType, itemName):
if self._match(itemName):
spaces = (itemType * 2) * ' '
scipion = join(os.environ['SCIPION_HOME'], 'scipion')
if (itemName.startswith("xmipp_")):
cmd = "%s %s %s" % (spaces, scipion, itemName)
else:
cmd = "%s %s test %s" % (spaces, scipion, itemName)
run = ((itemType == MODULE and self.mode == 'module') or
(itemType == CLASS and self.mode == 'classes') or
(itemType == TEST and self.mode == 'all'))
if run:
if self.log:
logFile = join(self.testsDir, '%s.txt' % itemName)
cmdFull = cmd + " > %s 2>&1" % logFile
else:
logFile = ''
cmdFull = cmd
print pwutils.green(cmdFull)
t = pwutils.Timer()
t.tic()
self.testCount += 1
result = os.system(cmdFull)
if self.log:
self._logTest(cmd.replace(scipion, 'scipion'),
t.getToc(), result, logFile)
def update(dataset, workingCopy=None, url=None, verbose=False):
""" Update local dataset with the contents of the remote one.
It compares the md5 of remote files in url/dataset/MANIFEST with the
ones in workingCopy/dataset/MANIFEST, and downloads only when necessary.
"""
# Get default values for variables if we got None.
workingCopy = workingCopy or os.environ['SCIPION_TESTS']
# Verbose log
def vlog(txt): sys.stdout.write(txt) if verbose else None
# Read contents of *remote* MANIFEST file, and create a dict {fname: md5}
manifest = urlopen('%s/%s/MANIFEST' % (url, dataset)).readlines()
md5sRemote = dict(x.strip().split() for x in manifest)
# Update and read contents of *local* MANIFEST file, and create a dict
datasetFolder = join(workingCopy, dataset)
try:
last = max(os.stat(join(datasetFolder, x)).st_mtime for x in md5sRemote)
t_manifest = os.stat(join(datasetFolder, 'MANIFEST')).st_mtime
assert t_manifest > last and time.time() - t_manifest < 60*60*24*7
except (OSError, IOError, AssertionError) as e:
print "Regenerating local MANIFEST..."
createMANIFEST(datasetFolder)
md5sLocal = dict(x.split() for x in open(join(datasetFolder, 'MANIFEST')))
# Check that all the files mentioned in MANIFEST are up-to-date
print "Verifying MD5s..."
filesUpdated = 0 # number of files that have been updated
taintedMANIFEST = False # can MANIFEST be out of sync?
for fname in md5sRemote:
vlog(" %s" % fname)
fpath = join(datasetFolder, fname)
try:
if exists(fpath) and md5sLocal[fname] == md5sRemote[fname]:
vlog("\r %s %s\n" % (green("OK"), fname))
pass # just to emphasize that we do nothing in this case
else:
vlog("\r %s %s (downloading... " % (red("XX"), fname))
if not isdir(dirname(fpath)):
os.makedirs(dirname(fpath))
open(fpath, 'w').writelines(
urlopen('%s/%s/%s' % (url, dataset, fname)))
vlog("done)\n")
filesUpdated += 1
except Exception as e:
print "\nError while updating %s: %s" % (fname, e)
taintedMANIFEST = True # if we don't update, it can be wrong
print "...done. Updated files: %d" % filesUpdated
# Save the new MANIFEST file in the folder of the downloaded dataset
if filesUpdated > 0:
open(join(datasetFolder, 'MANIFEST'), 'w').writelines(manifest)
if taintedMANIFEST:
print "Some files could not be updated. Regenerating local MANIFEST ..."
createMANIFEST(datasetFolder)
def runCase(self, args, mpi=0, changeDir=False,
preruns=None, postruns=None, validate=None,
outputs=None, random=False):
# Retrieve the correct case number from the test name id
# We asumme here that 'test_caseXXX' should be in the name
caseId = unittest.TestCase.id(self)
if not 'test_case' in caseId:
raise Exception("'test_case' string should be in the test function name followed by a number")
_counter = int(caseId.split('test_case')[1])
self._testDir = self.dataset.getPath()
self.outputDir = os.path.join('tmpLink', '%s_%02d' % (self.program, _counter))
self.outputDirAbs = os.path.join(self._testDir, self.outputDir)
self.goldDir = os.path.join(self._testDir, 'gold', '%s_%02d' % (self.program, _counter))
# Clean and create the program output folder if not exists
pwutils.cleanPath(self.outputDirAbs)
pwutils.makePath(self.outputDirAbs)
# Change to tests root folder (self._testDir)
cwd = os.getcwd()
os.chdir(self._testDir)
if preruns:
self._runCommands(preruns, 'preruns')
if mpi:
cmd = "mpirun -np %d `which %s`" % (mpi, self.program)
else:
cmd = self.program
args = self._parseArgs(args)
if changeDir:
cmd = "cd %s ; %s %s > stdout.txt 2> stderr.txt" % (self.outputDir, cmd, args)
else:
cmd = "%s %s > %s/stdout.txt 2> %s/stderr.txt" % (cmd, args, self.outputDir, self.outputDir)
print " Command: "
print " ", pwutils.green(cmd)
#run the test itself
command = Command(cmd, env=self.env)
self._command = command
try:
command.run(timeout=self._timeout)
except KeyboardInterrupt:
command.terminate()
if postruns:
self._runCommands(postruns, 'postruns')
if outputs:
self._checkOutputs(outputs,random)
if validate:
validate()
os.chdir(cwd)
def _visitTests(self, moduleName, tests, newItemCallback):
""" Show the list of tests available """
mode = self.mode
assert mode in ['modules', 'classes', 'onlyclasses', 'all'], 'Unknown mode %s' % mode
# First flatten the list of tests.
# testsFlat = list(iter(self.__iterTests(tests)))
testsFlat = []
toCheck = [t for t in tests]
while toCheck:
test = toCheck.pop()
if isinstance(test, unittest.TestSuite):
toCheck += [t for t in test]
else:
testsFlat.append(test)
# Follow the flattened list of tests and show the module, class
# and name, in a nice way.
lastClass = None
lastModule = None
if testsFlat:
for t in testsFlat:
testModuleName, className, testName = t.id().rsplit('.', 2)
# If there is a failure loading the test, show it
errorStr = 'unittest.loader.ModuleImportFailure.'
if testModuleName.startswith(errorStr):
newName = t.id().replace(errorStr, '')
if self._match(newName):
print(pwutils.red('Error loading the test. Please, run the test for more information:'), newName)
continue
if testModuleName != lastModule:
lastModule = testModuleName
if mode != 'onlyclasses':
newItemCallback(MODULE, "%s" % testModuleName)
if mode in ['classes', 'onlyclasses', 'all'] and className != lastClass:
lastClass = className
newItemCallback(CLASS, "%s.%s"
% (testModuleName, className))
if mode == 'all':
newItemCallback(TEST, "%s.%s.%s"
% (testModuleName, className, testName))
else:
if not self.grep:
print(pwutils.green(' The plugin does not have any test'))
def update(dataset, workingCopy=None, url=None, verbose=False):
""" Update local dataset with the contents of the remote one.
It compares the md5 of remote files in url/dataset/MANIFEST with the
ones in workingCopy/dataset/MANIFEST, and downloads only when necessary.
"""
# Get default values for variables if we got None.
workingCopy = workingCopy or pw.Config.SCIPION_TESTS
# Verbose log
def vlog(txt):
sys.stdout.write(txt) if verbose else None
# Read contents of *remote* MANIFEST file, and create a dict {fname: md5}
manifest = urlopen('%s/%s/MANIFEST' % (url, dataset)).readlines()
md5sRemote = dict(x.decode("utf-8").strip().split() for x in manifest)
# Update and read contents of *local* MANIFEST file, and create a dict
datasetFolder = join(workingCopy, dataset)
try:
last = max(
os.stat(join(datasetFolder, x)).st_mtime for x in md5sRemote)
t_manifest = os.stat(join(datasetFolder, 'MANIFEST')).st_mtime
assert t_manifest > last and time.time() - t_manifest < 60 * 60 * 24 * 7
except (OSError, IOError, AssertionError) as e:
print("Regenerating local MANIFEST...")
createMANIFEST(datasetFolder)
md5sLocal = dict(x.strip().split()
for x in open(join(datasetFolder, 'MANIFEST')))
# Check that all the files mentioned in MANIFEST are up-to-date
print("Verifying MD5s...")
filesUpdated = 0 # number of files that have been updated
taintedMANIFEST = False # can MANIFEST be out of sync?
downloadingPrinted = False
for fname in md5sRemote:
fpath = join(datasetFolder, fname)
try:
if exists(fpath) and md5sLocal[fname] == md5sRemote[fname]:
vlog("\r %s %s\n" % (green("OK"), fname))
pass # just to emphasize that we do nothing in this case
else:
if not downloadingPrinted:
verboseMsg = " Next time use -v for more details." if not verbose else ""
print("Differences detected. Downloading data.%s" %
verboseMsg)
vlog("\r %s %s (downloading... " % (red("XX"), fname))
if not isdir(dirname(fpath)):
os.makedirs(dirname(fpath))
urlretrieve('%s/%s/%s' % (url, dataset, fname), fpath)
vlog("done)\n")
filesUpdated += 1
except Exception as e:
print("\nError while updating %s: %s" % (fname, e))
taintedMANIFEST = True # if we don't update, it can be wrong
print("...done. Updated files: %d" % filesUpdated)
# Save the new MANIFEST file in the folder of the downloaded dataset
if filesUpdated > 0:
open(join(datasetFolder, 'MANIFEST'), 'w').writelines(md5sRemote)
if taintedMANIFEST:
print(
"Some files could not be updated. Regenerating local MANIFEST ...")
createMANIFEST(datasetFolder)
def runCase(self,
args,
mpi=0,
changeDir=False,
preruns=None,
postruns=None,
validate=None,
outputs=None,
random=False,
errorthreshold=0.001):
# Retrieve the correct case number from the test name id
# We asumme here that 'test_caseXXX' should be in the name
caseId = unittest.TestCase.id(self)
if not 'test_case' in caseId:
raise Exception(
"'test_case' string should be in the test function name followed by a number"
)
_counter = int(caseId.split('test_case')[1])
self._testDir = self.dataset.getPath()
self.outputDir = os.path.join('tmpLink',
'%s_%02d' % (self.program, _counter))
self.outputDirAbs = os.path.join(self._testDir, self.outputDir)
self.goldDir = os.path.join(self._testDir, 'gold',
'%s_%02d' % (self.program, _counter))
# Clean and create the program output folder if not exists
pwutils.cleanPath(self.outputDirAbs)
pwutils.makePath(self.outputDirAbs)
# Change to tests root folder (self._testDir)
cwd = os.getcwd()
os.chdir(self._testDir)
if preruns:
self._runCommands(preruns, 'preruns')
if mpi:
cmd = "mpirun -np %d `which %s`" % (mpi, self.program)
else:
cmd = self.program
args = self._parseArgs(args)
if changeDir:
cmd = "cd %s ; %s %s > stdout.txt 2> stderr.txt" % (self.outputDir,
cmd, args)
else:
cmd = "%s %s > %s/stdout.txt 2> %s/stderr.txt" % (
cmd, args, self.outputDir, self.outputDir)
print " Command: "
print " ", pwutils.green(cmd)
#run the test itself
command = Command(cmd, env=self.env)
self._command = command
try:
command.run(timeout=self._timeout)
except KeyboardInterrupt:
command.terminate()
if postruns:
self._runCommands(postruns, 'postruns')
if outputs:
self._checkOutputs(outputs, random, errorthreshold=errorthreshold)
if validate:
validate()
os.chdir(cwd)
manager = Manager()
if not manager.hasProject(projName):
usage("Unexistent project: %s" % pwutils.red(projName))
if not os.path.exists(searchDir):
usage("Unexistent SEARCH_DIR: %s" % pwutils.red(searchDir))
project = manager.loadProject(projName)
runs = project.getRuns()
for prot in runs:
broken = False
if isinstance(prot, em.ProtImport):
for _, attr in prot.iterOutputEM():
fn = attr.getFiles()
for f in attr.getFiles():
if not os.path.exists(f):
if not broken:
broken = True
print "Found broken links in run: ", pwutils.magenta(prot.getRunName())
print " Missing: ", pwutils.magenta(f)
if os.path.islink(f):
print " -> ", pwutils.red(os.path.realpath(f))
newFile = pwutils.findFile(os.path.basename(f), searchDir, recursive=True)
if newFile:
print " Found file %s, creating link..." % newFile
print pwutils.green(" %s -> %s" % (f, newFile))
pwutils.createAbsLink(newFile, f)
usage("Unexistent project: %s" % pwutils.red(projName))
if not os.path.exists(searchDir):
usage("Unexistent SEARCH_DIR: %s" % pwutils.red(searchDir))
project = manager.loadProject(projName)
runs = project.getRuns()
for prot in runs:
broken = False
if isinstance(prot, em.ProtImport):
for _, attr in prot.iterOutputEM():
fn = attr.getFiles()
for f in attr.getFiles():
if ':' in f:
f = f.split(':')[0]
if not os.path.exists(f):
if not broken:
broken = True
print "Found broken links in run: ", pwutils.magenta(prot.getRunName())
print " Missing: ", pwutils.magenta(f)
if os.path.islink(f):
print " -> ", pwutils.red(os.path.realpath(f))
newFile = pwutils.findFile(os.path.basename(f), searchDir, recursive=True)
if newFile:
print " Found file %s, creating link..." % newFile
print pwutils.green(" %s -> %s" % (f, newFile))
pwutils.createAbsLink(newFile, f)