def printTests(tests, mode='modules'):
""" Show the list of tests available """
assert mode in ['modules', 'classes', 'all'], 'Unknown mode %s' % mode
# First flatten the list of tests.
testsFlat = []
toCheck = [t for t in tests]
while toCheck:
test = toCheck.pop()
if isinstance(test, unittest.TestSuite):
toCheck += [t for t in test]
else:
testsFlat.append(test)
# Follow the flattened list of tests and show the module, class
# and name, in a nice way.
lastClass = None
lastModule = None
for t in testsFlat:
moduleName, className, testName = t.id().rsplit('.', 2)
# If there is a failure loading the test, show it
if moduleName.startswith('unittest.loader.ModuleImportFailure'):
print pwutils.red(moduleName), " test:", t.id()
continue
if moduleName != lastModule:
lastModule = moduleName
print "scipion test %s" % moduleName
if mode in ['classes', 'all'] and className != lastClass:
lastClass = className
print " scipion test %s.%s" % (moduleName, className)
if mode == 'all':
print " scipion test %s.%s.%s" % (moduleName, className, testName)
def _visitTests(self, tests, newItemCallback):
""" Show the list of tests available """
mode = self.mode
assert mode in ['modules', 'classes', 'all'], 'Unknown mode %s' % mode
# First flatten the list of tests.
testsFlat = []
toCheck = [t for t in tests]
while toCheck:
test = toCheck.pop()
if isinstance(test, unittest.TestSuite):
toCheck += [t for t in test]
else:
testsFlat.append(test)
# Follow the flattened list of tests and show the module, class
# and name, in a nice way.
lastClass = None
lastModule = None
for t in testsFlat:
moduleName, className, testName = t.id().rsplit('.', 2)
# If there is a failure loading the test, show it
if moduleName.startswith('unittest.loader.ModuleImportFailure'):
print pwutils.red(moduleName), " test:", t.id()
continue
def _hasLabels():
if self.labels is None:
return True
# Check if class has a label that matches input labels.
import importlib
my_module = importlib.import_module(moduleName)
MyClass = getattr(my_module, className)
return pwtests.hasLabel(MyClass, self.labels)
if _hasLabels():
if moduleName != lastModule:
lastModule = moduleName
newItemCallback(MODULE, moduleName)
if mode in ['classes', 'all'] and className != lastClass:
lastClass = className
newItemCallback(CLASS, "%s.%s" % (moduleName, className))
if mode == 'all':
newItemCallback(
TEST, "%s.%s.%s" % (moduleName, className, testName))
# If label is "pull" then add all basic xmipp tests.
if (self.labels is not None):
if PULL_REQUEST in self.labels:
self.discoverXmippTest(newItemCallback)
else:
self.discoverXmippTest(newItemCallback)
def _visitTests(self, tests, newItemCallback):
""" Show the list of tests available """
mode = self.mode
assert mode in ['modules', 'classes', 'all'], 'Unknown mode %s' % mode
# First flatten the list of tests.
testsFlat = []
toCheck = [t for t in tests]
while toCheck:
test = toCheck.pop()
if isinstance(test, unittest.TestSuite):
toCheck += [t for t in test]
else:
testsFlat.append(test)
# Follow the flattened list of tests and show the module, class
# and name, in a nice way.
lastClass = None
lastModule = None
for t in testsFlat:
moduleName, className, testName = t.id().rsplit('.', 2)
# If there is a failure loading the test, show it
if moduleName.startswith('unittest.loader.ModuleImportFailure'):
print pwutils.red(moduleName), " test:", t.id()
continue
def _hasLabels():
if self.labels is None:
return True
# Check if class has a label that matches input labels.
import importlib
my_module = importlib.import_module(moduleName)
MyClass = getattr(my_module, className)
return pwtests.hasLabel(MyClass, self.labels)
if _hasLabels():
if moduleName != lastModule:
lastModule = moduleName
newItemCallback(MODULE, moduleName)
if mode in ['classes', 'all'] and className != lastClass:
lastClass = className
newItemCallback(CLASS, "%s.%s" % (moduleName, className))
if mode == 'all':
newItemCallback(TEST, "%s.%s.%s" % (moduleName, className, testName))
# If label is "pull" then add all basic xmipp tests.
if (self.labels is not None):
if PULL_REQUEST in self.labels:
self.discoverXmippTest(newItemCallback)
else:
self.discoverXmippTest(newItemCallback)
def run(self, timeout):
def target():
self.process = subprocess.Popen(self.cmd, shell=True, env=self.env)
self.process.communicate()
thread = threading.Thread(target=target)
thread.start()
thread.join(timeout)
if thread.is_alive():
print pwutils.red('ERROR: timeout reached for this process')
self.process.terminate()
thread.join()
self.process = None
def run(self, timeout):
def target():
self.process = subprocess.Popen(self.cmd, shell=True, env=self.env)
self.process.communicate()
thread = threading.Thread(target=target)
thread.start()
thread.join(timeout)
if thread.is_alive():
print pwutils.red('ERROR: timeout reached for this process')
self.process.terminate()
thread.join()
self.process = None
def update(dataset, workingCopy=None, url=None, verbose=False):
""" Update local dataset with the contents of the remote one.
It compares the md5 of remote files in url/dataset/MANIFEST with the
ones in workingCopy/dataset/MANIFEST, and downloads only when necessary.
"""
# Get default values for variables if we got None.
workingCopy = workingCopy or os.environ['SCIPION_TESTS']
# Verbose log
def vlog(txt): sys.stdout.write(txt) if verbose else None
# Read contents of *remote* MANIFEST file, and create a dict {fname: md5}
manifest = urlopen('%s/%s/MANIFEST' % (url, dataset)).readlines()
md5sRemote = dict(x.strip().split() for x in manifest)
# Update and read contents of *local* MANIFEST file, and create a dict
datasetFolder = join(workingCopy, dataset)
try:
last = max(os.stat(join(datasetFolder, x)).st_mtime for x in md5sRemote)
t_manifest = os.stat(join(datasetFolder, 'MANIFEST')).st_mtime
assert t_manifest > last and time.time() - t_manifest < 60*60*24*7
except (OSError, IOError, AssertionError) as e:
print "Regenerating local MANIFEST..."
createMANIFEST(datasetFolder)
md5sLocal = dict(x.split() for x in open(join(datasetFolder, 'MANIFEST')))
# Check that all the files mentioned in MANIFEST are up-to-date
print "Verifying MD5s..."
filesUpdated = 0 # number of files that have been updated
taintedMANIFEST = False # can MANIFEST be out of sync?
for fname in md5sRemote:
vlog(" %s" % fname)
fpath = join(datasetFolder, fname)
try:
if exists(fpath) and md5sLocal[fname] == md5sRemote[fname]:
vlog("\r %s %s\n" % (green("OK"), fname))
pass # just to emphasize that we do nothing in this case
else:
vlog("\r %s %s (downloading... " % (red("XX"), fname))
if not isdir(dirname(fpath)):
os.makedirs(dirname(fpath))
open(fpath, 'w').writelines(
urlopen('%s/%s/%s' % (url, dataset, fname)))
vlog("done)\n")
filesUpdated += 1
except Exception as e:
print "\nError while updating %s: %s" % (fname, e)
taintedMANIFEST = True # if we don't update, it can be wrong
print "...done. Updated files: %d" % filesUpdated
# Save the new MANIFEST file in the folder of the downloaded dataset
if filesUpdated > 0:
open(join(datasetFolder, 'MANIFEST'), 'w').writelines(manifest)
if taintedMANIFEST:
print "Some files could not be updated. Regenerating local MANIFEST ..."
createMANIFEST(datasetFolder)
def download(dataset, destination=None, url=None, verbose=False):
""" Download all the data files mentioned in url/dataset/MANIFEST """
# Get default values for variables if we got None.
destination = destination or pw.Config.SCIPION_TESTS
# First make sure that we ask for a known dataset.
if dataset not in [
x.decode('utf-8').strip('./\n')
for x in urlopen('%s/MANIFEST' % url)
]:
print("Unknown dataset: %s" % red(dataset))
print("Use --list to see the available datasets.")
return
# Retrieve the dataset's MANIFEST file.
# It contains a list of "file md5sum" of all files included in the dataset.
datasetFolder = join(destination, dataset)
os.makedirs(datasetFolder)
manifest = join(destination, dataset, 'MANIFEST')
try:
if verbose:
print("Retrieving MANIFEST file")
open(manifest,
'wb').writelines(urlopen('%s/%s/MANIFEST' % (url, dataset)))
except Exception as e:
print("ERROR reading %s/%s/MANIFEST (%s)" % (url, dataset, e))
return
# Now retrieve all of the files mentioned in MANIFEST, and check their md5.
print('Fetching files of dataset "%s"...' % dataset)
lines = open(manifest).readlines()
done = 0.0 # fraction already done
inc = 1.0 / len(lines) # increment, how much each iteration represents
for line in lines:
fname, md5Remote = line.strip().split()
fpath = join(datasetFolder, fname)
try:
# Download content and create file with it.
if not isdir(dirname(fpath)):
os.makedirs(dirname(fpath))
open(fpath,
'wb').writelines(urlopen('%s/%s/%s' % (url, dataset, fname)))
md5 = md5sum(fpath)
assert md5 == md5Remote, \
"Bad md5. Expected: %s Computed: %s" % (md5Remote, md5)
done += inc
if verbose:
print(redB("%3d%% " % (100 * done)), fname)
else:
sys.stdout.write(
redB("#") * (int(50 * done) - int(50 * (done - inc))))
sys.stdout.flush()
except Exception as e:
print("\nError in %s (%s)" % (fname, e))
print("URL: %s/%s/%s" % (url, dataset, fname))
print("Destination: %s" % fpath)
if ask("Continue downloading? (y/[n]): ", ['y', 'n', '']) != 'y':
return
def fixLinks(self, searchDir):
runs = self.getRuns()
for prot in runs:
broken = False
if isinstance(prot, em.ProtImport):
for _, attr in prot.iterOutputEM():
fn = attr.getFiles()
for f in attr.getFiles():
if ':' in f:
f = f.split(':')[0]
if not os.path.exists(f):
if not broken:
broken = True
print "Found broken links in run: ", pwutils.magenta(prot.getRunName())
print " Missing: ", pwutils.magenta(f)
if os.path.islink(f):
print " -> ", pwutils.red(os.path.realpath(f))
newFile = pwutils.findFile(os.path.basename(f), searchDir, recursive=True)
if newFile:
print " Found file %s, creating link..." % newFile
print pwutils.green(" %s -> %s" % (f, newFile))
pwutils.createAbsLink(newFile, f)
def download(dataset, destination=None, url=None, verbose=False):
""" Download all the data files mentioned in url/dataset/MANIFEST """
# Get default values for variables if we got None.
destination = destination or os.environ['SCIPION_TESTS']
# First make sure that we ask for a known dataset.
if dataset not in [x.strip('./\n') for x in urlopen('%s/MANIFEST' % url)]:
print "Unknown dataset: %s" % red(dataset)
print "Use --list to see the available datasets."
return
# Retrieve the dataset's MANIFEST file.
# It contains a list of "file md5sum" of all files included in the dataset.
datasetFolder = join(destination, dataset)
os.makedirs(datasetFolder)
manifest = join(destination, dataset, 'MANIFEST')
try:
if verbose:
print "Retrieving MANIFEST file"
open(manifest, 'w').writelines(
urlopen('%s/%s/MANIFEST' % (url, dataset)))
except Exception as e:
print "ERROR reading %s/%s/MANIFEST (%s)" % (url, dataset, e)
return
# Now retrieve all of the files mentioned in MANIFEST, and check their md5.
print 'Fetching files of dataset "%s"...' % dataset
lines = open(manifest).readlines()
done = 0.0 # fraction already done
inc = 1.0 / len(lines) # increment, how much each iteration represents
for line in lines:
fname, md5Remote = line.strip().split()
fpath = join(datasetFolder, fname)
try:
# Download content and create file with it.
if not isdir(dirname(fpath)):
os.makedirs(dirname(fpath))
open(fpath, 'w').writelines(
urlopen('%s/%s/%s' % (url, dataset, fname)))
md5 = md5sum(fpath)
assert md5 == md5Remote, \
"Bad md5. Expected: %s Computed: %s" % (md5Remote, md5)
done += inc
if verbose:
print redB("%3d%% " % (100 * done)), fname
else:
sys.stdout.write(redB("#") * (int(50*done)-int(50*(done-inc))))
sys.stdout.flush()
except Exception as e:
print "\nError in %s (%s)" % (fname, e)
print "URL: %s/%s/%s" % (url, dataset, fname)
print "Destination: %s" % fpath
if ask("Continue downloading? (y/[n]): ", ['y', 'n', '']) != 'y':
return
print
def main():
if len(sys.argv) != 3:
usage("Incorrect number of input parameters")
projName = sys.argv[1]
searchDir = os.path.abspath(sys.argv[2])
# Create a new project
manager = Manager()
if not manager.hasProject(projName):
usage("Nonexistent project: %s" % pwutils.red(projName))
if not os.path.exists(searchDir):
usage("Nonexistent SEARCH_DIR: %s" % pwutils.red(searchDir))
project = manager.loadProject(projName)
project.fixLinks(searchDir)
def _visitTests(self, tests, newItemCallback):
""" Show the list of tests available """
mode = self.mode
assert mode in ['modules', 'classes', 'all'], 'Unknown mode %s' % mode
# First flatten the list of tests.
testsFlat = []
toCheck = [t for t in tests]
while toCheck:
test = toCheck.pop()
if isinstance(test, unittest.TestSuite):
toCheck += [t for t in test]
else:
testsFlat.append(test)
# Follow the flattened list of tests and show the module, class
# and name, in a nice way.
lastClass = None
lastModule = None
for t in testsFlat:
moduleName, className, testName = t.id().rsplit('.', 2)
# If there is a failure loading the test, show it
if moduleName.startswith('unittest.loader.ModuleImportFailure'):
print pwutils.red(moduleName), " test:", t.id()
continue
if moduleName != lastModule:
lastModule = moduleName
newItemCallback(MODULE, moduleName)
#print "scipion test %s" % moduleName
if mode in ['classes', 'all'] and className != lastClass:
lastClass = className
newItemCallback(CLASS, "%s.%s" % (moduleName, className))
#print " scipion test %s.%s" % (moduleName, className)
if mode == 'all':
newItemCallback(TEST,
"%s.%s.%s" % (moduleName, className, testName))
def _visitTests(self, moduleName, tests, newItemCallback):
""" Show the list of tests available """
mode = self.mode
assert mode in ['modules', 'classes', 'onlyclasses', 'all'], 'Unknown mode %s' % mode
# First flatten the list of tests.
# testsFlat = list(iter(self.__iterTests(tests)))
testsFlat = []
toCheck = [t for t in tests]
while toCheck:
test = toCheck.pop()
if isinstance(test, unittest.TestSuite):
toCheck += [t for t in test]
else:
testsFlat.append(test)
# Follow the flattened list of tests and show the module, class
# and name, in a nice way.
lastClass = None
lastModule = None
if testsFlat:
for t in testsFlat:
testModuleName, className, testName = t.id().rsplit('.', 2)
# If there is a failure loading the test, show it
errorStr = 'unittest.loader.ModuleImportFailure.'
if testModuleName.startswith(errorStr):
newName = t.id().replace(errorStr, '')
if self._match(newName):
print(pwutils.red('Error loading the test. Please, run the test for more information:'), newName)
continue
if testModuleName != lastModule:
lastModule = testModuleName
if mode != 'onlyclasses':
newItemCallback(MODULE, "%s" % testModuleName)
if mode in ['classes', 'onlyclasses', 'all'] and className != lastClass:
lastClass = className
newItemCallback(CLASS, "%s.%s"
% (testModuleName, className))
if mode == 'all':
newItemCallback(TEST, "%s.%s.%s"
% (testModuleName, className, testName))
else:
if not self.grep:
print(pwutils.green(' The plugin does not have any test'))
def openProject(projectName):
""" Opens a scipion project:
:param projectName: Name of a existing project to open,
or "here" to create a project in the current working dir,
or "last" to open the most recent project
"""
manager = Manager()
projName = os.path.basename(projectName)
# Handle special name 'here' to create a project
# from the current directory
if projName == 'here':
cwd = Config.SCIPION_CWD
if " " in cwd:
print("Projects can't have spaces in the name: %s" % cwd)
sys.exit(1)
print("\nYou are trying to create a project here:", pwutils.cyan(cwd))
if os.listdir(cwd):
print(pwutils.red('\nWARNING: this folder is not empty!!!'))
key = input("\nDo you want to create a project here? [y/N]?")
if key.lower().strip() != 'y':
print("\nAborting...")
sys.exit(0)
else:
print("\nCreating project....")
projName = os.path.basename(cwd)
projDir = os.path.dirname(cwd)
manager.createProject(projName, location=projDir)
elif projName == 'last': # Get last project
projects = manager.listProjects()
if not projects:
sys.exit("No projects yet, cannot open the last one.")
projName = projects[0].projName
projPath = manager.getProjectPath(projName)
projWindow = ProjectWindow(projPath)
projWindow.show()
""" % error
sys.exit(1)
argc = len(sys.argv)
if argc < 3 or argc > 4:
usage("Incorrect number of input parameters")
projName = sys.argv[1]
protId = sys.argv[2]
manager = Manager()
if not manager.hasProject(projName):
usage("Unexistent project: %s" % pwutils.red(projName))
project = manager.loadProject(projName)
prot = project.getProtocol(protId)
if prot is None:
usage("Unexistent protocol: %s" % protId)
outputParticles = getattr(prot, 'outputParticles', None)
if outputParticles is None:
usage("Protocol does not have 'outputParticles'")
realDims = outputParticles.getDimensions()
print "Real dimensions: ", realDims
def usage(error):
print("""
ERROR: %s
Usage: fixlinks.py PROJECT SEARCH_DIR
PROJECT: provide the project name to fix broken links in the imports.
SEARCH_DIR: provide a directory where to look for the files.
and fix the links.
""" % error)
sys.exit(1)
if len(sys.argv) != 3:
usage("Incorrect number of input parameters")
projName = sys.argv[1]
searchDir = os.path.abspath(sys.argv[2])
# Create a new project
manager = Manager()
if not manager.hasProject(projName):
usage("Nonexistent project: %s" % pwutils.red(projName))
if not os.path.exists(searchDir):
usage("Nonexistent SEARCH_DIR: %s" % pwutils.red(searchDir))
project = manager.loadProject(projName)
project.fixLinks(searchDir)
def usage(error):
print("""
ERROR: %s
stack2Volume will swap the dimension in the header of stack to make them
volumes. Something like 10 x 1 x 10 x 10 will be change to 1 x 10 x 10 x 10
Usage: stack2volume.py PATH
PATH: path to look for stack files
""" % error)
sys.exit(1)
if len(sys.argv) != 2:
usage("Incorrect number of input parameters")
path = sys.argv[1]
print("Looking for files like: %s" % path)
for file in glob(path):
print("Changing header of %s" % file)
try:
header = Ccp4Header(file, readHeader=True)
# Flag it as volume.
header.setISPG(401)
header.writeHeader()
except Exception as e:
print(pwutils.red("Failed to change header: % s" % e))
def step(self):
valuesDict = {}
valuesDict['table'] = self._tableName
cpu = valuesDict['cpu'] = psutil.cpu_percent(interval=0)
mem = valuesDict['mem'] = psutil.virtual_memory().percent
swap = valuesDict['swap'] = psutil.swap_memory().percent
# some code examples:
# https://github.com/ngi644/datadog_nvml/blob/master/nvml.py
if self.doGpu:
for i in self.gpusToUse:
try:
handle = nvmlDeviceGetHandleByIndex(i)
memInfo = nvmlDeviceGetMemoryInfo(handle)
valuesDict["gpuMem_%d" % i] = \
float(memInfo.used)*100./float(memInfo.total)
util = nvmlDeviceGetUtilizationRates(handle)
valuesDict["gpuUse_%d" % i] = util.gpu
temp = nvmlDeviceGetTemperature(handle,
NVML_TEMPERATURE_GPU)
valuesDict["gpuTem_%d" % i] = temp
except NVMLError as err:
msg = "ERROR monitoring GPU %d: %s." \
" Remove device %d from FORM" % (i, err, i)
print(red(msg))
if self.doNetwork:
try:
# measure a sort interval
pnic_before = psutil.net_io_counters(pernic=True)[self.nif]
time.sleep(self.samplingTime) # sec
pnic_after = psutil.net_io_counters(pernic=True)[self.nif]
bytes_sent = pnic_after.bytes_sent - pnic_before.bytes_sent
bytes_recv = pnic_after.bytes_recv - pnic_before.bytes_recv
valuesDict["%s_send" % self.nif] = \
bytes_sent * self.samplingTime / 1048576
valuesDict["%s_recv" % self.nif] = \
bytes_recv * self.samplingTime / 1048576
except Exception as ex:
msg = "cannot get information of network interface %s" % \
self.nif
if self.doDiskIO:
try:
# measure a sort interval
disk_before = psutil.disk_io_counters(perdisk=False)
time.sleep(self.samplingTime) # sec
disk_after = psutil.disk_io_counters(perdisk=False)
bytes_read = disk_after.read_bytes - disk_before.read_bytes
bytes_write = disk_after.write_bytes - disk_before.write_bytes
valuesDict["disk_read"] = \
self.samplingTime * bytes_read / self.mega
valuesDict["disk_write"] = \
self.samplingTime * bytes_write / self.mega
except Exception as ex:
msg = "cannot get information of disk usage "
if self.cpuAlert < 100 and cpu > self.cpuAlert:
self.warning("CPU allocation =%f." % cpu)
self.cpuAlert = cpu
if self.memAlert < 100 and mem > self.memAlert:
self.warning("Memory allocation =%f." % mem)
self.memAlert = mem
if self.swapAlert < 100 and swap > self.swapAlert:
self.warning("SWAP allocation =%f." % swap)
self.swapAlert = swap
sqlCommand = "INSERT INTO %(table)s ("
for label in self.labelList:
sqlCommand += "%s, " % label
# remove last comma
sqlCommand = sqlCommand[:-2]
sqlCommand += ") VALUES("
for label in self.labelList:
sqlCommand += "%" + "(%s)f, " % label
# remove last comma
sqlCommand = sqlCommand[:-2]
sqlCommand += ");"
sql = sqlCommand % valuesDict
try:
self.cur.execute(sql)
except Exception as e:
print("ERROR: saving one data point (monitor). I continue")
# Return finished = True if all protocols have finished
finished = []
for prot in self.protocols:
updatedProt = getUpdatedProtocol(prot)
finished.append(updatedProt.getStatus() != STATUS_RUNNING)
return all(finished)
def update(dataset, workingCopy=None, url=None, verbose=False):
""" Update local dataset with the contents of the remote one.
It compares the md5 of remote files in url/dataset/MANIFEST with the
ones in workingCopy/dataset/MANIFEST, and downloads only when necessary.
"""
# Get default values for variables if we got None.
workingCopy = workingCopy or pw.Config.SCIPION_TESTS
# Verbose log
def vlog(txt):
sys.stdout.write(txt) if verbose else None
# Read contents of *remote* MANIFEST file, and create a dict {fname: md5}
manifest = urlopen('%s/%s/MANIFEST' % (url, dataset)).readlines()
md5sRemote = dict(x.decode("utf-8").strip().split() for x in manifest)
# Update and read contents of *local* MANIFEST file, and create a dict
datasetFolder = join(workingCopy, dataset)
try:
last = max(
os.stat(join(datasetFolder, x)).st_mtime for x in md5sRemote)
t_manifest = os.stat(join(datasetFolder, 'MANIFEST')).st_mtime
assert t_manifest > last and time.time() - t_manifest < 60 * 60 * 24 * 7
except (OSError, IOError, AssertionError) as e:
print("Regenerating local MANIFEST...")
createMANIFEST(datasetFolder)
md5sLocal = dict(x.strip().split()
for x in open(join(datasetFolder, 'MANIFEST')))
# Check that all the files mentioned in MANIFEST are up-to-date
print("Verifying MD5s...")
filesUpdated = 0 # number of files that have been updated
taintedMANIFEST = False # can MANIFEST be out of sync?
downloadingPrinted = False
for fname in md5sRemote:
fpath = join(datasetFolder, fname)
try:
if exists(fpath) and md5sLocal[fname] == md5sRemote[fname]:
vlog("\r %s %s\n" % (green("OK"), fname))
pass # just to emphasize that we do nothing in this case
else:
if not downloadingPrinted:
verboseMsg = " Next time use -v for more details." if not verbose else ""
print("Differences detected. Downloading data.%s" %
verboseMsg)
vlog("\r %s %s (downloading... " % (red("XX"), fname))
if not isdir(dirname(fpath)):
os.makedirs(dirname(fpath))
urlretrieve('%s/%s/%s' % (url, dataset, fname), fpath)
vlog("done)\n")
filesUpdated += 1
except Exception as e:
print("\nError while updating %s: %s" % (fname, e))
taintedMANIFEST = True # if we don't update, it can be wrong
print("...done. Updated files: %d" % filesUpdated)
# Save the new MANIFEST file in the folder of the downloaded dataset
if filesUpdated > 0:
open(join(datasetFolder, 'MANIFEST'), 'w').writelines(md5sRemote)
if taintedMANIFEST:
print(
"Some files could not be updated. Regenerating local MANIFEST ...")
createMANIFEST(datasetFolder)
usage("The protocol class names to be ignored must be after a '--ignore' flag.")
projName = sys.argv[1]
# This fails, since it is triggering matplotlib.pyplot and then import error happens:
# ... pyworkflow/gui/no-tkinter/_tkinter.py: invalid ELF header. If we want this back we might need to
# invest some time "faking" tkinter again for python3.
# path = pw.join('gui', 'no-tkinter')
# sys.path.insert(1, path)
# Create a new project
manager = Manager()
if not manager.hasProject(projName):
usage("There is no project with this name: %s"
% pwutils.red(projName))
# the project may be a soft link which may be unavailable to the cluster so get the real path
try:
projectPath = os.readlink(manager.getProjectPath(projName))
except:
projectPath = manager.getProjectPath(projName)
project = Project(pw.Config.getDomain(), projectPath)
project.load()
runs = project.getRuns()
# Now assuming that there is no dependencies between runs
# and the graph is lineal
for prot in runs:
n = len(sys.argv)
if n <>2:
usage("This script accepts 1 parameter: the project name.")
projName = sys.argv[1]
path = os.path.join(os.environ['SCIPION_HOME'], 'pyworkflow', 'gui', 'no-tkinter')
sys.path.insert(1, path)
# Create a new project
manager = Manager()
if not manager.hasProject(projName):
usage("There is no project with this name: %s"
% pwutils.red(projName))
# the project may be a soft link which may be unavailable to the cluster so get the real path
try:
projectPath = os.readlink(manager.getProjectPath(projName))
except:
projectPath = manager.getProjectPath(projName)
project = Project(projectPath)
project.load()
runs = project.getRuns()
# Now assuming that there is no dependencies between runs
# and the graph is lineal
for prot in runs:
argc = len(sys.argv)
if argc < 3 or argc > 4:
usage("Incorrect number of input parameters")
projName = sys.argv[1]
protId = sys.argv[2]
n = int(sys.argv[3]) if argc == 4 else 3
# Create a new project
manager = Manager()
if not manager.hasProject(projName):
usage("Unexistent project: %s" % pwutils.red(projName))
project = manager.loadProject(projName)
protExtractParts = project.getProtocol(protId)
protVol = project.getProtocol(1256)
protRelionAllParts = project.getProtocol(1291)
protRelionBestParts = project.getProtocol(1359)
project.launchProtocol(protRelionAllParts, wait=True)
project.launchProtocol(protRelionBestParts, wait=True)
for i in range(n):
protSubSet = project.newProtocol(em.ProtSubSet,
objLabel='Subset #%d' %n,
def highlightStr(self, hstr):
for k in self.keywords:
hstr = hstr.replace(k, pwutils.red(k))
return hstr
except ImportError:
pass
if len(sys.argv) > 1:
manager = Manager()
projName = os.path.basename(sys.argv[1])
# Handle special name 'here' to create a project
# from the current directory
if projName == 'here':
cwd = os.environ['SCIPION_CWD']
print "\nYou are trying to create a project here:", pwutils.cyan(
cwd)
if os.listdir(cwd):
print pwutils.red('\nWARNING: this folder is not empty!!!')
key = raw_input("\nDo you want to create a project here? [y/N]?")
if key.lower().strip() != 'y':
print "\nAborting..."
sys.exit(0)
else:
print "\nCreating project...."
projName = os.path.basename(cwd)
projDir = os.path.dirname(cwd)
proj = manager.createProject(projName, location=projDir)
elif projName == 'last': # Get last project
projects = manager.listProjects()
if not projects:
sys.exit("No projects yet, cannot open the last one.")
sys.exit(1)
n = len(sys.argv)
if n < 2 or n > 4:
usage("Incorrect number of input parameters")
projName = sys.argv[1]
jsonFile = None if n < 3 else os.path.abspath(sys.argv[2])
location = None if n < 4 else sys.argv[3]
path = os.path.join(os.environ['SCIPION_HOME'], 'pyworkflow', 'gui',
'no-tkinter')
sys.path.insert(1, path)
# Create a new project
manager = Manager()
if manager.hasProject(projName):
usage("There is already a project with this name: %s" %
pwutils.red(projName))
if jsonFile is not None and not os.path.exists(jsonFile):
usage("Unexistent json file: %s" % pwutils.red(jsonFile))
project = manager.createProject(projName, location=location)
if jsonFile is not None:
protDict = project.loadProtocols(jsonFile)
""" % error
sys.exit(1)
n = len(sys.argv)
if n < 2 or n > 4:
usage("Incorrect number of input parameters")
projName = sys.argv[1]
jsonFile = None if n < 3 else os.path.abspath(sys.argv[2])
location = None if n < 4 else sys.argv[3]
path = os.path.join(os.environ['SCIPION_HOME'], 'pyworkflow', 'gui', 'no-tkinter')
sys.path.insert(1, path)
# Create a new project
manager = Manager()
if manager.hasProject(projName):
usage("There is already a project with this name: %s"
% pwutils.red(projName))
if jsonFile is not None and not os.path.exists(jsonFile):
usage("Unexistent json file: %s" % pwutils.red(jsonFile))
project = manager.createProject(projName, location=location)
if jsonFile is not None:
protDict = project.loadProtocols(jsonFile)
and fix the links.
""" % error
sys.exit(1)
if len(sys.argv) != 3:
usage("Incorrect number of input parameters")
projName = sys.argv[1]
searchDir = os.path.abspath(sys.argv[2])
# Create a new project
manager = Manager()
if not manager.hasProject(projName):
usage("Unexistent project: %s" % pwutils.red(projName))
if not os.path.exists(searchDir):
usage("Unexistent SEARCH_DIR: %s" % pwutils.red(searchDir))
project = manager.loadProject(projName)
runs = project.getRuns()
for prot in runs:
broken = False
if isinstance(prot, em.ProtImport):
for _, attr in prot.iterOutputEM():
fn = attr.getFiles()
for f in attr.getFiles():
if ':' in f:
def highlightStr(self, hstr):
for k in self.keywords:
hstr = hstr.replace(k, pwutils.red(k))
return hstr
def chimeraClashesStep(self):
labelDictAux = json.loads(self.chainStructure.get(),
object_pairs_hook=collections.OrderedDict)
labelDict = collections.OrderedDict(
sorted(labelDictAux.items(), key=itemgetter(1)))
# labelDict = collections.OrderedDict(sorted(list(labelDictAux.items()), key=itemgetter(1)))
pdbFileName = os.path.abspath(
self.pdbFileToBeRefined.get().getFileName())
# first element of dictionary
firstValue = labelDict[list(labelDict)[0]]
outFiles = []
f = open(self.getChimeraScriptFileName1(), "w")
f.write("from chimerax.core.commands import run\n")
f.write("run(session, 'open {}')\n".format(pdbFileName))
if self.sym == "Cn" and self.symOrder != 1:
f.write("run(session,'sym #1 C%d copies t')\n" % self.symOrder)
elif self.sym == "Dn" and self.symOrder != 1:
f.write("run(session,'sym #1 d%d copies t')\n" % self.symOrder)
elif self.sym == "T222" or self.sym == "TZ3":
f.write("run(session,'sym #1 t,%s copies t')\n" % self.sym[1:])
elif self.sym == "O":
f.write("run(session,'sym #1 O copies t')\n")
elif self.sym == "I222" or self.sym == "I222r" or self.sym == "In25" or \
self.sym == "In25r" or self.sym == "I2n3" or self.sym == "I2n3r" or \
self.sym == "I2n5" or self.sym == "I2n5r":
f.write("run(session,'sym #1 i,%s copies t')\n" % self.sym[1:])
self.SYMMETRY = self.SYMMETRY.get()
if self.SYMMETRY:
f.write("run(session,'delete #2 & #1 #>3')\n")
f.write("run(session,'save {symmetrizedModelName} #2')\n".format(
symmetrizedModelName=self.getSymmetrizedModelName()))
f.write("run(session, 'close #1')\n")
f.write("run(session, 'rename #2 id #1')\n")
self.endChimeraScript(firstValue, labelDict, outFiles, f)
f.write("run(session, 'exit')\n")
f.close()
args = " --nogui --script " + self.getChimeraScriptFileName1()
self._log.info('Launching: ' + Plugin.getProgram() + ' ' + args)
Chimera.runProgram(Plugin.getProgram(), args)
if self.SYMMETRY and not os.path.exists(
self.getSymmetrizedModelName()):
# When self.SYMMETRY = TRUE and no one neighbor unit cell has not been
# generated at less than 3 Angstroms, probably because the symmetry
# center is not equal to the origin of coordinates, at least we have the
# contacts that are within the unit cell.
print(
red("Error: No neighbor unit cells are available. "
"Is the symmetry center equal to the origin of "
"coordinates?"))
self.SYMMETRY = False
f = open(self.getChimeraScriptFileName2(), "w")
f.write("from chimerax.core.commands import run\n")
f.write("session, run('open {}')\n".format(pdbFileName))
self.endChimeraScript(firstValue, labelDict, outFiles, f)
f.write("run(session, 'exit')\n")
f.close()
args = " --nogui --script " + self.getChimeraScriptFileName2()
self._log.info('Launching: ' + Plugin.getProgram() + ' ' + args)
Chimera.runProgram(Plugin.getProgram(), args)
# parse all files created by chimera
c, conn = self.prepareDataBase()
self.parseFiles(outFiles, c)
conn.commit()
conn.close()
def terminate(self):
if self.process != None:
self.process.terminate()
print pwutils.red('Ctrl-c pressed, aborting this test')
n = len(sys.argv)
if n <> 2:
usage("This script accepts 1 parameter: the project name.")
projName = sys.argv[1]
path = os.path.join(os.environ['SCIPION_HOME'], 'pyworkflow', 'gui',
'no-tkinter')
sys.path.insert(1, path)
# Create a new project
manager = Manager()
if not manager.hasProject(projName):
usage("There is no project with this name: %s" % pwutils.red(projName))
# the project may be a soft link which may be unavailable to the cluster so get the real path
try:
projectPath = os.readlink(manager.getProjectPath(projName))
except:
projectPath = manager.getProjectPath(projName)
project = Project(projectPath)
project.load()
runs = project.getRuns()
# Now assuming that there is no dependencies between runs
# and the graph is lineal
for prot in runs:
if len(sys.argv) != 3:
usage("Incorrect number of input parameters")
projName = sys.argv[1]
searchDir = sys.argv[2]
#!/usr/bin/env python
# Create a new project
manager = Manager()
if not manager.hasProject(projName):
usage("Unexistent project: %s" % pwutils.red(projName))
if not os.path.exists(searchDir):
usage("Unexistent SEARCH_DIR: %s" % pwutils.red(searchDir))
project = manager.loadProject(projName)
runs = project.getRuns()
for prot in runs:
broken = False
if isinstance(prot, em.ProtImport):
for _, attr in prot.iterOutputEM():
fn = attr.getFiles()
for f in attr.getFiles():
if not os.path.exists(f):
def terminate(self):
if self.process != None:
self.process.terminate()
print pwutils.red('Ctrl-c pressed, aborting this test')
signal(SIGUSR2, lambda sig, frame: start_embedded_debugger('a'))
except ImportError:
pass
if len(sys.argv) > 1:
manager = Manager()
projName = os.path.basename(sys.argv[1])
# Handle special name 'here' to create a project
# from the current directory
if projName == 'here':
cwd = os.environ['SCIPION_CWD']
print "\nYou are trying to create a project here:", pwutils.cyan(cwd)
if os.listdir(cwd):
print pwutils.red('\nWARNING: this folder is not empty!!!')
key = raw_input("\nDo you want to create a project here? [y/N]?")
if key.lower().strip() != 'y':
print "\nAborting..."
sys.exit(0)
else:
print "\nCreating project...."
projName = os.path.basename(cwd)
projDir = os.path.dirname(cwd)
proj = manager.createProject(projName, location=projDir)
elif projName == 'last': # Get last project
projects = manager.listProjects()
if not projects:
sys.exit("No projects yet, cannot open the last one.")
