def runJavaIJapp(memory, appName, args, env=None): env = env or {} getEnviron = Config.getDomain().importFromPlugin('xmipp3', 'Plugin').getEnviron env.update(getEnviron(xmippFirst=False)) # Add scipion variables env[SCIPION_DOMAIN] = Config.SCIPION_DOMAIN env[SCIPION_HOME_VAR]= Config.SCIPION_HOME if Config.debugOn(): env[SCIPION_DEBUG] = "1" # Add out python path to the PATH env["PATH"] = ":".join([sys.executable, env.get("PATH", "")]) # Try chimera try: chimeraPlugin = Config.getDomain().importFromPlugin('chimera', 'Plugin', doRaise=True) env["CHIMERA_LAUNCHER"] = chimeraPlugin.getProgram() + " %s" except Exception as e: pass args = getJavaIJappArguments(memory, appName, args) print('java %s' % args) # return subprocess.Popen('java ' + args, shell=True, env=env) cmd = ['java'] + shlex.split(args) return subprocess.Popen(cmd, env=env)
def setUpClass(cls): pwtests.setupTestOutput(cls) # Set the application domain Config.setDomain("pyworkflowtests") pwtests.setupTestProject(cls)
def main(): program = sys.argv[1] params = ' '.join('"%s"' % x for x in sys.argv[2:]) # Initialize plugin discovery and initialization Config.getDomain().getPlugins() runProgram(program, params)
def getEnviron(cls, xmippFirst=True): """ Create the needed environment for Xmipp programs. """ environ = pwutils.Environ(os.environ) pos = pwutils.Environ.BEGIN if xmippFirst else pwutils.Environ.END environ.update({ 'PATH': pwem.Config.CUDA_BIN, 'LD_LIBRARY_PATH': pwem.Config.CUDA_LIB }, position=pwutils.Environ.END) if os.path.isfile(getXmippPath('xmippEnv.json')): with open(getXmippPath('xmippEnv.json'), 'r') as f: compilationEnv = json.load(f) environ.update(compilationEnv, position=pos) environ.update({ 'PATH': getXmippPath('bin'), 'LD_LIBRARY_PATH': getXmippPath('lib'), 'PYTHONPATH': getXmippPath('pylib') }, position=pos) environ['XMIPP_HOME'] = getXmippPath() # Add path to python lib folder environ.addLibrary(Config.getPythonLibFolder()) return environ
def _drawNode(self, node): """ Allocate node with x=0 and y=0. """ try: parents = node.getParents() if not parents: print("EMPTY NODE ask JM") return maxParent = parents[0] for p in parents[1:]: if p.y > maxParent.y: maxParent = p siblings = maxParent.getChilds() if len(siblings) == 1: node.x = maxParent.x node.y = maxParent.y + self.DY else: rightSibling = siblings[0] for s in siblings: if s.x > rightSibling.x: rightSibling = s node.x = rightSibling.x + rightSibling.width / 2 + self.DX + node.width / 2 node.y = rightSibling.y except Exception as e: from pyworkflow.utils import envVarOn if Config.debugOn(): print("Can't draw node: %s" % node, e) import traceback traceback.print_stack() else: # Do nothing return
def _getPlugin(self): if self._plugin is None: try: dirname = self.getDirName() self._plugin = Config.getDomain().getPlugin(dirname) except: pass return self._plugin
def startDebugger(password='******'): if Config.debugOn(): try: # FIXME: rpdb2 does not support python 3 from rpdb2 import start_embedded_debugger print("Starting debugger...") start_embedded_debugger(password) except Exception: print( "Error importing rpdb2 debugging module, consider installing winpdb." )
def addScipionTemplates(self, tempId=None): # Check if there is any .json.template in the template folder # get the template folder (we only want it to be included once) templateFolder = Config.getExternalJsonTemplates() for templateName in glob.glob1(templateFolder, "*" + SCIPION_JSON_TEMPLATES): t = Template("local", os.path.join(templateFolder, templateName)) if tempId is not None: if t.getObjId() == tempId: self.addTemplate(t) break else: self.addTemplate(t)
def runCommand(command, env=None, cwd=None): """ Execute command with given environment env and directory cwd """ # First let us create core dumps if in debug mode if Config.debugOn(env): import resource resource.setrlimit(resource.RLIMIT_CORE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY)) # This is like "ulimit -u 99999999", so we can create core dumps # TODO: maybe have to set PBS_NODEFILE in case it is used by "command" # (useful for example with gnu parallel) check_call(command, shell=True, stdout=sys.stdout, stderr=sys.stderr, env=env, cwd=cwd)
def addPluginTemplates(self, tempId=None): """ Get the templates provided by all plugins. :return: a list of templates """ # Check if other plugins have json.templates domain = Config.getDomain() # Check if there is any .json.template in the template folder # get the template folder (we only want it to be included once) for pluginName, pluginModule in domain.getPlugins().items(): tempListPlugin = pluginModule.Plugin.getTemplates() for t in tempListPlugin: if tempId is not None: if t.getObjId() == tempId: self.addTemplate(t) break else: self.addTemplate(t)
def evalParamCondition(self, paramName): """Evaluate if a condition is True for a give param with the values of a particular Protocol""" param = self.getParam(paramName) if not param.hasCondition(): return True condStr = param.condition.get() localDict = {} globalDict = dict(globals()) # FIXME: Check why this import is here from pyworkflow import Config globalDict.update(Config.getDomain().getObjects()) for t in param._conditionParams: if self.hasParam(t) or self._protocol.hasAttribute(t): localDict[t] = self._protocol.getAttributeValue(t) return eval(condStr, globalDict, localDict)
while numParents == 1 and classRef is not Protocol: classRef = classRef.__bases__[0] numParents = len(classRef.__bases__) if numParents > 1: return True else: return False if __name__ == '__main__': count = 0 withDoc = '--with-doc' in sys.argv asciidoc = '--asciidoc' in sys.argv extended = '--extended' in sys.argv emProtocolsDict = Config.getDomain().getProtocols() emCategories = [('Imports', ProtImport, []), ('Micrographs', ProtMicrographs, []), ('Particles', ProtParticles, []), ('2D', Prot2D, []), ('3D', Prot3D, [])] protDict = {} # Group protocols by package name for k, v in emProtocolsDict.items(): packageName = v.getClassPackageName() if packageName not in protDict: protDict[packageName] = [] if not issubclass(v, Viewer) and not v.isBase(): if extended:
else: if len(sys.argv) == 1: # no extra arguments, show current directory showDir(os.getcwd()) else: args = {'-i': []} lastOpt = '-i' for a in sys.argv[1:]: if a.startswith('-'): lastOpt = a if lastOpt not in args: args[lastOpt] = [] else: args[lastOpt].append(a) inputFiles = args['-i'] del args['-i'] viewParams = {} for k, v in args.items(): viewParams[k.replace('-', '')] = ' '.join(v) # Trigger plugin initialization Config.getDomain().getPlugins() for fn in inputFiles: if os.path.isdir(fn): showDir(fn) else: showFile(fn, viewParams)
def main(self, args=None): print("Running tests....") # Trigger plugin's variable definition Config.getDomain().getPlugins() parser = argparse.ArgumentParser(prog=self.getTestsCommand(), description=__doc__) g = parser.add_mutually_exclusive_group() g.add_argument('--run', action='store_true', help='run the selected tests') g.add_argument('--show', action='store_true', help='show available tests', default=True) add = parser.add_argument # shortcut add('--pattern', default='test*.py', help='pattern for the files that will be used in the tests') add('--grep', default=None, nargs='+', help='only show/run tests containing the provided words') add('--skip', default=None, nargs='+', help='skip tests that contains these words') add('--log', default=None, nargs='?', help="Generate logs files with the output of each test.") add('--mode', default='classes', choices=['modules', 'classes', 'onlyclasses', 'all'], help='how much detail to give in show mode') add('tests', metavar='TEST', nargs='*', help='test case from string identifier (module, class or callable)' ) args = parser.parse_args(args) if not args.run and not args.show and not args.tests: sys.exit(parser.format_help()) testsDict = OrderedDict() testLoader = unittest.defaultTestLoader if args.tests: # In this case the tests are passed as argument. # The full name of the test will be used to load it. testsDict['tests'] = unittest.TestSuite() for t in args.tests: try: testsDict['tests'].addTests( testLoader.loadTestsFromName(t)) except Exception as e: print('Cannot load test %s -- skipping' % t) import traceback traceback.print_exc() else: # In this other case, we will load the test available # from pyworkflow and the other plugins # self.paths = [('pyworkflow', os.path.dirname(os.path.dirname(pw.__file__)))] self.paths = [] for name, plugin in pw.Config.getDomain().getPlugins().items(): self.paths.append((name, os.path.dirname(plugin.__path__[0]))) for k, p in self.paths: testPath = os.path.join(p, k, 'tests') if os.path.exists(testPath): testsDict[k] = testLoader.discover(testPath, pattern=args.pattern, top_level_dir=p) self.grep = [g.lower() for g in args.grep] if args.grep else [] self.skip = args.skip self.mode = args.mode self.log = args.log if args.tests: self.runSingleTest(testsDict['tests']) elif args.run: for moduleName, tests in testsDict.items(): print(pwutils.cyan(">>>> %s" % moduleName)) self.runTests(moduleName, tests) elif args.grep: pattern = args.grep[0] for moduleName, tests in testsDict.items(): self.printTests(pattern, tests) else: for moduleName, tests in testsDict.items(): if self._match(moduleName): print(pwutils.cyan(">>>> %s" % moduleName)) self.printTests(moduleName, tests)
def runJobMPISlave(mpiComm): """ This slave will be receiving commands to execute until 'None' is received. """ rank = mpiComm.Get_rank() hostname = getLocalHostName() print(" Running MPIWorker: ", rank) exitResult = 0 # Listen for commands until we get 'None' cwd = None # We run without changing directory by default env = None # And we don't change the environment either! while True: # Receive command in a non-blocking way req_recv = mpiComm.irecv(source=0, tag=TAG_RUN_JOB + rank) while True: done, command = req_recv.test() if done: break sleep(1) print(" Worker %s(rank %d) received command." % (hostname, rank)) # We need to convert to string because req_recv.test() returns bytes or None if command == 'None': print(" Stopping...") return else: command = loads(command) # Run the command and get the result (exit code or exception) try: if command.startswith("cwd="): cwd = command.split("=", 1)[-1] print(" Changing to dir %s ..." % cwd) elif command.startswith("env="): env = command.split("=", 1)[-1] env = eval(env) print(" Setting the environment...") if Config.debugOn(): print(env) else: runCommand(command, cwd=cwd, env=env) cwd = None # unset directory env = None # unset environment except Exception as e: print(" Error in process %d (rank %d)" % (os.getpid(), rank)) import traceback traceback.print_exc() exitResult = str(e) # Communicate to master, either error os success req_send = mpiComm.isend(exitResult, dest=0, tag=TAG_RUN_JOB + rank) t0 = time() while not req_send.test()[0]: sleep(1) if time() - t0 > TIMEOUT: msg = " Error in process %d, cannot send error message to master." print(msg % os.getpid()) break
def defineBinaries(cls, env): """ Define the Xmipp binaries/source available tgz. In addition, define extra software needed by some Xmipp methods such as deepLearningToolkit. Scipion-defined software can be used as dependencies by using its name as string. """ ## XMIPP SOFTWARE ## xmippDeps = [] # Deps should be at requirements.txt (old: scons, joblib, scikit_learn) # Installation vars for commands formating verToken = getXmippPath('v%s' % _currentVersion) confToken = getXmippPath("xmipp.conf") installVars = {'installedToken': "installation_finished", 'bindingsToken': "bindings_linked", 'verToken': verToken, 'nProcessors': env.getProcessors(), 'xmippHome': getXmippPath(), 'bindingsSrc': getXmippPath('bindings', 'python'), 'bindingsDst': Config.getBindingsFolder(), 'xmippLib': getXmippPath('lib', 'libXmipp.so'), 'coreLib': getXmippPath('lib', 'libXmippCore.so'), 'libsDst': Config.getLibFolder(), 'confToken': confToken, 'strPlaceHolder': '%s', # to be replaced in the future 'currVersion': _currentVersion } ## Installation commands (removing bindingsToken) installCmd = ("cd {cwd} && {configCmd} && {compileCmd} N={nProcessors:d} && " "ln -srfn build {xmippHome} && cd - && " "touch {installedToken} && rm {bindingsToken} 2> /dev/null") installTgt = [getXmippPath('bin', 'xmipp_reconstruct_significant'), getXmippPath("lib/libXmippJNI.so"), installVars['installedToken']] ## Linking bindings (removing installationToken) bindingsAndLibsCmd = ("find {bindingsSrc} -maxdepth 1 -mindepth 1 " r"! -name __pycache__ -exec ln -srfn {{}} {bindingsDst} \; && " "ln -srfn {coreLib} {libsDst} && " "touch {bindingsToken} && " "rm {installedToken} 2> /dev/null") bindingsAndLibsTgt = [os.path.join(Config.getBindingsFolder(), 'xmipp_base.py'), os.path.join(Config.getBindingsFolder(), 'xmippLib.so'), os.path.join(Config.getLibFolder(), 'libXmipp.so'), installVars['bindingsToken']] sourceTgt = [getXmippPath('xmipp.bashrc')] # Target for xmippSrc and xmippDev ## Allowing xmippDev if devel mode detected # plugin = scipion-em-xmipp <-- xmipp3 <-- __init__.py pluginDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # bundle = xmipp-bundle <- src <- scipion-em-xmipp bundleDir = os.path.dirname(os.path.dirname(pluginDir)) isPypiDev = os.path.isfile(os.path.join(pluginDir, 'setup.py')) isXmippBu = (os.path.isdir(os.path.join(bundleDir, 'src')) and os.path.isfile(os.path.join(bundleDir, 'xmipp'))) develMode = isPypiDev and isXmippBu if develMode: env.addPackage('xmippDev', tar='void.tgz', commands=[(installCmd.format(**installVars, cwd=bundleDir, configCmd='pwd', compileCmd='./xmipp all'), installTgt+sourceTgt), (bindingsAndLibsCmd.format(**installVars), bindingsAndLibsTgt)], deps=xmippDeps, default=False) avoidConfig = os.environ.get('XMIPP_NOCONFIG', 'False') == 'True' alreadyCompiled = os.path.isfile(getXmippPath('v'+_currentVersion)) # compilation token (see the xmipp script) configSrc = ('./xmipp check_config' if avoidConfig else './xmipp config noAsk && ./xmipp check_config') env.addPackage('xmippSrc', version=_currentVersion, # adding 'v' before version to fix a package target (post-link) tar='xmippSrc-v'+_currentVersion+'.tgz', commands=[(installCmd.format(**installVars, cwd='.', configCmd=configSrc, compileCmd='./xmipp compileAndInstall'), installTgt + sourceTgt), (bindingsAndLibsCmd.format(**installVars), bindingsAndLibsTgt)], deps=xmippDeps, default=not (develMode or alreadyCompiled)) ## EXTRA PACKAGES ## installDeepLearningToolkit(cls, env)
def installPluginMethods(): """ Deals with plugin installation methods""" # Trigger plugin's variable definition Config.getDomain().getPlugins() invokeCmd = SCIPION_CMD + " " + sys.argv[1] pluginRepo = PluginRepository() parser = argparse.ArgumentParser( formatter_class=argparse.RawTextHelpFormatter) subparsers = parser.add_subparsers( help='mode "installp", "uninstallp" or "listb"', dest='mode', title='Mode', description='available modes are "installp" or "uninstallp"') ############################################################################ # Install parser # ############################################################################ installParser = subparsers.add_parser( "installp", formatter_class=argparse.RawTextHelpFormatter, usage="%s [-h] [--noBin] [-p pluginName [pipVersion ...]]" % invokeCmd, epilog="Example: %s -p scipion-em-motioncorr 1.0.6 " "-p scipion-em-relion -p scipion-em-eman2 \n\n" % invokeCmd, add_help=False) installParser.add_argument('-h', '--help', action='store_true', help='show help') installParser.add_argument( '--noBin', action='store_true', help='Optional flag to install plugins only as a python module,\n' 'without installing the plugin binaries. This will affect\n' 'all plugins specified in the command.') installParser.add_argument( '--checkUpdates', action='store_true', help='Optional flag to check which plugins have new releases.\n') installParser.add_argument( '-p', '--plugin', action='append', nargs='+', metavar=('pluginName', 'pluginVersion'), help= '- pluginName: the name of the plugin to install from the list\n' ' of available plugins shown below.\n' '- pluginVersion: (optional) pip version to install. If not specified,\n' ' will install the latest compatible with current Scipion.' ) installParser.add_argument( '--devel', action='store_true', help= 'Optional flag to indicate that we will pass install sources instead\n' 'of pip names. Sources might be local paths or git urls. With local\n' 'paths, will do pip install -e (editable mode). It is expected to find\n' 'the plugin name in the basename of the path or in the repo name. \n' '(i.e. it needs to match the one specified in setup.py). E.g:\n' 'scipion installp -p path/to/pluginName --devel \n' 'scipion installp -p https://github.com/someOrg/pluginName.git --devel' ) installParser.add_argument('-j', default='1', metavar='j', help='Number of CPUs to use for compilation \n') ############################################################################ # Uninstall parser # ############################################################################ uninstallParser = subparsers.add_parser( "uninstallp", formatter_class=argparse.RawTextHelpFormatter, usage="%s [-h] [-p pluginName [binVersion ...]]" % invokeCmd, epilog="Example: %s -p scipion-em-eman2 scipion-em-motioncorr \n\n" % invokeCmd, add_help=False) uninstallParser.add_argument('-h', '--help', action='store_true', help='show help') uninstallParser.add_argument( '--noBin', action='store_true', help='Optional flag to uninstall plugins only as a python module,\n' 'without uninstalling the plugin binaries. This will affect\n' 'all plugins specified in the command.') uninstallParser.add_argument( '-p', '--plugin', action='append', metavar='pluginName', help='The name of the plugin to uninstall from the list\n' 'of available plugins shown below.\n') ############################################################################ # Install Bins parser # ############################################################################ installBinParser = subparsers.add_parser( "installb", formatter_class=argparse.RawTextHelpFormatter, usage="%s [-h] binName1 binName2-1.2.3 binName3 ..." % invokeCmd, epilog="Example: %s ctffind4 eman-2.3\n\n" % invokeCmd, add_help=False) # installBinParser.add_argument('pluginName', metavar='pluginName', # help='The name of the plugin whose bins we want to uninstall.\n') installBinParser.add_argument('-h', '--help', action='store_true', help='show help') installBinParser.add_argument( 'binName', nargs='*', metavar='binName(s)', help='The name(s) of the bins we want install, optionally with \n' 'version in the form name-version. If no version is specified,\n' 'will install the last one.') installBinParser.add_argument( '-j', default='1', metavar='j', help='Number of CPUs to use for compilation \n') ############################################################################ # Uninstall Bins parser # ############################################################################ uninstallBinParser = subparsers.add_parser( "uninstallb", formatter_class=argparse.RawTextHelpFormatter, usage="%s [-h] binName1 binName2-1.2.3 binName3 ..." % invokeCmd, epilog="Example: %s ctffind4 relion-3.0\n\n " % invokeCmd, add_help=False) # uninstallBinParser.add_argument('pluginName', metavar='pluginName', # help='The name of the plugin whose bins we want to uninstall.\n') uninstallBinParser.add_argument('-h', '--help', action='store_true', help='show help') uninstallBinParser.add_argument( 'binName', nargs='+', metavar='binName(s)', help='The name(s) of the bins we want to uninstall\n' '(optionally with version in the form name-version). \n' 'If no version is specified, will uninstall the last one.\n') modeToParser = { MODE_INSTALL_BINS: installBinParser, MODE_UNINSTALL_BINS: uninstallBinParser, MODE_INSTALL_PLUGIN: installParser, MODE_UNINSTALL_PLUGIN: uninstallParser } parsedArgs = parser.parse_args(sys.argv[1:]) mode = parsedArgs.mode parserUsed = modeToParser[mode] exitWithErrors = False if parsedArgs.help or (mode in [MODE_INSTALL_BINS, MODE_UNINSTALL_BINS] and len(parsedArgs.binName) == 0): if mode not in [MODE_INSTALL_BINS, MODE_UNINSTALL_BINS]: parserUsed.epilog += pluginRepo.printPluginInfoStr() else: env = Environment() env.setDefault(False) installedPlugins = Config.getDomain().getPlugins() for p, pobj in iteritems(installedPlugins): try: pobj.Plugin.defineBinaries(env) except Exception as e: print( redStr("Error retrieving plugin %s binaries: " % str(p)), e) parserUsed.epilog += env.printHelp() parserUsed.print_help() parserUsed.exit(0) elif mode == MODE_INSTALL_PLUGIN: if parsedArgs.checkUpdates: print(pluginRepo.printPluginInfoStr(withUpdates=True)) installParser.exit(0) if parsedArgs.devel: for p in parsedArgs.plugin: pluginSrc = p[0] pluginName = "" if os.path.exists(pluginSrc): pluginName = os.path.basename( os.path.abspath(pluginSrc).rstrip('/')) else: # we assume it is a git url m = re.match('https://github.com/(.*)/(.*).git', pluginSrc) if m: pluginName = m.group(2) if not pluginName: print("ERROR: Couldn't find pluginName for source %s" % pluginSrc) exitWithErrors = True else: plugin = PluginInfo(pipName=pluginName, pluginSourceUrl=pluginSrc, remote=False) numberProcessor = parsedArgs.j installed = plugin.installPipModule() if installed and not parsedArgs.noBin: plugin.installBin({'args': ['-j', numberProcessor]}) else: pluginsToInstall = list(zip(*parsedArgs.plugin))[0] pluginDict = pluginRepo.getPlugins(pluginList=pluginsToInstall, getPipData=True) if not pluginDict: exitWithErrors = True else: for cmdTarget in parsedArgs.plugin: pluginName = cmdTarget[0] pluginVersion = "" if len(cmdTarget) == 1 else cmdTarget[1] numberProcessor = parsedArgs.j plugin = pluginDict.get(pluginName, None) if plugin: installed = plugin.installPipModule( version=pluginVersion) if installed and not parsedArgs.noBin: plugin.installBin( {'args': ['-j', numberProcessor]}) else: print("WARNING: Plugin %s does not exist." % pluginName) exitWithErrors = True elif parsedArgs.mode == MODE_UNINSTALL_PLUGIN: for pluginName in parsedArgs.plugin: plugin = PluginInfo(pluginName, pluginName, remote=False) if plugin.isInstalled(): if not parsedArgs.noBin: plugin.uninstallBins() plugin.uninstallPip() else: print("WARNING: Plugin %s is not installed." % pluginName) elif parsedArgs.mode == MODE_INSTALL_BINS: binToInstallList = parsedArgs.binName binToPlugin = pluginRepo.getBinToPluginDict() for binTarget in binToInstallList: pluginTargetName = binToPlugin.get(binTarget, None) if pluginTargetName is None: print('ERROR: Could not find target %s' % binTarget) continue pmodule = Config.getDomain().getPlugin(pluginTargetName) numberProcessor = parsedArgs.j pinfo = PluginInfo(name=pluginTargetName, plugin=pmodule, remote=False) pinfo.installBin({'args': [binTarget, '-j', numberProcessor]}) elif parsedArgs.mode == MODE_UNINSTALL_BINS: binToInstallList = parsedArgs.binName binToPlugin = pluginRepo.getBinToPluginDict() for binTarget in binToInstallList: pluginTargetName = binToPlugin.get(binTarget, None) if pluginTargetName is None: print('ERROR: Could not find target %s' % binTarget) continue pmodule = Config.getDomain().getPlugin(pluginTargetName) pinfo = PluginInfo(name=pluginTargetName, plugin=pmodule, remote=False) pinfo.uninstallBins([binTarget]) if exitWithErrors: parserUsed.exit(1) else: parserUsed.exit(0)