def generate_gn_args(mode): out = [] if mode == "release": out += ["is_official_build=true"] elif mode == "debug": out += ["is_debug=true"] else: print "Bad mode {}. Use 'release' or 'debug' (default)" % mode sys.exit(1) if "DENO_BUILD_ARGS" in os.environ: out += os.environ["DENO_BUILD_ARGS"].split() cacher = prebuilt.load_sccache() if not os.path.exists(cacher): cacher = find_executable("sccache") or find_executable("ccache") # Check if ccache or sccache are in the path, and if so we set cc_wrapper. cc_wrapper = cacher if cc_wrapper: # The gn toolchain does not shell escape cc_wrapper, so do it here. out += ['cc_wrapper=%s' % gn_string(shell_quote(cc_wrapper))] if os.name == "nt": # Disable treat_warnings_as_errors until this sccache bug is fixed: # https://github.com/mozilla/sccache/issues/264 out += ["treat_warnings_as_errors=false"] # Look for sccache; if found, set rustc_wrapper. rustc_wrapper = cacher if rustc_wrapper: out += ['rustc_wrapper=%s' % gn_string(rustc_wrapper)] return out
def test_make_distribution(self): # check if tar and gzip are installed if (find_executable('tar') is None or find_executable('gzip') is None): return # now building a sdist dist, cmd = self.get_cmd() # creating a gztar then a tar cmd.formats = ['gztar', 'tar'] cmd.ensure_finalized() cmd.run() # making sure we have two files dist_folder = join(self.tmp_dir, 'dist') result = os.listdir(dist_folder) result.sort() self.assertEqual(result, ['fake-1.0.tar', 'fake-1.0.tar.gz']) os.remove(join(dist_folder, 'fake-1.0.tar')) os.remove(join(dist_folder, 'fake-1.0.tar.gz')) # now trying a tar then a gztar cmd.formats = ['tar', 'gztar'] cmd.ensure_finalized() cmd.run() result = os.listdir(dist_folder) result.sort() self.assertEqual(result, ['fake-1.0.tar', 'fake-1.0.tar.gz'])
def get(): launcher_val = overrides.get('CHPL_LAUNCHER') if not launcher_val: comm_val = chpl_comm.get() platform_val = chpl_platform.get('target') compiler_val = chpl_compiler.get('target') if platform_val.startswith('cray-x') or chpl_platform.is_cross_compiling(): has_aprun = find_executable('aprun') has_slurm = find_executable('srun') if has_aprun and has_slurm: launcher_val = 'none' elif has_aprun: launcher_val = 'aprun' elif has_slurm or platform_val == 'aarch64': launcher_val = 'slurm-srun' else: # FIXME: Need to detect aprun/srun differently. On a cray # system with an eslogin node, it is possible that aprun # will not be available on the eslogin node (only on the # login node). # # has_aprun and has_slurm should look other places # (maybe the modules?) to decide. # (thomasvandoren, 2014-08-12) sys.stderr.write( 'Warning: Cannot detect launcher on this system. Please ' 'set CHPL_LAUNCHER in the environment.\n') elif platform_val == 'marenostrum': launcher_val = 'marenostrum' elif comm_val == 'gasnet': substrate_val = chpl_comm_substrate.get() if substrate_val == 'udp': launcher_val = 'amudprun' elif substrate_val == 'mpi': launcher_val = 'gasnetrun_mpi' elif substrate_val == 'ibv': if platform_val == 'pwr6': # our loadleveler launcher is not yet portable/stable/ # flexible enough to serve as a good default #launcher_val = 'loadleveler' launcher_val = 'none' else: launcher_val = 'gasnetrun_ibv' elif substrate_val == 'mxm': launcher_val = 'gasnetrun_ibv' elif substrate_val == 'lapi': # our loadleveler launcher is not yet portable/stable/flexible # enough to serve as a good default #launcher_val = 'loadleveler' launcher_val = 'none' elif comm_val == 'mpi': launcher_val = 'mpirun' else: launcher_val = 'none' if launcher_val is None: launcher_val = 'none' return launcher_val
def find_binary_path(self,path=None, channel="nightly"): """Looks for the firefox binary in the virtual environment""" platform = { "Linux": "linux", "Windows": "win", "Darwin": "mac" }.get(uname[0]) application_name = { "stable": "Firefox.app", "beta": "Firefox.app", "nightly": "Firefox Nightly.app" }.get(channel) if path is None: #os.getcwd() doesn't include the venv path path = os.path.join(os.getcwd(), "_venv", "browsers", channel) binary = None if platform == "linux": binary = find_executable("firefox", os.path.join(path, "firefox")) elif platform == "win": import mozinstall binary = mozinstall.get_binary(path, "firefox") elif platform == "mac": binary = find_executable("firefox", os.path.join(path, application_name, "Contents", "MacOS")) return binary
def setup_sudoers(user): if not os.path.exists('/etc/sudoers.d'): os.makedirs('/etc/sudoers.d') set_permissions = False if not os.path.exists('/etc/sudoers'): set_permissions = True with open('/etc/sudoers', 'a') as f: f.write('\n#includedir /etc/sudoers.d\n') if set_permissions: os.chmod('/etc/sudoers', 0o440) sudoers_file = '/etc/sudoers.d/frappe' template = env.get_template('frappe_sudoers') frappe_sudoers = template.render(**{ 'user': user, 'service': find_executable('service'), 'systemctl': find_executable('systemctl'), 'supervisorctl': find_executable('supervisorctl'), 'nginx': find_executable('nginx'), 'bench': find_executable('bench') }) frappe_sudoers = safe_decode(frappe_sudoers) with open(sudoers_file, 'w') as f: f.write(frappe_sudoers) os.chmod(sudoers_file, 0o440)
def test_40_can_resize(self): '''Test for installed filesystem resize utility with CanResize''' offline_shrink = 0b00010 offline_grow = 0b00100 online_shrink = 0b01000 online_grow = 0b10000 manager = self.get_interface(self.manager_obj, '.Manager') with self.assertRaises(dbus.exceptions.DBusException): manager.CanResize('nilfs2') avail, mode, util = manager.CanResize('xfs') # the resize mode flage values are defined in the method documentation self.assertEqual(mode, online_grow | offline_grow) if avail: self.assertEqual(util, '') else: self.assertEqual(util, 'xfs_growfs') self.assertEqual(avail, find_executable('xfs_growfs') is not None) avail, mode, util = manager.CanResize('ext4') self.assertEqual(mode, offline_shrink | offline_grow | online_grow) if avail: self.assertEqual(util, '') else: self.assertEqual(util, 'resize2fs') self.assertEqual(avail, find_executable('resize2fs') is not None) avail, mode, util = manager.CanResize('vfat') self.assertTrue(avail) # libparted, no executable self.assertEqual(util, '') self.assertEqual(mode, offline_shrink | offline_grow)
def _findEditorTools(usdFileName, readOnly): from distutils.spawn import find_executable # Ensure the usdcat executable has been installed usdcatCmd = find_executable("usdcat") if not usdcatCmd: sys.exit("Error: Couldn't find 'usdcat'. Expected it to be in PATH.") # Ensure we have a suitable editor available editorCmd = (os.getenv("EDITOR") or find_executable("emacs") or find_executable("vim")) if not editorCmd: sys.exit("Error: Couldn't find a suitable text editor to use. Expected " "either $EDITOR to be set, or emacs/vim to be installed.") # special handling for emacs users if 'emacs' in editorCmd: title = '"usdedit %s%s"' % ("--noeffect " if readOnly else "", usdFileName) editorCmd += " -name %s" % title return (usdcatCmd, editorCmd)
def _test_deploy_remove(self, system): if system == "nssm": args = find_executable("python") or "c:\\python27\\python" else: args = find_executable("python2") or "/usr/bin/python2" init_system = {"--init-system=": system} opts = { "-n": self.service_name, "-a": "-m SimpleHTTPServer", "-d": None, "-s": None, "-v": None, "--overwrite": None, } opts.update(init_system) _invoke_click("generate", [args], opts) self._verify_port_open() if not utils.IS_WIN: _invoke_click("stop", [self.service_name], init_system) self._verify_port_closed() _invoke_click("start", [self.service_name], init_system) self._verify_port_open() _invoke_click("restart", [self.service_name], init_system) self._verify_port_open() _invoke_click("remove", [self.service_name], init_system) self._verify_port_closed()
def test_40_can_format(self): '''Test for installed filesystem creation utility with CanFormat''' manager = self.get_interface(self.manager_obj, '.Manager') with self.assertRaises(dbus.exceptions.DBusException): manager.CanFormat('wxyz') avail, util = manager.CanFormat('xfs') if avail: self.assertEqual(util, '') else: self.assertEqual(util, 'mkfs.xfs') self.assertEqual(avail, find_executable('mkfs.xfs') is not None) avail, util = manager.CanFormat('f2fs') if avail: self.assertEqual(util, '') else: self.assertEqual(util, 'mkfs.f2fs') self.assertEqual(avail, find_executable('mkfs.f2fs') is not None) avail, util = manager.CanFormat('ext4') if avail: self.assertEqual(util, '') else: self.assertEqual(util, 'mkfs.ext4') self.assertEqual(avail, find_executable('mkfs.ext4') is not None) for fs in map(str, self.get_property(self.manager_obj, '.Manager', 'SupportedFilesystems').value): avail, util = manager.CanFormat(fs) # currently UDisks relies on executables for filesystem creation if avail: self.assertEqual(util, '') else: self.assertGreater(len(util), 0)
def execute(bench_path): expected_node_ver = Version('5.0.0') node_exec = find_executable('node') or find_executable('nodejs') if node_exec: result = subprocess.check_output([node_exec, '-v']).decode() else: click.echo(''' No node executable was found on your machine. Please install latest node version before running "bench update". For installation instructions please refer "Debian and Ubuntu based Linux distributions" section or "Enterprise Linux and Fedora" section depending upon your OS on the following link, "https://nodejs.org/en/download/package-manager/" ''') sys.exit(1) node_ver = Version(result.rstrip('\n').lstrip('v')) if node_ver < expected_node_ver: click.echo(''' Please update node to latest version before running "bench update". Please install latest node version before running "bench update". For installation instructions please refer "Debian and Ubuntu based Linux distributions" section or "Enterprise Linux and Fedora" section depending upon your OS on the following link, "https://nodejs.org/en/download/package-manager/" ''') sys.exit(1)
def android_update_sdk_path(self): """Checks PATH variable and edits bashrc/profile/path for Android SDK.""" tools_update_path = True platform_update_path = True if find_executable("android"): tools_update_path = False if find_executable("ndk-build"): platform_update_path = False if tools_update_path or platform_update_path: if self.bash: # LINUX or MAC with open(self.bash, "a") as f: if tools_update_path: f.write("export PATH=" + os.path.join(self.sdk_path, "tools") + ":$PATH\n") if platform_update_path: f.write("export PATH=" + os.path.join(self.sdk_path, "platform" "-tools") + ":$PATH\n") else: # WINDOWS if tools_update_path: self.windows_path_update = os.path.join( self.sdk_path, "tools") + os.pathsep + self.windows_path_update if platform_update_path: self.windows_path_update = (os.path.join( self.sdk_path, "platform-tools") + os.pathsep + self.windows_path_update) self.bash_changed = True
def init(self): self.cli = find_executable(self.cli_str) if self.container and not self.cli: host_path = [] for path in os.environ.get("PATH").split(":"): host_path.append("/host%s" % path) self.cli = find_executable(self.cli_str, path=":".join(host_path)) if not self.cli: # if run as non-root we need a symlink in the container os.symlink("/host/usr/bin/openshift", "/usr/bin/oc") self.cli = "/usr/bin/oc" if not self.cli or not os.access(self.cli, os.X_OK): raise ProviderFailedException("Command %s not found" % self.cli) else: logger.debug("Using %s to run OpenShift commands.", self.cli) if "openshiftconfig" in self.config: self.config_file = self.config["openshiftconfig"] if self.container: self.config_file = os.path.join("/host", self.config_file.lstrip("/")) else: logger.warning("Configuration option 'openshiftconfig' not found") if not self.config_file or not os.access(self.config_file, os.R_OK): raise ProviderFailedException( "Cannot access configuration file %s. Try adding " "'openshiftconfig = /path/to/your/.kube/config' in the " "[general] section of the answers.conf file." % self.config_file)
def test_x11_remote(): from nose.plugins.skip import SkipTest from distutils.spawn import find_executable if not find_executable('Xorg') or not find_executable('xterm'): raise SkipTest("Testing X11Remote requires X11 and xterm") with utils.named_temporary_directory() as tmp, \ temporary_x_session() as display: r = uri_to_remote('x11:%s' % display) subprocess.Popen( ['xterm', '-l', '-lf', 'xterm.log'], env={'DISPLAY': display, 'PATH': os.environ['PATH']}, cwd=tmp, stderr=open('/dev/null', 'w')) # Can't be sure how long xterm will take to get ready: for _ in range(0, 20): for keysym in ['t', 'o', 'u', 'c', 'h', 'space', 'g', 'o', 'o', 'd', 'Return']: r.press(keysym) if os.path.exists(tmp + '/good'): break time.sleep(0.5) with open(tmp + '/xterm.log', 'r') as log: for line in log: print "xterm.log: " + line, assert os.path.exists(tmp + '/good')
def _gcc_search(library): """ Search a library path by calling and parsing a C linker call. Run GCC's linker with the -t (aka --trace) option and examine the library name it prints out. The GCC command will fail because we haven't supplied a proper program with main(), but that does not matter. """ linker = find_executable('cc') or find_executable('gcc') if linker is None: return None try: with NamedTemporaryFile() as tmp: command = '{linker} -Wl,-t -o "{program}" -l"{library}"'.format( linker=linker, program=tmp.name, library=library ) trace = _execute(command) except OSError as e: # ENOENT is raised if the file was already removed, which is the normal # behaviour of GCC if linking fails if e.errno != ENOENT: raise if trace is None: return None expr = r'[^\(\)\s]*lib{}\.[^\(\)\s]*'.format(escape(library)) match = search(expr, trace) if not match: return None library_path = normpath(match.group(0)) return _final_path(library_path)
def test_x11_remote(): from unittest import SkipTest from .x11 import x_server if not find_executable("Xorg") or not find_executable("xterm"): raise SkipTest("Testing X11Remote requires X11 and xterm") with utils.named_temporary_directory() as tmp, x_server(320, 240) as display: r = uri_to_remote("x11:%s" % display) subprocess.Popen( ["xterm", "-l", "-lf", "xterm.log"], env={"DISPLAY": display, "PATH": os.environ["PATH"]}, cwd=tmp, stderr=open("/dev/null", "w"), ) # Can't be sure how long xterm will take to get ready: for _ in range(0, 20): for keysym in ["KEY_T", "KEY_O", "KEY_U", "KEY_C", "KEY_H", "KEY_SPACE", "g", "o", "o", "d", "KEY_OK"]: r.press(keysym) if os.path.exists(tmp + "/good"): break time.sleep(0.5) with open(tmp + "/xterm.log", "r") as log: for line in log: print "xterm.log: " + line, assert os.path.exists(tmp + "/good")
def check_for_blast(): makeblastdb_path = spawn.find_executable('makeblastdb') blastn_path = spawn.find_executable('blastn') blast_installed = (makeblastdb_path != None and blastn_path != None) if not blast_installed: print('Error: could not find BLAST program', file=sys.stderr) quit()
def to_text(path): """Wraps Tesseract OCR. Parameters ---------- path : str path of electronic invoice in JPG or PNG format Returns ------- extracted_str : str returns extracted text from image in JPG or PNG format """ import subprocess from distutils import spawn # Check for dependencies. Needs Tesseract and Imagemagick installed. if not spawn.find_executable('tesseract'): raise EnvironmentError('tesseract not installed.') if not spawn.find_executable('convert'): raise EnvironmentError('imagemagick not installed.') # convert = "convert -density 350 %s -depth 8 tiff:-" % (path) convert = ['convert', '-density', '350', path, '-depth', '8', 'png:-'] p1 = subprocess.Popen(convert, stdout=subprocess.PIPE, shell=True) tess = ['tesseract', 'stdin', 'stdout'] p2 = subprocess.Popen(tess, stdin=p1.stdout, stdout=subprocess.PIPE) out, err = p2.communicate() extracted_str = out return extracted_str
def deployDataprocessor(keyfile, hostname, dbServer): updateConfigFile(dbServer) if sshFileCheck(keyfile): if find_executable('scp') is not None and os.path.isdir('dataProcessor'): #Copy dataProcessor to home for couchDB setup status_code = subprocess.call(['scp', '-i', keyfile, '-r', './dataProcessor/', 'ubuntu@' + hostname +':~/dataProcessor']) if status_code != 0: sys.exit(1) status_code = subprocess.call(['scp', '-i', keyfile, '-r', './nltk_data', 'ubuntu@' + hostname +':~/']) if status_code != 0: sys.exit(1) else: print('Please install scp or dataProcessor or nltk_data is in place and run the program again') sys.exit(1) else: print('public key is missing') deployView(hostname) if find_executable('ansible-playbook') is not None and os.path.isfile('dataProcessor.yml'): status_code = subprocess.call(['ansible-playbook', 'dataProcessor.yml']) if status_code != 0: sys.exit(1) print('Ansible deploying dataProcessor done') else: print('ansible-playbook is not installed or dataProcessor.yml might be missing')
def setUp(self): super(BrowserJsUnitTestCase, self).setUp() # If java not available, skip the test since it is needed # by jsTestDriver if not find_executable( "java" ): raise SkipTest( "java is needed to do the " "GUI JavaScript unit testing" ) # run headless if xvfb is available if find_executable( "xvfb-run" ): java_cmd = "xvfb-run java" else: java_cmd = "java" # What is the path to the exe of this browser, if any browser_name, browser_exe_filepath = self.get_browser_info() if not browser_exe_filepath: raise SkipTest("The browser '%s' is not available for use in " "GUI JavaScript unit testing" % browser_name) # Set some env vars used by jsTestDriver os.environ[ 'JSTESTDRIVER' ] = """%(java_cmd)s -jar %(jstd_path)s --port %(port_num)d --captureConsole --browser %(browser)s""" % \ { 'java_cmd': java_cmd, 'jstd_path': jstestdriver_path, 'port_num' : port_number, 'browser': browser_exe_filepath, } os.environ[ 'JSTESTDRIVER_PORT' ] = str( port_number ) os.environ[ 'JSTESTDRIVER_SERVER' ] = \ 'http://localhost:%d' % ( port_number )
def setUp(self): if None in [spawn.find_executable('ocropus-rpred'), spawn.find_executable('ocropus-gpageseg'), spawn.find_executable('ocropus-hocr')]: raise SkipTest self.config_mock = MagicMock() storage_path = unicode(tempfile.mkdtemp()) self.config_mock.nidaba_cfg = { 'storage_path': storage_path, 'lang_dicts': {}, 'ocropus_models': {'ocropus': ('test', 'en-default.pyrnn.gz')}, 'plugins_load': {} } self.patches = { 'nidaba.config': self.config_mock, } self.patcher = patch.dict('sys.modules', self.patches) self.patcher2 = patch('nidaba.storage.nidaba_cfg', self.config_mock.nidaba_cfg) self.addCleanup(self.patcher2.stop) self.addCleanup(self.patcher.stop) self.patcher.start() self.patcher2.start() self.storage_path = storage_path shutil.copytree(resources, self.storage_path + '/test') from nidaba.plugins import ocropus ocropus.setup() self.ocropus = ocropus
def build_man_page(self): """Build man pages for all installable programs.""" self.announce("building man pages") descriptions = {} descriptions['synopsis'] = """simple frontend to the Synopsis framework, a multi-language source code introspection tool that provides a variety of representations for the parsed code, to enable further processing such as documentation extraction, reverse engineering, and source-to-source translation.""" descriptions['sxr-server'] = """the Synopsis Cross-Reference http server. Allows users to query and browse cross-referenced source code.""" help2man = find_executable('help2man') if not help2man: self.warn("cannot build man pages") return gzip = find_executable('gzip') section = 1 man_dir = 'share/man/man%d'%section mkpath(man_dir, 0777, self.verbose, self.dry_run) for s in ['synopsis', 'sxr-server']: command = [help2man, '-N', '-n', descriptions[s]] executable = os.path.join('scripts', s) output = '%s/%s.%d'%(man_dir, s, section) command += ['-o', output, executable] spawn(command) if gzip: spawn(['gzip', '-f', output])
def run(self): final_build_dir = os.path.join(self.build_base, self.cmake_dir) self.cmake_exe = find_executable('cmake', os.path.join(final_build_dir, 'bin')) if self.cmake_exe is None: self.__build_cmake(self.cmake_version, final_build_dir, self.build_base) self.cmake_exe = find_executable('cmake', os.path.join(final_build_dir, 'bin')) assert self.cmake_exe
def get_ifconfig(): # This is only useful for Arch Linux which does not contain ifconfig by default if not find_executable('ifconfig'): install = raw_input( ('[' + T + '*' + W + '] ifconfig not found. ' + 'install now? [y/n] ') ) if install == 'y': if os.path.isfile('/usr/bin/pacman'): os.system('pacman -S net-tools') else: sys.exit(( '\n[' + R + '-' + W + '] Don\'t know how to install ifconfig for your distribution.\n' + '[' + G + '+' + W + '] Rerun the script after installing it manually.\n' + '[' + R + '!' + W + '] Closing' )) else: sys.exit(('[' + R + '-' + W + '] ifconfig' + ' not found')) if not find_executable('ifconfig'): sys.exit(( '\n[' + R + '-' + W + '] Unable to install the \'net-tools\' package!\n' + '[' + T + '*' + W + '] This process requires a persistent internet connection!\n' + '[' + G + '+' + W + '] Run pacman -Syu to make sure you are up to date first.\n' + '[' + G + '+' + W + '] Rerun the script to install net-tools.\n' + '[' + R + '!' + W + '] Closing' ))
def process(self, progress): from distutils.spawn import find_executable res = find_executable("cygpath", os.path.join(config['paths']['build'], "cygwin", "bin")) if res is not None: current_dir_cygwin = subprocess.check_output("{0} {1}" .format(res, os.path.join(config["paths"]["build"], "icu", "dist"))) soutpath = os.path.join(self._context["build_path"], "stdout.log") serrpath = os.path.join(self._context["build_path"], "stderr.log") with open(soutpath, "w") as sout: with open(serrpath, "w") as serr: res = find_executable("bash", os.path.join(config['paths']['build'], "cygwin", "bin")) proc = subprocess.Popen([res, "runConfigureICU", "Cygwin/MSVC", "--prefix" , "{}".format(current_dir_cygwin)], env=icu_environment(), cwd=os.path.join(self._context["build_path"], "source"), shell=True, stdout=sout, stderr=serr) proc.communicate() if proc.returncode != 0: logging.error("failed to run icu runConfigureICU (returncode %s), see %s and %s", proc.returncode, soutpath, serrpath) return False return True
def check_requirements(): for cmd in (('convert','imagemagick'),('identify','imagemagick'),('mogrify','imagemagick')): if not spawn.find_executable(cmd[0]): print("Missing required program '%s'." %cmd[1]) if opsys == "Linux": print("Please install from the package manager and try again") else: print("Please install the imagemagick suite from http://imagemagick.org/script/binary-releases.php#windows and try again") sys.exit(1) for cmd in (('exiftool','exiftool'),): if not spawn.find_executable(cmd[0]): print("Missing required program '%s'." %cmd[1]) if opsys == "Linux": print("Please install from the package manager and try again") else: print("Please install exiftool from http://www.sno.phy.queensu.ca/~phil/exiftool/ and try again") sys.exit(1) for cmd in (('awk','awk'),): if not spawn.find_executable(cmd[0]): print("Missing required program '%s'." %cmd[1]) if opsys == "Linux": print("Please install from the package manager and try again") else: print("Please install awk from http://gnuwin32.sourceforge.net/packages/gawk.htm and try again") sys.exit(1)
def setup_sudoers(user): if not os.path.exists("/etc/sudoers.d"): os.makedirs("/etc/sudoers.d") set_permissions = False if not os.path.exists("/etc/sudoers"): set_permissions = True with open("/etc/sudoers", "a") as f: f.write("\n#includedir /etc/sudoers.d\n") if set_permissions: os.chmod("/etc/sudoers", 0440) sudoers_file = "/etc/sudoers.d/frappe" template = env.get_template("frappe_sudoers") frappe_sudoers = template.render( **{ "user": user, "service": find_executable("service"), "systemctl": find_executable("systemctl"), "supervisorctl": find_executable("supervisorctl"), "nginx": find_executable("nginx"), "bench": find_executable("bench"), } ) with open(sudoers_file, "w") as f: f.write(frappe_sudoers.encode("utf-8")) os.chmod(sudoers_file, 0440)
def mysql_system_database(run_services, mysql_data_dir, mysql_defaults_file, memory_temp_dir, lock_dir, services_log): """Install database to given path.""" if run_services: mysql_install_db = find_executable('mysql_install_db') assert mysql_install_db, 'You have to install mysql_install_db script.' my_print_defaults = find_executable('my_print_defaults') assert my_print_defaults, 'You have to install my_print_defaults script.' mysql_basedir = os.path.dirname(os.path.dirname(my_print_defaults)) try: services_log.debug('Starting mysql_install_db.') check_output([ mysql_install_db, '--defaults-file={0}'.format(mysql_defaults_file), '--datadir={0}'.format(mysql_data_dir), '--basedir={0}'.format(mysql_basedir), '--user={0}'.format(os.environ['USER']) ]) except CalledProcessWithOutputError as e: services_log.error( '{e.cmd} failed with output:\n{e.output}\nand erorr:\n{e.err}. ' 'Please ensure you disabled apparmor for /run/shm/** or for whole mysql'.format(e=e)) raise finally: services_log.debug('mysql_install_db was executed.')
def run(self): if True or not self.dry_run: antlr3 = None # First, try locally-hosted antlr3 if os.path.exists('antlr-3.1.3.jar'): antlr3 = find_executable('java') if antlr3 is not None: antlr3 = [antlr3, '-cp', 'antlr-3.1.3.jar', 'org.antlr.Tool'] # Then, try to find system-provided one if antlr3 is None: antlr3 = find_executable('antlr3') if antlr3 is None: raise RuntimeError("antlr3 (>= 3.1 but < 3.2) is required") antlr3 = [antlr3] # TODO: antlr3 jar and python runtime version check? source_file = os.path.join('orderlyjson', 'OrderlyJSON.g') if self.inplace: target_dir = 'orderlyjson' else: target_dir = os.path.join(self.build_lib, 'orderlyjson') tokens_file = os.path.join(target_dir, 'OrderlyJSON.tokens') self.mkpath(target_dir) self.spawn(antlr3 + ['-fo', target_dir, source_file]) os.unlink(tokens_file) build_ext.run(self)
def run(self): if self.compiler is None: self.compiler = distutils.ccompiler.get_default_compiler() bootstrap_args = [] win64 = sys.maxsize > 2**32 if self.compiler == 'msvc': bootstrap_args = ['--platform=msvc'] distutils.msvc9compiler.VERSION = 11.0 compiler = distutils.ccompiler.new_compiler(compiler='msvc') compiler.initialize('win-amd64' if win64 else 'win32') elif self.compiler == 'mingw32': bootstrap_args = ['--platform=mingw'] ninja_info = dict(user=self.ninja_github_user, repo=self.ninja_github_repo, branch=self.ninja_github_branch) self.ninja_build_dir += '_' + self.compiler final_dir = os.path.join(self.build_base, self.ninja_build_dir) if os.path.isdir(final_dir): print("Found '{}', assuming it contains {}:{}".format( final_dir, ninja_info['repo'], ninja_info['branch'])) else: get_from_github(ninja_info, final_dir, self.build_base) self.ninja_exe = find_executable('ninja', final_dir) if self.ninja_exe is None: subprocess.check_call([sys.executable, 'bootstrap.py'] + bootstrap_args, cwd=final_dir) self.ninja_exe = find_executable('ninja', final_dir) assert self.ninja_exe
def get_solver_home(solvername): from distutils.spawn import find_executable if solvername == CPLEX: # Try for environmental variable first env_path = os.getenv('CPLEXDIR') if env_path and len(env_path.strip()) > 0: return env_path # Try to find the cplex binary in the PATH ex_path = find_executable('cplex') if ex_path: ex_path = os.path.realpath(ex_path) # Expand symbolic links if any ex_dir = os.path.dirname(ex_path) # Path to the bin directory return os.path.abspath(os.path.join(ex_dir, os.pardir, os.pardir)) elif solvername == GUROBI: # Try for environmental variable first env_path = os.getenv('GUROBI_HOME') if env_path and len(env_path.strip()) > 0: return env_path # Try to find the gurobi_cl binary in the PATH ex_path = find_executable('gurobi_cl') if ex_path: ex_path = os.path.realpath(ex_path) # Expand symbolic links if any ex_dir = os.path.dirname(ex_path) # Path to the bin directory return os.path.abspath(os.path.join(ex_dir, os.pardir)) else: print >> sys.stderr, "Error unknown solver name", solvername return ""
def handle(self, *args, **options): if not options["interactive"]: options["hostname"] = options["hostname"] or get_host_name() # blank allows ansible scripts to dump errors cleanly. print(" ") print(" _ __ ___ _ _ _ ") print(" | | / / / _ \ | | (_) | ") print(" | |/ / / /_\ \ | | _| |_ ___ ") print(" | \ | _ | | | | | __/ _ \ ") print(" | |\ \| | | | | |___| | || __/ ") print(" \_| \_/\_| |_/ \_____/_|\__\___| ") print(" ") print("https://learningequality.org/ka-lite/") print(" ") print(" version %s" % VERSION) print(" ") if sys.version_info < (2, 7): raise CommandError("Support for Python version 2.6 and below had been discontinued, please upgrade.") elif sys.version_info >= (2, 8): raise CommandError( "Your Python version is: %d.%d.%d -- which is not supported. Please use the Python 2.7 series or wait for Learning Equality to release Kolibri.\n" % sys.version_info[:3]) elif sys.version_info < (2, 7, 6): logging.warning( "It's recommended that you install Python version 2.7.6. Your version is: %d.%d.%d\n" % sys.version_info[:3]) if options["interactive"]: print( "--------------------------------------------------------------------------------") print( "This script will configure the database and prepare it for use.") print( "--------------------------------------------------------------------------------") raw_input("Press [enter] to continue...") # Assuming uid '0' is always root if not is_windows() and hasattr(os, "getuid") and os.getuid() == 0: print( "-------------------------------------------------------------------") print("WARNING: You are installing KA-Lite as root user!") print( "\tInstalling as root may cause some permission problems while running") print("\tas a normal user in the future.") print( "-------------------------------------------------------------------") if options["interactive"]: if not raw_input_yn("Do you wish to continue and install it as root?"): raise CommandError("Aborting script.\n") git_migrate_path = options["git_migrate_path"] if git_migrate_path: call_command("gitmigrate", path=git_migrate_path, interactive=options["interactive"]) database_kind = settings.DATABASES["default"]["ENGINE"] if "sqlite" in database_kind: database_file = settings.DATABASES["default"]["NAME"] else: database_file = None database_exists = database_file and os.path.isfile(database_file) # An empty file is created automatically even when the database dosn't # exist. But if it's empty, it's safe to overwrite. database_exists = database_exists and os.path.getsize(database_file) > 0 install_clean = not database_exists if database_file: if not database_exists: install_clean = True else: # We found an existing database file. By default, # we will upgrade it; users really need to work hard # to delete the file (but it's possible, which is nice). print( "-------------------------------------------------------------------") print("WARNING: Database file already exists!") print( "-------------------------------------------------------------------") if not options["interactive"] \ or raw_input_yn("Keep database file and upgrade to KA Lite version %s? " % VERSION) \ or not raw_input_yn("Remove database file '%s' now? " % database_file) \ or not raw_input_yn("WARNING: all data will be lost! Are you sure? "): install_clean = False print("Upgrading database to KA Lite version %s" % VERSION) else: install_clean = True print("OK. We will run a clean install; ") # After all, don't delete--just move. print( "the database file will be moved to a deletable location.") if not install_clean and not database_file: # Make sure that, for non-sqlite installs, the database exists. raise Exception( "For databases not using SQLite, you must set up your database before running setup.") # Do all input at once, at the beginning if install_clean and options["interactive"]: if not options["username"] or not options["password"]: print( "Please choose a username and password for the admin account on this device.") print( "\tYou must remember this login information, as you will need") print( "\tto enter it to administer this installation of KA Lite.") (username, password) = get_username_password( options["username"], options["password"]) email = options["email"] (hostname, description) = get_hostname_and_description( options["hostname"], options["description"]) else: username = options["username"] = ( options["username"] or getattr(settings, "INSTALL_ADMIN_USERNAME", None) or get_clean_default_username() ) password = options["password"] or getattr(settings, "INSTALL_ADMIN_PASSWORD", None) email = options["email"] # default is non-empty hostname = options["hostname"] description = options["description"] if username and not validate_username(username): raise CommandError( "Username must contain only letters, digits, and underscores, and start with a letter.\n") ######################## # Now do stuff ######################## # Clean *pyc files if we are in a git repo if settings.IS_SOURCE: clean_pyc(settings.SOURCE_DIR) else: # Because we install dependencies as data_files, we run into problems, # namely that the pyc files are left dangling. distributed_packages = [ os.path.join(kalite.ROOT_DATA_PATH, 'dist-packages'), os.path.join(kalite.ROOT_DATA_PATH, 'python-packages'), ] # Try locating django for dir_to_clean in distributed_packages: clean_pyc(dir_to_clean) # Move database file (if exists) if install_clean and database_file and os.path.exists(database_file): if not settings.DB_TEMPLATE_DEFAULT or database_file != settings.DB_TEMPLATE_DEFAULT: # This is an overwrite install; destroy the old db dest_file = tempfile.mkstemp()[1] print( "(Re)moving database file to temp location, starting clean install. Recovery location: %s" % dest_file) shutil.move(database_file, dest_file) if settings.DB_TEMPLATE_DEFAULT and not database_exists and os.path.exists(settings.DB_TEMPLATE_DEFAULT): print("Copying database file from {0} to {1}".format(settings.DB_TEMPLATE_DEFAULT, settings.DEFAULT_DATABASE_PATH)) shutil.copy(settings.DB_TEMPLATE_DEFAULT, settings.DEFAULT_DATABASE_PATH) else: print("Baking a fresh database from scratch or upgrading existing database.") call_command("syncdb", interactive=False, verbosity=options.get("verbosity")) call_command("migrate", merge=True, verbosity=options.get("verbosity")) Settings.set("database_version", VERSION) # Copy all content item db templates reset_content_db(force=install_clean) # download the english content pack # This can take a long time and lead to Travis stalling. None of this # is required for tests, and does not apply to the central server. if options.get("no-assessment-items", False): logging.warning("Skipping content pack downloading and configuration.") else: # Outdated location of assessment items - move assessment items from their # old location (CONTENT_ROOT/khan where they were mixed with other content # items) # TODO(benjaoming) for 0.15, remove the "move assessment items" # mechanism writable_assessment_items = os.access(KHAN_ASSESSMENT_ITEM_ROOT, os.W_OK) # Remove old assessment items if os.path.exists(OLD_ASSESSMENT_ITEMS_LOCATION) and os.access(OLD_ASSESSMENT_ITEMS_LOCATION, os.W_OK): logging.info("Deleting old assessment items") shutil.rmtree(OLD_ASSESSMENT_ITEMS_LOCATION) if options['force-assessment-item-dl']: # user wants to force a new download; do it if we can, else error if writable_assessment_items: call_command("retrievecontentpack", "download", "en") else: raise RuntimeError("Got force-assessment-item-dl but directory not writable") elif english_content_pack_and_assessment_resources_are_current(): logging.warning("English content pack is already up-to-date; skipping download and configuration.") elif not writable_assessment_items: # skip if we're not going to be able to unpack it anyway logging.warning("Assessment item directory not writable; skipping content pack download.") elif settings.RUNNING_IN_TRAVIS: # skip if we're running on Travis logging.warning("Running in Travis; skipping content pack download.") elif not options['interactive']: # skip if we're not running in interactive mode (and it wasn't forced) logging.warning("Not running in interactive mode; skipping content pack download.") else: # if we get this far, then we need to ask the user whether/how they want to get the content pack print( "\nIn order to access many of the available exercises, you need to load a content pack for the latest version.") print( "If you have an Internet connection, you can download the needed package. Warning: this may take a long time!") print( "If you have already downloaded the content pack, you can specify the location of the file in the next step.") print("Otherwise, we will download it from {url}.".format(url=CONTENTPACK_URL)) if raw_input_yn("Do you wish to download and install the content pack now?"): ass_item_filename = CONTENTPACK_URL retrieval_method = "download" elif raw_input_yn("Do you have a local copy of the content pack already downloaded that you want to install?"): ass_item_filename = get_assessment_items_filename() retrieval_method = "local" else: ass_item_filename = None retrieval_method = "local" if not ass_item_filename: logging.warning( "No content pack given. You will need to download and install it later.") else: call_command("retrievecontentpack", retrieval_method, "en", ass_item_filename, foreground=True) # Individually generate any prerequisite models/state that is missing if not Settings.get("private_key"): call_command("generatekeys", verbosity=options.get("verbosity")) if not Device.objects.count(): call_command( "initdevice", hostname, description, verbosity=options.get("verbosity")) if not Facility.objects.count(): Facility.initialize_default_facility() # Create the admin user # blank password (non-interactive) means don't create a superuser if password: admin = get_object_or_None(User, username=username) if not admin: call_command("createsuperuser", username=username, email=email, interactive=False, verbosity=options.get("verbosity")) admin = User.objects.get(username=username) admin.set_password(password) admin.save() # Now deploy the static files logging.info("Copying static media...") ensure_dir(settings.STATIC_ROOT) # The following file ignores have to be preserved from a # collectstatic(clear=True), due to being bundled with content packs, # and we thus have now way of getting them back. collectstatic_ignores = [ "*.vtt", "*.srt", # subtitle files come with language packs -- don't delete "*/perseus/ke/exercises/*", # exercises come with language packs, and we have no way to replicate ] call_command("collectstatic", interactive=False, verbosity=0, ignore_patterns=collectstatic_ignores, clear=True) call_command("collectstatic_js_reverse", interactive=False) # This is not possible in a distributed env if not settings.CENTRAL_SERVER: kalite_executable = 'kalite' if not spawn.find_executable('kalite'): if os.name == 'posix': start_script_path = os.path.realpath( os.path.join(settings.PROJECT_PATH, "..", "bin", kalite_executable)) else: start_script_path = os.path.realpath( os.path.join(settings.PROJECT_PATH, "..", "bin", "windows", "kalite.bat")) else: start_script_path = kalite_executable # Run annotate_content_items, on the distributed server. print("Annotating availability of all content, checking for content in this directory: (%s)" % settings.CONTENT_ROOT) try: call_command("annotate_content_items") except OperationalError: pass # done; notify the user. print("\nCONGRATULATIONS! You've finished setting up the KA Lite server software.") print("You can now start KA Lite with the following command:\n\n\t%s start\n\n" % start_script_path) if options['interactive']: if raw_input_yn("Do you wish to start the server now?"): print("Running {0} start".format(start_script_path)) p = subprocess.Popen([start_script_path, "start"], env=os.environ) p.wait()
def _find_adb(self): adb_loc = os.path.join(os.environ.get("ANDROID_SDK_ROOT", ""), "platform-tools", "adb") if os.path.exists(adb_loc) and os.access(adb_loc, os.X_OK): return adb_loc return find_executable("adb")
def to_text(path, language='fra', **kwargs): """Wraps Tesseract 4 OCR with custom language model. Parameters ---------- path : str path of electronic invoice in JPG or PNG format Returns ------- extracted_str : str returns extracted text from image in JPG or PNG format """ import subprocess from distutils import spawn import tempfile import time # Check for dependencies. Needs Tesseract and Imagemagick installed. if not spawn.find_executable('tesseract'): raise EnvironmentError('tesseract not installed.') if not spawn.find_executable('convert'): raise EnvironmentError('imagemagick not installed.') if not spawn.find_executable('gs'): raise EnvironmentError('ghostscript not installed.') with tempfile.NamedTemporaryFile(suffix='.tiff') as tf: # Step 1: Convert to TIFF gs_cmd = [ 'gs', '-q', '-dNOPAUSE', '-r600x600', '-sDEVICE=tiff24nc', '-sOutputFile=' + tf.name, path, '-c', 'quit', ] subprocess.Popen(gs_cmd) time.sleep(3) # Step 2: Enhance TIFF magick_cmd = [ 'convert', tf.name, '-colorspace', 'gray', '-type', 'grayscale', '-contrast-stretch', '0', '-sharpen', '0x1', 'tiff:-', ] p1 = subprocess.Popen(magick_cmd, stdout=subprocess.PIPE) tess_cmd = [ 'tesseract', '-l', language, '--oem', '1', '--psm', '3', 'stdin', 'stdout' ] p2 = subprocess.Popen(tess_cmd, stdin=p1.stdout, stdout=subprocess.PIPE) out, err = p2.communicate() extracted_str = out return extracted_str
./example.py Author: Diego Caviedes Nozal - 23.10.2020 ''' import argparse import acoustic_gps as agp import numpy as np from matplotlib import pyplot as plt from helper import * from matplotlib import rc import os from distutils.spawn import find_executable rc('font', **{'family': 'serif', 'serif': ['Computer Modern Roman']}) if find_executable('latex'): print("latex installed") rc('text', usetex=True) # TODO: Explain how Bivariate to Complex works in the code. # TODO: Comment code properly # TODO: Header and paper citing at the top (any CC license)? # TODO: Reconstruction plot -> colorbars. axis labels # TODO: Create folder if it doesn't exist parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) # parser.add_argument('foo', help="Name of file to process") parser.parse_args()
def compile_cython_modules(profile=False, coverage=False, compile_minimal=False, compile_more=False, cython_with_refnanny=False): source_root = os.path.abspath(os.path.dirname(__file__)) compiled_modules = [ "Cython.Plex.Actions", "Cython.Plex.Scanners", "Cython.Compiler.FlowControl", "Cython.Compiler.Scanning", "Cython.Compiler.Visitor", "Cython.Runtime.refnanny", ] if not compile_minimal: compiled_modules.extend([ "Cython.Plex.Machines", "Cython.Plex.Transitions", "Cython.Plex.DFA", "Cython.Compiler.FusedNode", "Cython.Tempita._tempita", "Cython.StringIOTree", "Cython.Utils", ]) if compile_more and not compile_minimal: compiled_modules.extend([ "Cython.Compiler.Code", "Cython.Compiler.Lexicon", "Cython.Compiler.Parsing", "Cython.Compiler.Pythran", "Cython.Build.Dependencies", "Cython.Compiler.ParseTreeTransforms", "Cython.Compiler.Nodes", "Cython.Compiler.ExprNodes", "Cython.Compiler.ModuleNode", "Cython.Compiler.Optimize", ]) from distutils.spawn import find_executable from distutils.sysconfig import get_python_inc pgen = find_executable( 'pgen', os.pathsep.join([ os.environ['PATH'], os.path.join(get_python_inc(), '..', 'Parser') ])) if not pgen: sys.stderr.write( "Unable to find pgen, not compiling formal grammar.\n") else: parser_dir = os.path.join(os.path.dirname(__file__), 'Cython', 'Parser') grammar = os.path.join(parser_dir, 'Grammar') subprocess.check_call([ pgen, os.path.join(grammar), os.path.join(parser_dir, 'graminit.h'), os.path.join(parser_dir, 'graminit.c'), ]) cst_pyx = os.path.join(parser_dir, 'ConcreteSyntaxTree.pyx') if os.stat(grammar)[stat.ST_MTIME] > os.stat(cst_pyx)[stat.ST_MTIME]: mtime = os.stat(grammar)[stat.ST_MTIME] os.utime(cst_pyx, (mtime, mtime)) compiled_modules.extend([ "Cython.Parser.ConcreteSyntaxTree", ]) defines = [] if cython_with_refnanny: defines.append(('CYTHON_REFNANNY', '1')) if coverage: defines.append(('CYTHON_TRACE', '1')) extensions = [] for module in compiled_modules: source_file = os.path.join(source_root, *module.split('.')) pyx_source_file = source_file + ".py" if not os.path.exists(pyx_source_file): pyx_source_file += "x" # .py -> .pyx dep_files = [] if os.path.exists(source_file + '.pxd'): dep_files.append(source_file + '.pxd') extensions.append( Extension( module, sources=[pyx_source_file], define_macros=defines if '.refnanny' not in module else [], depends=dep_files)) # XXX hack around setuptools quirk for '*.pyx' sources extensions[-1].sources[0] = pyx_source_file # optimise build parallelism by starting with the largest modules extensions.sort(key=lambda ext: os.path.getsize(ext.sources[0]), reverse=True) from Cython.Distutils.build_ext import build_ext from Cython.Compiler.Options import get_directive_defaults get_directive_defaults().update( language_level=2, binding=False, always_allow_keywords=False, autotestdict=False, ) if profile: get_directive_defaults()['profile'] = True sys.stderr.write("Enabled profiling for the Cython binary modules\n") if coverage: get_directive_defaults()['linetrace'] = True sys.stderr.write( "Enabled line tracing and profiling for the Cython binary modules\n" ) # not using cythonize() directly to let distutils decide whether building extensions was requested add_command_class("build_ext", build_ext) setup_args['ext_modules'] = extensions
import subprocess doc = DocStringFormatTester(mol2) def test_load_mol2(): trj = md.load(get_fn('imatinib.mol2')) ref_trj = md.load(get_fn('imatinib.pdb')) eq(trj.xyz, ref_trj.xyz) ref_top, ref_bonds = ref_trj.top.to_dataframe() top, bonds = trj.top.to_dataframe() eq(bonds, ref_bonds) @skipif( find_executable('obabel') is None, 'You need obabel installed to run this test') @skipif( os.environ.get("TRAVIS", None) == 'true', "Skip testing of entire FreeSolv database on Travis.") def test_load_freesolv_gaffmol2_vs_sybylmol2_vs_obabelpdb(): with enter_temp_directory(): tar_filename = "freesolve_v0.3.tar.bz2" tar = tarfile.open(get_fn(tar_filename), mode="r:bz2") tar.extractall() tar.close() database = pickle.load(open("./v0.3/database.pickle")) for key in database:
class TestShutil(unittest.TestCase): def setUp(self) -> None: super().setUp() self.tempdirs = List[str]() def tearDown(self) -> None: super().tearDown() while self.tempdirs: d = self.tempdirs.pop() shutil.rmtree(d, os.name in ('nt', 'cygwin')) @overload def write_file(self, path: str, content: str = 'xxx') -> None: """Writes a file in the given path. path can be a string or a sequence. """ f = open(path, 'w') try: f.write(content) finally: f.close() @overload def write_file(self, path: Sequence[str], content: str = 'xxx') -> None: # JLe: work around mypy issue #238 self.write_file(os.path.join(*list(path)), content) def mkdtemp(self) -> str: """Create a temporary directory that will be cleaned up. Returns the path of the directory. """ d = tempfile.mkdtemp() self.tempdirs.append(d) return d def test_rmtree_errors(self) -> None: # filename is guaranteed not to exist filename = tempfile.mktemp() self.assertRaises(OSError, shutil.rmtree, filename) # See bug #1071513 for why we don't run this on cygwin # and bug #1076467 for why we don't run this as root. if (hasattr(os, 'chmod') and sys.platform[:6] != 'cygwin' and not (hasattr(os, 'geteuid') and os.geteuid() == 0)): def test_on_error(self) -> None: self.errorState = 0 os.mkdir(TESTFN) self.childpath = os.path.join(TESTFN, 'a') f = open(self.childpath, 'w') f.close() old_dir_mode = os.stat(TESTFN).st_mode old_child_mode = os.stat(self.childpath).st_mode # Make unwritable. os.chmod(self.childpath, stat.S_IREAD) os.chmod(TESTFN, stat.S_IREAD) shutil.rmtree(TESTFN, onerror=self.check_args_to_onerror) # Test whether onerror has actually been called. self.assertEqual( self.errorState, 2, "Expected call to onerror function did not happen.") # Make writable again. os.chmod(TESTFN, old_dir_mode) os.chmod(self.childpath, old_child_mode) # Clean up. shutil.rmtree(TESTFN) def check_args_to_onerror( self, func: Function[[str], Any], arg: str, exc: Tuple[type, BaseException, Traceback]) -> None: # test_rmtree_errors deliberately runs rmtree # on a directory that is chmod 400, which will fail. # This function is run when shutil.rmtree fails. # 99.9% of the time it initially fails to remove # a file in the directory, so the first time through # func is os.remove. # However, some Linux machines running ZFS on # FUSE experienced a failure earlier in the process # at os.listdir. The first failure may legally # be either. if self.errorState == 0: if func is os.remove: self.assertEqual(arg, self.childpath) else: self.assertIs(func, os.listdir, "func must be either os.remove or os.listdir") self.assertEqual(arg, TESTFN) self.assertTrue(issubclass(exc[0], OSError)) self.errorState = 1 else: self.assertEqual(func, os.rmdir) self.assertEqual(arg, TESTFN) self.assertTrue(issubclass(exc[0], OSError)) self.errorState = 2 def test_rmtree_dont_delete_file(self) -> None: # When called on a file instead of a directory, don't delete it. handle, path = tempfile.mkstemp() os.fdopen(handle).close() self.assertRaises(OSError, shutil.rmtree, path) os.remove(path) def _write_data(self, path: str, data: str) -> None: f = open(path, "w") f.write(data) f.close() def test_copytree_simple(self) -> None: def read_data(path: str) -> str: f = open(path) data = f.read() f.close() return data src_dir = tempfile.mkdtemp() dst_dir = os.path.join(tempfile.mkdtemp(), 'destination') self._write_data(os.path.join(src_dir, 'test.txt'), '123') os.mkdir(os.path.join(src_dir, 'test_dir')) self._write_data(os.path.join(src_dir, 'test_dir', 'test.txt'), '456') try: shutil.copytree(src_dir, dst_dir) self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test.txt'))) self.assertTrue(os.path.isdir(os.path.join(dst_dir, 'test_dir'))) self.assertTrue( os.path.isfile(os.path.join(dst_dir, 'test_dir', 'test.txt'))) actual = read_data(os.path.join(dst_dir, 'test.txt')) self.assertEqual(actual, '123') actual = read_data(os.path.join(dst_dir, 'test_dir', 'test.txt')) self.assertEqual(actual, '456') finally: for path in ( os.path.join(src_dir, 'test.txt'), os.path.join(dst_dir, 'test.txt'), os.path.join(src_dir, 'test_dir', 'test.txt'), os.path.join(dst_dir, 'test_dir', 'test.txt'), ): if os.path.exists(path): os.remove(path) for path in (src_dir, os.path.dirname(dst_dir)): if os.path.exists(path): shutil.rmtree(path) def test_copytree_with_exclude(self) -> None: def read_data(path: str) -> str: f = open(path) data = f.read() f.close() return data # creating data join = os.path.join exists = os.path.exists src_dir = tempfile.mkdtemp() try: dst_dir = join(tempfile.mkdtemp(), 'destination') self._write_data(join(src_dir, 'test.txt'), '123') self._write_data(join(src_dir, 'test.tmp'), '123') os.mkdir(join(src_dir, 'test_dir')) self._write_data(join(src_dir, 'test_dir', 'test.txt'), '456') os.mkdir(join(src_dir, 'test_dir2')) self._write_data(join(src_dir, 'test_dir2', 'test.txt'), '456') os.mkdir(join(src_dir, 'test_dir2', 'subdir')) os.mkdir(join(src_dir, 'test_dir2', 'subdir2')) self._write_data(join(src_dir, 'test_dir2', 'subdir', 'test.txt'), '456') self._write_data(join(src_dir, 'test_dir2', 'subdir2', 'test.py'), '456') # testing glob-like patterns try: patterns = shutil.ignore_patterns('*.tmp', 'test_dir2') shutil.copytree(src_dir, dst_dir, ignore=patterns) # checking the result: some elements should not be copied self.assertTrue(exists(join(dst_dir, 'test.txt'))) self.assertTrue(not exists(join(dst_dir, 'test.tmp'))) self.assertTrue(not exists(join(dst_dir, 'test_dir2'))) finally: if os.path.exists(dst_dir): shutil.rmtree(dst_dir) try: patterns = shutil.ignore_patterns('*.tmp', 'subdir*') shutil.copytree(src_dir, dst_dir, ignore=patterns) # checking the result: some elements should not be copied self.assertTrue(not exists(join(dst_dir, 'test.tmp'))) self.assertTrue( not exists(join(dst_dir, 'test_dir2', 'subdir2'))) self.assertTrue( not exists(join(dst_dir, 'test_dir2', 'subdir'))) finally: if os.path.exists(dst_dir): shutil.rmtree(dst_dir) # testing callable-style try: def _filter(src: str, names: Sequence[str]) -> List[str]: res = List[str]() for name in names: path = os.path.join(src, name) if (os.path.isdir(path) and path.split()[-1] == 'subdir'): res.append(name) elif os.path.splitext(path)[-1] in ('.py'): res.append(name) return res shutil.copytree(src_dir, dst_dir, ignore=_filter) # checking the result: some elements should not be copied self.assertTrue(not exists( join(dst_dir, 'test_dir2', 'subdir2', 'test.py'))) self.assertTrue( not exists(join(dst_dir, 'test_dir2', 'subdir'))) finally: if os.path.exists(dst_dir): shutil.rmtree(dst_dir) finally: shutil.rmtree(src_dir) shutil.rmtree(os.path.dirname(dst_dir)) @unittest.skipUnless(hasattr(os, 'link'), 'requires os.link') def test_dont_copy_file_onto_link_to_itself(self) -> None: # Temporarily disable test on Windows. if os.name == 'nt': return # bug 851123. os.mkdir(TESTFN) src = os.path.join(TESTFN, 'cheese') dst = os.path.join(TESTFN, 'shop') try: with open(src, 'w') as f: f.write('cheddar') os.link(src, dst) self.assertRaises(shutil.Error, shutil.copyfile, src, dst) with open(src, 'r') as f: self.assertEqual(f.read(), 'cheddar') os.remove(dst) finally: shutil.rmtree(TESTFN, ignore_errors=True) @support.skip_unless_symlink def test_dont_copy_file_onto_symlink_to_itself(self) -> None: # bug 851123. os.mkdir(TESTFN) src = os.path.join(TESTFN, 'cheese') dst = os.path.join(TESTFN, 'shop') try: with open(src, 'w') as f: f.write('cheddar') # Using `src` here would mean we end up with a symlink pointing # to TESTFN/TESTFN/cheese, while it should point at # TESTFN/cheese. os.symlink('cheese', dst) self.assertRaises(shutil.Error, shutil.copyfile, src, dst) with open(src, 'r') as f: self.assertEqual(f.read(), 'cheddar') os.remove(dst) finally: shutil.rmtree(TESTFN, ignore_errors=True) @support.skip_unless_symlink def test_rmtree_on_symlink(self) -> None: # bug 1669. os.mkdir(TESTFN) try: src = os.path.join(TESTFN, 'cheese') dst = os.path.join(TESTFN, 'shop') os.mkdir(src) os.symlink(src, dst) self.assertRaises(OSError, shutil.rmtree, dst) finally: shutil.rmtree(TESTFN, ignore_errors=True) if hasattr(os, "mkfifo"): # Issue #3002: copyfile and copytree block indefinitely on named pipes def test_copyfile_named_pipe(self) -> None: os.mkfifo(TESTFN) try: self.assertRaises(shutil.SpecialFileError, shutil.copyfile, TESTFN, TESTFN2) self.assertRaises(shutil.SpecialFileError, shutil.copyfile, __file__, TESTFN) finally: os.remove(TESTFN) @support.skip_unless_symlink def test_copytree_named_pipe(self) -> None: os.mkdir(TESTFN) try: subdir = os.path.join(TESTFN, "subdir") os.mkdir(subdir) pipe = os.path.join(subdir, "mypipe") os.mkfifo(pipe) try: shutil.copytree(TESTFN, TESTFN2) except shutil.Error as e: errors = e.args[0] self.assertEqual(len(errors), 1) src, dst, error_msg = errors[0] self.assertEqual("`%s` is a named pipe" % pipe, error_msg) else: self.fail("shutil.Error should have been raised") finally: shutil.rmtree(TESTFN, ignore_errors=True) shutil.rmtree(TESTFN2, ignore_errors=True) def test_copytree_special_func(self) -> None: src_dir = self.mkdtemp() dst_dir = os.path.join(self.mkdtemp(), 'destination') self._write_data(os.path.join(src_dir, 'test.txt'), '123') os.mkdir(os.path.join(src_dir, 'test_dir')) self._write_data(os.path.join(src_dir, 'test_dir', 'test.txt'), '456') copied = List[Tuple[str, str]]() def _copy(src: str, dst: str) -> None: copied.append((src, dst)) shutil.copytree(src_dir, dst_dir, copy_function=_copy) self.assertEqual(len(copied), 2) @support.skip_unless_symlink def test_copytree_dangling_symlinks(self) -> None: # a dangling symlink raises an error at the end src_dir = self.mkdtemp() dst_dir = os.path.join(self.mkdtemp(), 'destination') os.symlink('IDONTEXIST', os.path.join(src_dir, 'test.txt')) os.mkdir(os.path.join(src_dir, 'test_dir')) self._write_data(os.path.join(src_dir, 'test_dir', 'test.txt'), '456') self.assertRaises(Error, shutil.copytree, src_dir, dst_dir) # a dangling symlink is ignored with the proper flag dst_dir = os.path.join(self.mkdtemp(), 'destination2') shutil.copytree(src_dir, dst_dir, ignore_dangling_symlinks=True) self.assertNotIn('test.txt', os.listdir(dst_dir)) # a dangling symlink is copied if symlinks=True dst_dir = os.path.join(self.mkdtemp(), 'destination3') shutil.copytree(src_dir, dst_dir, symlinks=True) self.assertIn('test.txt', os.listdir(dst_dir)) def _copy_file(self, method: Function[[str, str], None]) -> Tuple[str, str]: fname = 'test.txt' tmpdir = self.mkdtemp() self.write_file([tmpdir, fname]) file1 = os.path.join(tmpdir, fname) tmpdir2 = self.mkdtemp() method(file1, tmpdir2) file2 = os.path.join(tmpdir2, fname) return (file1, file2) @unittest.skipUnless(hasattr(os, 'chmod'), 'requires os.chmod') def test_copy(self) -> None: # Ensure that the copied file exists and has the same mode bits. file1, file2 = self._copy_file(shutil.copy) self.assertTrue(os.path.exists(file2)) self.assertEqual(os.stat(file1).st_mode, os.stat(file2).st_mode) @unittest.skipUnless(hasattr(os, 'chmod'), 'requires os.chmod') @unittest.skipUnless(hasattr(os, 'utime'), 'requires os.utime') def test_copy2(self) -> None: # Ensure that the copied file exists and has the same mode and # modification time bits. file1, file2 = self._copy_file(shutil.copy2) self.assertTrue(os.path.exists(file2)) file1_stat = os.stat(file1) file2_stat = os.stat(file2) self.assertEqual(file1_stat.st_mode, file2_stat.st_mode) for attr in 'st_atime', 'st_mtime': # The modification times may be truncated in the new file. self.assertLessEqual(getattr(file1_stat, attr), getattr(file2_stat, attr) + 1) if hasattr(os, 'chflags') and hasattr(file1_stat, 'st_flags'): self.assertEqual(getattr(file1_stat, 'st_flags'), getattr(file2_stat, 'st_flags')) @unittest.skipUnless(zlib, "requires zlib") def test_make_tarball(self) -> None: # creating something to tar tmpdir = self.mkdtemp() self.write_file([tmpdir, 'file1'], 'xxx') self.write_file([tmpdir, 'file2'], 'xxx') os.mkdir(os.path.join(tmpdir, 'sub')) self.write_file([tmpdir, 'sub', 'file3'], 'xxx') tmpdir2 = self.mkdtemp() # force shutil to create the directory os.rmdir(tmpdir2) unittest.skipUnless( splitdrive(tmpdir)[0] == splitdrive(tmpdir2)[0], "source and target should be on same drive") base_name = os.path.join(tmpdir2, 'archive') # working with relative paths to avoid tar warnings old_dir = os.getcwd() os.chdir(tmpdir) try: _make_tarball(splitdrive(base_name)[1], '.') finally: os.chdir(old_dir) # check if the compressed tarball was created tarball = base_name + '.tar.gz' self.assertTrue(os.path.exists(tarball)) # trying an uncompressed one base_name = os.path.join(tmpdir2, 'archive') old_dir = os.getcwd() os.chdir(tmpdir) try: _make_tarball(splitdrive(base_name)[1], '.', compress=None) finally: os.chdir(old_dir) tarball = base_name + '.tar' self.assertTrue(os.path.exists(tarball)) def _tarinfo(self, path: str) -> tuple: tar = tarfile.open(path) try: names = tar.getnames() names.sort() return tuple(names) finally: tar.close() def _create_files(self) -> Tuple[str, str, str]: # creating something to tar tmpdir = self.mkdtemp() dist = os.path.join(tmpdir, 'dist') os.mkdir(dist) self.write_file([dist, 'file1'], 'xxx') self.write_file([dist, 'file2'], 'xxx') os.mkdir(os.path.join(dist, 'sub')) self.write_file([dist, 'sub', 'file3'], 'xxx') os.mkdir(os.path.join(dist, 'sub2')) tmpdir2 = self.mkdtemp() base_name = os.path.join(tmpdir2, 'archive') return tmpdir, tmpdir2, base_name @unittest.skipUnless(zlib, "Requires zlib") @unittest.skipUnless( find_executable('tar') and find_executable('gzip'), 'Need the tar command to run') def test_tarfile_vs_tar(self) -> None: tmpdir, tmpdir2, base_name = self._create_files() old_dir = os.getcwd() os.chdir(tmpdir) try: _make_tarball(base_name, 'dist') finally: os.chdir(old_dir) # check if the compressed tarball was created tarball = base_name + '.tar.gz' self.assertTrue(os.path.exists(tarball)) # now create another tarball using `tar` tarball2 = os.path.join(tmpdir, 'archive2.tar.gz') tar_cmd = ['tar', '-cf', 'archive2.tar', 'dist'] gzip_cmd = ['gzip', '-f9', 'archive2.tar'] old_dir = os.getcwd() os.chdir(tmpdir) try: with captured_stdout() as s: spawn(tar_cmd) spawn(gzip_cmd) finally: os.chdir(old_dir) self.assertTrue(os.path.exists(tarball2)) # let's compare both tarballs self.assertEqual(self._tarinfo(tarball), self._tarinfo(tarball2)) # trying an uncompressed one base_name = os.path.join(tmpdir2, 'archive') old_dir = os.getcwd() os.chdir(tmpdir) try: _make_tarball(base_name, 'dist', compress=None) finally: os.chdir(old_dir) tarball = base_name + '.tar' self.assertTrue(os.path.exists(tarball)) # now for a dry_run base_name = os.path.join(tmpdir2, 'archive') old_dir = os.getcwd() os.chdir(tmpdir) try: _make_tarball(base_name, 'dist', compress=None, dry_run=True) finally: os.chdir(old_dir) tarball = base_name + '.tar' self.assertTrue(os.path.exists(tarball)) @unittest.skipUnless(zlib, "Requires zlib") @unittest.skipUnless(ZIP_SUPPORT, 'Need zip support to run') def test_make_zipfile(self) -> None: # creating something to tar tmpdir = self.mkdtemp() self.write_file([tmpdir, 'file1'], 'xxx') self.write_file([tmpdir, 'file2'], 'xxx') tmpdir2 = self.mkdtemp() # force shutil to create the directory os.rmdir(tmpdir2) base_name = os.path.join(tmpdir2, 'archive') _make_zipfile(base_name, tmpdir) # check if the compressed tarball was created tarball = base_name + '.zip' self.assertTrue(os.path.exists(tarball)) def test_make_archive(self) -> None: tmpdir = self.mkdtemp() base_name = os.path.join(tmpdir, 'archive') self.assertRaises(ValueError, make_archive, base_name, 'xxx') @unittest.skipUnless(zlib, "Requires zlib") def test_make_archive_owner_group(self) -> None: # testing make_archive with owner and group, with various combinations # this works even if there's not gid/uid support if UID_GID_SUPPORT: group = grp.getgrgid(0).gr_name owner = pwd.getpwuid(0).pw_name else: group = owner = 'root' base_dir, root_dir, base_name = self._create_files() base_name = os.path.join(self.mkdtemp(), 'archive') res = make_archive(base_name, 'zip', root_dir, base_dir, owner=owner, group=group) self.assertTrue(os.path.exists(res)) res = make_archive(base_name, 'zip', root_dir, base_dir) self.assertTrue(os.path.exists(res)) res = make_archive(base_name, 'tar', root_dir, base_dir, owner=owner, group=group) self.assertTrue(os.path.exists(res)) res = make_archive(base_name, 'tar', root_dir, base_dir, owner='kjhkjhkjg', group='oihohoh') self.assertTrue(os.path.exists(res)) @unittest.skipUnless(zlib, "Requires zlib") @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support") def test_tarfile_root_owner(self) -> None: tmpdir, tmpdir2, base_name = self._create_files() old_dir = os.getcwd() os.chdir(tmpdir) group = grp.getgrgid(0).gr_name owner = pwd.getpwuid(0).pw_name try: archive_name = _make_tarball(base_name, 'dist', compress=None, owner=owner, group=group) finally: os.chdir(old_dir) # check if the compressed tarball was created self.assertTrue(os.path.exists(archive_name)) # now checks the rights archive = tarfile.open(archive_name) try: for member in archive.getmembers(): self.assertEqual(member.uid, 0) self.assertEqual(member.gid, 0) finally: archive.close() def test_make_archive_cwd(self) -> None: current_dir = os.getcwd() def _breaks(*args: Any, **kw: Any) -> None: raise RuntimeError() register_archive_format('xxx', _breaks, [], 'xxx file') try: try: make_archive('xxx', 'xxx', root_dir=self.mkdtemp()) except Exception: pass self.assertEqual(os.getcwd(), current_dir) finally: unregister_archive_format('xxx') def test_register_archive_format(self) -> None: self.assertRaises(TypeError, register_archive_format, 'xxx', 1) self.assertRaises(TypeError, register_archive_format, 'xxx', lambda: 1 / 0, 1) self.assertRaises(TypeError, register_archive_format, 'xxx', lambda: 1 / 0, [(1, 2), (1, 2, 3)]) register_archive_format('xxx', lambda: 1 / 0, [('x', 2)], 'xxx file') formats = [name for name, params in get_archive_formats()] self.assertIn('xxx', formats) unregister_archive_format('xxx') formats = [name for name, params in get_archive_formats()] self.assertNotIn('xxx', formats) def _compare_dirs(self, dir1: str, dir2: str) -> List[str]: # check that dir1 and dir2 are equivalent, # return the diff diff = List[str]() for root, dirs, files in os.walk(dir1): for file_ in files: path = os.path.join(root, file_) target_path = os.path.join(dir2, os.path.split(path)[-1]) if not os.path.exists(target_path): diff.append(file_) return diff @unittest.skipUnless(zlib, "Requires zlib") def test_unpack_archive(self) -> None: formats = ['tar', 'gztar', 'zip'] if BZ2_SUPPORTED: formats.append('bztar') for format in formats: tmpdir = self.mkdtemp() base_dir, root_dir, base_name = self._create_files() tmpdir2 = self.mkdtemp() filename = make_archive(base_name, format, root_dir, base_dir) # let's try to unpack it now unpack_archive(filename, tmpdir2) diff = self._compare_dirs(tmpdir, tmpdir2) self.assertEqual(diff, []) # and again, this time with the format specified tmpdir3 = self.mkdtemp() unpack_archive(filename, tmpdir3, format=format) diff = self._compare_dirs(tmpdir, tmpdir3) self.assertEqual(diff, []) self.assertRaises(shutil.ReadError, unpack_archive, TESTFN) self.assertRaises(ValueError, unpack_archive, TESTFN, format='xxx') def test_unpack_registery(self) -> None: formats = get_unpack_formats() def _boo(filename: str, extract_dir: str, extra: int) -> None: self.assertEqual(extra, 1) self.assertEqual(filename, 'stuff.boo') self.assertEqual(extract_dir, 'xx') register_unpack_format('Boo', ['.boo', '.b2'], _boo, [('extra', 1)]) unpack_archive('stuff.boo', 'xx') # trying to register a .boo unpacker again self.assertRaises(RegistryError, register_unpack_format, 'Boo2', ['.boo'], _boo) # should work now unregister_unpack_format('Boo') register_unpack_format('Boo2', ['.boo'], _boo) self.assertIn(('Boo2', ['.boo'], ''), get_unpack_formats()) self.assertNotIn(('Boo', ['.boo'], ''), get_unpack_formats()) # let's leave a clean state unregister_unpack_format('Boo2') self.assertEqual(get_unpack_formats(), formats)
import argparse from PIL import Image import os import sys import math import ast from distutils.spawn import find_executable import subprocess # Allow large images (this could lead to a denial of service attach if you're # running this script on user-submitted images.) Image.MAX_IMAGE_PIXELS = None # Find external programs try: nona = find_executable('nona') except KeyError: # Handle case of PATH not being set nona = None # Subclass parser to add explaination for semi-option nona flag class GenParser(argparse.ArgumentParser): def error(self, message): if '--nona' in message: sys.stderr.write('''IMPORTANT: The location of nona must be specified with -n, since it was not found on the PATH!\n\n''') super(GenParser, self).error(message) # Parse input parser = GenParser(description='Generate a Pannellum multires tile set from a full or partial equirectangular or cylindrical panorama.', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
from distutils.command.build_py import build_py as _build_py from distutils.spawn import find_executable # Find the Protocol Compiler. if 'PROTOC' in os.environ and os.path.exists(os.environ['PROTOC']): protoc = os.environ['PROTOC'] elif os.path.exists("../src/protoc"): protoc = "../src/protoc" elif os.path.exists("../src/protoc.exe"): protoc = "../src/protoc.exe" elif os.path.exists("../vsprojects/Debug/protoc.exe"): protoc = "../vsprojects/Debug/protoc.exe" elif os.path.exists("../vsprojects/Release/protoc.exe"): protoc = "../vsprojects/Release/protoc.exe" else: protoc = find_executable("protoc") def GetVersion(): """Gets the version from google/protobuf/__init__.py Do not import google.protobuf.__init__ directly, because an installed protobuf library may be loaded instead.""" with open(os.path.join('google', 'protobuf', '__init__.py')) as version_file: exec(version_file.read(), globals()) return __version__ def generate_proto(source, require=True):
def create_config_file(watch, start_cmd, pidfile, port=None, env_vars=None, max_memory=None, syslog_server=None, check_port=False): """ Writes a monit configuration file for a service. Args: watch: A string which identifies this process with monit. start_cmd: The start command to start the process. pidfile: The location of the pidfile that the process creates. port: An integer specifying the port for the process. env_vars: A dictionary specifying environment variables. max_memory: An integer that specifies the maximum amount of memory in megabytes that the process should use. syslog_server: The IP address of the remote syslog server to use. check_port: A boolean specifying that monit should check host and port. """ if check_port: assert port is not None, 'When using check_port, port must be defined' process_name = watch if port is not None: process_name += '-{}'.format(port) env_vars_str = '' if env_vars is not None: for key in env_vars: env_vars_str += '{}={} '.format(key, env_vars[key]) bash = find_executable('bash') start_stop_daemon = find_executable('start-stop-daemon') logfile = os.path.join('/', 'var', 'log', 'appscale', '{}.log'.format(process_name)) if syslog_server is None: stop = 'start-stop-daemon --stop --pidfile {0} && rm {0}'.format( pidfile) bash_exec = 'exec env {vars} {start_cmd} >> {log} 2>&1'.format( vars=env_vars_str, start_cmd=start_cmd, log=logfile) else: # Kill entire process group. get_pgid = 'ps -o pgid= $(cat {}) | grep -o "[0-9]*"'.format(pidfile) stop = 'kill -- -$({}) && rm {}'.format(get_pgid, pidfile) bash_exec = 'exec env {vars} {start_cmd} 2>&1 | tee -a {log} | '\ 'logger -t {watch} -u /tmp/ignored -n {syslog_server} -P 514'.\ format(vars=env_vars_str, start_cmd=start_cmd, log=logfile, watch=watch, syslog_server=syslog_server) start_line = ' '.join([ start_stop_daemon, '--start', '--background', '--pidfile', pidfile, '--startas', "{} -- -c '{}'".format(bash, bash_exec) ]) stop_line = "{} -c '{}'".format(bash, stop) with open(TEMPLATE_LOCATION) as template: output = template.read() output = output.format(process_name=process_name, match_clause='PIDFILE "{}"'.format(pidfile), group=watch, start_line=start_line, stop_line=stop_line) if max_memory is not None: output += ' if totalmem > {} MB for 10 cycles then restart\n'.format( max_memory) if check_port: private_ip = appscale_info.get_private_ip() output += ' if failed host {} port {} then restart\n'.format( private_ip, port) config_file = os.path.join(MONIT_CONFIG_DIR, 'appscale-{}.cfg'.format(process_name)) file_io.write(config_file, output) return
class TC_20_DispVMMixin(object): def setUp(self): super(TC_20_DispVMMixin, self).setUp() if 'whonix-gw' in self.template: self.skipTest( 'whonix-gw is not supported as DisposableVM Template') self.init_default_template(self.template) self.disp_base = self.app.add_new_vm( qubes.vm.appvm.AppVM, name=self.make_vm_name('dvm'), label='red', template_for_dispvms=True, ) self.loop.run_until_complete(self.disp_base.create_on_disk()) self.app.default_dispvm = self.disp_base self.app.save() def tearDown(self): self.app.default_dispvm = None super(TC_20_DispVMMixin, self).tearDown() def test_010_simple_dvm_run(self): dispvm = self.loop.run_until_complete( qubes.vm.dispvm.DispVM.from_appvm(self.disp_base)) try: self.loop.run_until_complete(dispvm.start()) (stdout, _) = self.loop.run_until_complete( dispvm.run_service_for_stdio('qubes.VMShell', input=b"echo test")) self.assertEqual(stdout, b"test\n") finally: self.loop.run_until_complete(dispvm.cleanup()) @unittest.skipUnless(spawn.find_executable('xdotool'), "xdotool not installed") def test_020_gui_app(self): dispvm = self.loop.run_until_complete( qubes.vm.dispvm.DispVM.from_appvm(self.disp_base)) try: self.loop.run_until_complete(dispvm.start()) self.loop.run_until_complete(self.wait_for_session(dispvm)) p = self.loop.run_until_complete( dispvm.run_service('qubes.VMShell', stdin=subprocess.PIPE, stdout=subprocess.PIPE)) # wait for DispVM startup: p.stdin.write(b"echo test\n") self.loop.run_until_complete(p.stdin.drain()) l = self.loop.run_until_complete(p.stdout.readline()) self.assertEqual(l, b"test\n") self.assertTrue(dispvm.is_running()) try: window_title = 'user@%s' % (dispvm.name, ) # close xterm on Return, but after short delay, to allow # xdotool to send also keyup event p.stdin.write("xterm -e " "\"sh -c 'echo \\\"\033]0;{}\007\\\";read x;" "sleep 0.1;'\"\n".format(window_title).encode()) self.loop.run_until_complete(p.stdin.drain()) self.wait_for_window(window_title) time.sleep(0.5) self.enter_keys_in_window(window_title, ['Return']) # Wait for window to close self.wait_for_window(window_title, show=False) p.stdin.close() self.loop.run_until_complete(asyncio.wait_for(p.wait(), 30)) except: with suppress(ProcessLookupError): p.terminate() self.loop.run_until_complete(p.wait()) raise finally: del p finally: self.loop.run_until_complete(dispvm.cleanup()) dispvm_name = dispvm.name del dispvm # give it a time for shutdown + cleanup self.loop.run_until_complete(asyncio.sleep(5)) self.assertNotIn(dispvm_name, self.app.domains, "DispVM not removed from qubes.xml") def _handle_editor(self, winid, copy=False): (window_title, _) = subprocess.Popen( ['xdotool', 'getwindowname', winid], stdout=subprocess.PIPE).\ communicate() window_title = window_title.decode().strip().\ replace('(', '\(').replace(')', '\)') time.sleep(1) if "gedit" in window_title or 'KWrite' in window_title or \ 'Mousepad' in window_title or 'Geany' in window_title: subprocess.check_call( ['xdotool', 'windowactivate', '--sync', winid]) if copy: subprocess.check_call([ 'xdotool', 'key', '--window', winid, 'key', 'ctrl+a', 'ctrl+c', 'ctrl+shift+c' ]) else: subprocess.check_call(['xdotool', 'type', 'Test test 2']) subprocess.check_call( ['xdotool', 'key', '--window', winid, 'key', 'Return']) time.sleep(0.5) subprocess.check_call(['xdotool', 'key', 'ctrl+s']) time.sleep(0.5) subprocess.check_call(['xdotool', 'key', 'ctrl+q']) elif "LibreOffice" in window_title: # wait for actual editor (we've got splash screen) search = subprocess.Popen([ 'xdotool', 'search', '--sync', '--onlyvisible', '--all', '--name', '--class', 'disp*|Writer' ], stdout=subprocess.PIPE, stderr=open(os.path.devnull, 'w')) retcode = search.wait() if retcode == 0: winid = search.stdout.read().strip() time.sleep(0.5) subprocess.check_call( ['xdotool', 'windowactivate', '--sync', winid]) if copy: subprocess.check_call([ 'xdotool', 'key', '--window', winid, 'key', 'ctrl+a', 'ctrl+c', 'ctrl+shift+c' ]) else: subprocess.check_call(['xdotool', 'type', 'Test test 2']) subprocess.check_call( ['xdotool', 'key', '--window', winid, 'key', 'Return']) time.sleep(0.5) subprocess.check_call( ['xdotool', 'key', '--delay', '100', 'ctrl+s', 'Return']) time.sleep(0.5) subprocess.check_call(['xdotool', 'key', 'ctrl+q']) elif "emacs" in window_title: subprocess.check_call( ['xdotool', 'windowactivate', '--sync', winid]) if copy: subprocess.check_call( ['xdotool', 'key', 'ctrl+x', 'h', 'alt+w', 'ctrl+shift+c']) else: subprocess.check_call(['xdotool', 'type', 'Test test 2']) subprocess.check_call( ['xdotool', 'key', '--window', winid, 'key', 'Return']) time.sleep(0.5) subprocess.check_call(['xdotool', 'key', 'ctrl+x', 'ctrl+s']) time.sleep(0.5) subprocess.check_call(['xdotool', 'key', 'ctrl+x', 'ctrl+c']) elif "vim" in window_title or "user@" in window_title: subprocess.check_call( ['xdotool', 'windowactivate', '--sync', winid]) if copy: raise NotImplementedError('copy not implemented for vim') else: subprocess.check_call( ['xdotool', 'key', 'i', 'type', 'Test test 2']) subprocess.check_call( ['xdotool', 'key', '--window', winid, 'key', 'Return']) subprocess.check_call( ['xdotool', 'key', 'Escape', 'colon', 'w', 'q', 'Return']) else: raise KeyError(window_title) @unittest.skipUnless(spawn.find_executable('xdotool'), "xdotool not installed") def test_030_edit_file(self): self.testvm1 = self.app.add_new_vm( qubes.vm.appvm.AppVM, name=self.make_vm_name('vm1'), label='red', template=self.app.domains[self.template]) self.loop.run_until_complete(self.testvm1.create_on_disk()) self.app.save() self.loop.run_until_complete(self.testvm1.start()) self.loop.run_until_complete( self.testvm1.run_for_stdio("echo test1 > /home/user/test.txt")) p = self.loop.run_until_complete( self.testvm1.run("qvm-open-in-dvm /home/user/test.txt", stdout=subprocess.PIPE, stderr=subprocess.STDOUT)) # if first 5 windows isn't expected editor, there is no hope winid = None for _ in range(5): try: winid = self.wait_for_window('disp[0-9]*', search_class=True, include_tray=False, timeout=60) except Exception as e: try: self.loop.run_until_complete(asyncio.wait_for(p.wait(), 1)) except asyncio.TimeoutError: raise e else: stdout = self.loop.run_until_complete(p.stdout.read()) self.fail('qvm-open-in-dvm exited prematurely with {}: {}'. format(p.returncode, stdout)) # let the application initialize self.loop.run_until_complete(asyncio.sleep(1)) try: self._handle_editor(winid) break except KeyError: winid = None if winid is None: self.fail('Timeout waiting for editor window') self.loop.run_until_complete(p.communicate()) (test_txt_content, _) = self.loop.run_until_complete( self.testvm1.run_for_stdio("cat /home/user/test.txt")) # Drop BOM if added by editor if test_txt_content.startswith(b'\xef\xbb\xbf'): test_txt_content = test_txt_content[3:] self.assertEqual(test_txt_content, b"Test test 2\ntest1\n") def _get_open_script(self, application): """Generate a script to instruct *application* to open *filename*""" if application == 'org.gnome.Nautilus': return ("#!/usr/bin/python3\n" "import sys, os" "from dogtail import tree, config\n" "config.config.actionDelay = 1.0\n" "config.config.defaultDelay = 1.0\n" "config.config.searchCutoffCount = 10\n" "app = tree.root.application('org.gnome.Nautilus')\n" "app.child(os.path.basename(sys.argv[1])).doubleClick()\n" ).encode() if application in ('mozilla-thunderbird', 'thunderbird'): with open( '/usr/share/qubes/tests-data/' 'dispvm-open-thunderbird-attachment', 'rb') as f: return f.read() assert False @unittest.skipUnless(spawn.find_executable('xdotool'), "xdotool not installed") def test_100_open_in_dispvm(self): self.testvm1 = self.app.add_new_vm( qubes.vm.appvm.AppVM, name=self.make_vm_name('vm1'), label='red', template=self.app.domains[self.template]) self.loop.run_until_complete(self.testvm1.create_on_disk()) self.app.save() app_id = 'mozilla-thunderbird' if 'debian' in self.template or 'whonix' in self.template: app_id = 'thunderbird' self.testvm1.features['service.app-dispvm.' + app_id] = '1' self.loop.run_until_complete(self.testvm1.start()) self.loop.run_until_complete( self.testvm1.run_for_stdio("echo test1 > /home/user/test.txt")) self.loop.run_until_complete( self.testvm1.run_for_stdio("cat > /home/user/open-file", input=self._get_open_script(app_id))) self.loop.run_until_complete( self.testvm1.run_for_stdio("chmod +x /home/user/open-file")) self.loop.run_until_complete( self.testvm1.run_for_stdio( 'gsettings set org.gnome.desktop.interface ' 'toolkit-accessibility true')) app = self.loop.run_until_complete( self.testvm1.run_service("qubes.StartApp+" + app_id)) try: click_to_open = self.loop.run_until_complete( self.testvm1.run_for_stdio('./open-file test.txt', stdout=subprocess.PIPE, stderr=subprocess.STDOUT)) except subprocess.CalledProcessError as err: with contextlib.suppress(asyncio.TimeoutError): self.loop.run_until_complete(asyncio.wait_for(app.wait(), 30)) if app.returncode == 127: self.skipTest('{} not installed'.format(app_id)) self.fail("'./open-file test.txt' failed with {}: {}{}".format( err.cmd, err.returncode, err.stdout, err.stderr)) # if first 5 windows isn't expected editor, there is no hope winid = None for _ in range(5): winid = self.wait_for_window('disp[0-9]*', search_class=True, include_tray=False, timeout=60) # let the application initialize self.loop.run_until_complete(asyncio.sleep(1)) try: # copy, not modify - attachment is set as read-only self._handle_editor(winid, copy=True) break except KeyError: winid = None if winid is None: self.fail('Timeout waiting for editor window') self.loop.run_until_complete( self.wait_for_window_hide_coro("editor", winid)) with open('/var/run/qubes/qubes-clipboard.bin', 'rb') as f: test_txt_content = f.read() self.assertEqual(test_txt_content.strip(), b"test1")
def run(self): if not self.dry_run: #------------------------------------------------------------------- # Check and set the environment up #------------------------------------------------------------------- target_dir = os.path.join(self.build_lib, 'PiPilot', 'ui') if os.path.exists(target_dir): rmtree(target_dir) ui_path = os.path.join(os.getcwd(), 'ui') if not os.path.exists(ui_path): print('[!] The ui directory does not exist') sys.exit(1) npm = find_executable('npm') if npm is None: print('[!] You need to have node installed to build this app') sys.exit(1) #------------------------------------------------------------------- # Build the JavaScript code #------------------------------------------------------------------- ret = run_command([npm, 'install'], ui_path) if ret != 0: print('[!] Installation of JavaScript dependencies failed') sys.exit(1) ret = run_command([npm, 'run-script', 'build'], ui_path) if ret != 0: print('[!] Build of JavaScript artefacts failed') sys.exit(1) #------------------------------------------------------------------- # Create a list of artefacts #------------------------------------------------------------------- artefacts = [ 'asset-manifest.json', 'favicon.png', 'index.html', 'manifest.json', 'service-worker.js' ] build_dir = 'ui/build' asset_manifest = os.path.join(build_dir, artefacts[0]) if not os.path.exists(asset_manifest): print('[!] Asset manifest does not exist.') sys.exit(1) assets = json.loads(open(asset_manifest, 'r').read()) for _, asset in assets.items(): artefacts.append(asset) #------------------------------------------------------------------- # Copy the artefacts to the dist root #------------------------------------------------------------------- print('Copying JavaScript artefacts to', target_dir) for artefact in artefacts: source_file = os.path.join(build_dir, artefact) target_file = os.path.join(target_dir, artefact) target_prefix = os.path.dirname(target_file) if not os.path.exists(target_prefix): os.makedirs(target_prefix) copyfile(source_file, target_file) build_py.run(self)
def check_gulp(): return bool(spawn.find_executable('gulp')), 'gulp', '#gulp'
__author__ = 'laharah' __version__ = '0.3.0' import re import os import tempfile import inspect import sys import warnings from types import MethodType import functools import killableprocess from distutils import spawn FILEBOT_EXE = spawn.find_executable('filebot') if not FILEBOT_EXE: locations = [ r'C:\Program Files\FileBot\filebot.exe', r'/usr/bin/filebot', r'/usr/local/bin/filebot', r'/snap/bin/filebot' ] for loc in locations: if os.path.exists(loc): FILEBOT_EXE = loc break else: warnings.warn("Could not find filebot executable!", stacklevel=2) FILEBOT_EXE = 'filebot' FILEBOT_MODES = [ 'rename', 'move', 'check', 'get-missing-subtitles', 'get-subtitles', 'list',
def check_virtualenv(): return bool( spawn.find_executable('virtualenv')), 'virtualenv', '#virtualenv'
from shutil import which as find_executable NAME = "kfp-pipeline-spec" VERSION = "0.1.4" PROTO_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), os.pardir)) PKG_DIR = os.path.realpath( os.path.join(os.path.dirname(__file__), "kfp", "pipeline_spec")) # Find the Protocol Compiler. (Taken from protobuf/python/setup.py) if "PROTOC" in os.environ and os.path.exists(os.environ["PROTOC"]): PROTOC = os.environ["PROTOC"] else: PROTOC = find_executable("protoc") def GenerateProto(source): """Generate a _pb2.py from a .proto file. Invokes the Protocol Compiler to generate a _pb2.py from the given .proto file. Does nothing if the output already exists and is newer than the input. Args: source: The source proto file that needs to be compiled. """ output = source.replace(".proto", "_pb2.py")
def check_nodejs(): return bool(spawn.find_executable('node')), 'Node.js', '#nodejs'
def check_bower(): return bool(spawn.find_executable('bower')), 'bower', '#bower'
def effect(self): self.zoom = float( self.document.xpath('//sodipodi:namedview/@inkscape:zoom', namespaces=inkex.NSS)[0]) self.width = 1/self.zoom * \ float(self.document.xpath( '//sodipodi:namedview/@inkscape:window-width', namespaces=inkex.NSS)[0]) self.height = 1/self.zoom * \ float(self.document.xpath( '//sodipodi:namedview/@inkscape:window-height', namespaces=inkex.NSS)[0]) self.width = self.unittouu(str(self.width) + 'px') self.height = self.unittouu(str(self.height) + 'px') self.options.scale = float(self.options.scale) action = self.options.action.strip("\"") if action == "viewold": for i in self.options.ids: node = self.selected[i] if node.tag != '{%s}g' % SVG_NS: continue if '{%s}text' % WriteTexNS in node.attrib: if self.options.tosvg == "true": doc = inkex.etree.fromstring( '<text x="%g" y="%g">%s</text>' % (self.view_center[0] - self.width / 6, self.view_center[1] - self.height / 6, node.attrib.get('{%s}text' % WriteTexNS, '').decode('string-escape'))) p = node.getparent() # p.remove(node) p.append(doc) else: print >> sys.stderr, node.attrib.get( '{%s}text' % WriteTexNS, '').decode('string-escape') return print >> sys.stderr, "No text find." return else: if action == "new": self.text = self.options.formula else: f = open(self.options.inputfile) self.text = f.read() f.close() if self.text == "": print >> sys.stderr, "empty LaTeX input. Nothing is changed." return tmp_dir = tempfile.mkdtemp("", "writetex-") tex_file = os.path.join(tmp_dir, "writetex.tex") svg_file = os.path.join(tmp_dir, "writetex.svg") pdf_file = os.path.join(tmp_dir, "writetex.pdf") log_file = os.path.join(tmp_dir, "writetex.log") out_file = os.path.join(tmp_dir, "writetex.out") err_file = os.path.join(tmp_dir, "writetex.err") aux_file = os.path.join(tmp_dir, "writetex.aux") crop_file = os.path.join(tmp_dir, "writetex-crop.pdf") if self.options.preline == "true": preamble = self.options.preamble else: if self.options.preamble == "": preamble = "" else: f = open(self.options.preamble) preamble = f.read() f.close() self.tex = r""" \documentclass[landscape,a3paper]{article} \usepackage{geometry} %s \pagestyle{empty} \begin{document} \noindent %s \end{document} """ % (preamble, self.text) tex = open(tex_file, 'w') tex.write(self.tex) tex.close() if self.options.latexcmd.lower() == "xelatex": subprocess.call( 'xelatex "-output-directory=%s" -interaction=nonstopmode -halt-on-error "%s" > "%s"' % (tmp_dir, tex_file, out_file), shell=True) elif self.options.latexcmd.lower() == "pdflatex": subprocess.call( 'pdflatex "-output-directory=%s" -interaction=nonstopmode -halt-on-error "%s" > "%s"' % (tmp_dir, tex_file, out_file), shell=True) else: # Setting `latexcmd` to following string produces the same result as xelatex condition: # 'xelatex "-output-directory={tmp_dir}" -interaction=nonstopmode -halt-on-error "{tex_file}" > "{out_file}"' subprocess.call(self.options.latexcmd.format( tmp_dir=tmp_dir, tex_file=tex_file, out_file=out_file), shell=True) try: if not isinstance(spawn.find_executable('pdfcrop'), type(None)): # Here is a bug in pdfcrop, no idea how to fix. crop_cmd = 'pdfcrop "%s"' % pdf_file crop = subprocess.Popen(crop_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) out = crop.communicate() if len(out[1]) > 0: inkex.errormsg("Error in pdfcrop:") inkex.errormsg(" CMD executed: %s" % crop_cmd) for msg in out: inkex.errormsg(msg) inkex.errormsg("Process will continue without crop") if os.path.exists(crop_file): os.remove(pdf_file) os.rename(crop_file, pdf_file) except: pass if not os.path.exists(pdf_file): print >> sys.stderr, "Latex error: check your latex file and preamble." print >> sys.stderr, open(log_file).read() return else: if self.options.pdftosvg == '1': subprocess.call('pdf2svg %s %s' % (pdf_file, svg_file), shell=True) self.merge_pdf2svg_svg(svg_file) else: subprocess.call( 'pstoedit -f plot-svg "%s" "%s" -dt -ssp -psarg -r9600x9600 > "%s" 2> "%s"' % (pdf_file, svg_file, out_file, err_file), shell=True) self.merge_pstoedit_svg(svg_file) os.remove(tex_file) os.remove(log_file) os.remove(out_file) if os.path.exists(err_file): os.remove(err_file) if os.path.exists(aux_file): os.remove(aux_file) if os.path.exists(svg_file): os.remove(svg_file) if os.path.exists(pdf_file): os.remove(pdf_file) os.rmdir(tmp_dir)
def check_pip(): return bool(spawn.find_executable('pip')), 'pip', '#pip'
# Root directory of project __C.ROOT_DIR = osp.abspath(osp.join(osp.dirname(__file__), '..', '..')) # Data directory __C.DATA_DIR = osp.abspath(osp.join(__C.ROOT_DIR, 'data')) # Model directory __C.MODELS_DIR = osp.abspath(osp.join(__C.ROOT_DIR, 'models', 'pascal_voc')) # Name (or path to) the matlab executable __C.MATLAB = 'matlab' # Place outputs under an experiments directory __C.EXP_DIR = 'default' if spawn.find_executable("nvcc"): # Use GPU implementation of non-maximum suppression __C.USE_GPU_NMS = True # Default GPU device id __C.GPU_ID = 0 else: __C.USE_GPU_NMS = False def get_output_dir(imdb, weights_filename): """Return the directory where experimental artifacts are placed. If the directory does not exist, it is created. A canonical path is built using the name from an imdb and a network (if not None).
def check_git(): return bool(spawn.find_executable('git')), 'Git', '#git'
def __init__(self): self.cmakeExecutable = find_executable('cmake') if not self.cmakeExecutable or not os.path.exists( self.cmakeExecutable): raise Exception( "Couldn't find cmake executable. Please download and install it from www.cmake.org/download" ) self.generatorHelpList = [] self.generatorNames = [] self.generatorAliasMap = {} self._cmakeError = None errOutput = "" try: errOutFile, errOutFilePath = tempfile.mkstemp() try: cmakeHelp = subprocess.check_output( "\"" + self.cmakeExecutable + "\" --help", shell=True, universal_newlines=True, stderr=errOutFile) finally: os.close(errOutFile) try: with open(errOutFilePath, "r") as f: errOutput = f.read().strip() except: pass os.remove(errOutFilePath) except subprocess.CalledProcessError as e: self._cmakeError = error.CMakeProblemError(e, errOutput) if self._cmakeError is None: cmakeHelp = cmakeHelp.strip() # the generator list is at the end of the output. And the entries are all indented. for line in reversed(cmakeHelp.splitlines()): if not line or not line.startswith(" "): break self.generatorHelpList.append(line) self.generatorHelpList.reverse() for line in self.generatorHelpList: if line.startswith(" ") and len(line) > 2 and line[2] != " ": line = line.strip() name, sep, rest = line.partition(" = ") if sep: name = name.strip() if name: self.generatorNames.append(name) vsPrefix = "Visual Studio " for name in self.generatorNames: if name.startswith(vsPrefix): words = name[len(vsPrefix):].strip().split() if len(words) >= 2: try: internalVersion = int(words[0]) yearVersion = int(words[1]) self.generatorAliasMap[ "vs" + words[1]] = vsPrefix + words[0] + " " + words[1] except Exception as e: # ignore exceptions. The generator string does not have the expected format. pass self.generatorAliasMap["make"] = "Unix Makefiles" self.generatorAliasMap["nmake"] = "NMake Makefiles" self.generatorAliasMap["msysmake"] = "MSYS Makefiles" self.generatorAliasMap["mingwmake"] = "MinGW Makefiles" if "CodeBlocks - Unix Makefiles" in self.generatorNames: self.generatorAliasMap[ "codeblocks"] = "CodeBlocks - Unix Makefiles" if "CodeLite - Unix Makefiles" in self.generatorNames: self.generatorAliasMap[ "codelite"] = "CodeLite - Unix Makefiles" self.generatorAliasHelp = "Aliases for build system names:\n" for aliasName in sorted(self.generatorAliasMap.keys()): self.generatorAliasHelp += "\n%s = %s" % ( aliasName, self.generatorAliasMap[aliasName])
def validate(self, parser, options, context): """Validate generic options.""" # and android doesn't use 'app' the same way, so skip validation if parser.app != 'android': if options.app is None: if build_obj: from mozbuild.base import BinaryNotFoundException try: options.app = build_obj.get_binary_path() except BinaryNotFoundException as e: print('{}\n\n{}\n'.format(e, e.help())) sys.exit(1) else: parser.error( "could not find the application path, --appname must be specified" ) elif options.app == "dist" and build_obj: options.app = build_obj.get_binary_path(where='staged-package') options.app = self.get_full_path(options.app, parser.oldcwd) if not os.path.exists(options.app): parser.error( "Error: Path {} doesn't exist. Are you executing " "$objdir/_tests/testing/mochitest/runtests.py?".format( options.app)) if options.flavor is None: options.flavor = 'plain' for value in ALL_FLAVORS.values(): if options.flavor in value['aliases']: options.flavor = value['suite'] break if options.gmp_path is None and options.app and build_obj: # Need to fix the location of gmp_fake which might not be shipped in the binary gmp_modules = (('gmp-fake', '1.0'), ('gmp-clearkey', '0.1'), ('gmp-fakeopenh264', '1.0')) options.gmp_path = os.pathsep.join( os.path.join(build_obj.bindir, *p) for p in gmp_modules) if options.totalChunks is not None and options.thisChunk is None: parser.error( "thisChunk must be specified when totalChunks is specified") if options.extra_mozinfo_json: if not os.path.isfile(options.extra_mozinfo_json): parser.error("Error: couldn't find mozinfo.json at '%s'." % options.extra_mozinfo_json) options.extra_mozinfo_json = json.load( open(options.extra_mozinfo_json)) if options.totalChunks: if not 1 <= options.thisChunk <= options.totalChunks: parser.error("thisChunk must be between 1 and totalChunks") if options.chunkByDir and options.chunkByRuntime: parser.error( "can only use one of --chunk-by-dir or --chunk-by-runtime") if options.xrePath is None: # default xrePath to the app path if not provided # but only if an app path was explicitly provided if options.app != parser.get_default('app'): options.xrePath = os.path.dirname(options.app) if mozinfo.isMac: options.xrePath = os.path.join( os.path.dirname(options.xrePath), "Resources") elif build_obj is not None: # otherwise default to dist/bin options.xrePath = build_obj.bindir else: parser.error( "could not find xre directory, --xre-path must be specified" ) # allow relative paths if options.xrePath: options.xrePath = self.get_full_path(options.xrePath, parser.oldcwd) if options.profilePath: options.profilePath = self.get_full_path(options.profilePath, parser.oldcwd) if options.utilityPath: options.utilityPath = self.get_full_path(options.utilityPath, parser.oldcwd) if options.certPath: options.certPath = self.get_full_path(options.certPath, parser.oldcwd) elif build_obj: options.certPath = os.path.join(build_obj.topsrcdir, 'build', 'pgo', 'certs') if options.symbolsPath and len(urlparse( options.symbolsPath).scheme) < 2: options.symbolsPath = self.get_full_path(options.symbolsPath, parser.oldcwd) elif not options.symbolsPath and build_obj: options.symbolsPath = os.path.join(build_obj.distdir, 'crashreporter-symbols') if options.debugOnFailure and not options.jsdebugger: parser.error("--debug-on-failure requires --jsdebugger.") if options.jsdebuggerPath and not options.jsdebugger: parser.error("--jsdebugger-path requires --jsdebugger.") if options.debuggerArgs and not options.debugger: parser.error("--debugger-args requires --debugger.") if options.valgrind or options.debugger: # valgrind and some debuggers may cause Gecko to start slowly. Make sure # marionette waits long enough to connect. options.marionette_startup_timeout = 900 options.marionette_socket_timeout = 540 if options.store_chrome_manifest: options.store_chrome_manifest = os.path.abspath( options.store_chrome_manifest) if not os.path.isdir(os.path.dirname( options.store_chrome_manifest)): parser.error( "directory for %s does not exist as a destination to copy a " "chrome manifest." % options.store_chrome_manifest) if options.jscov_dir_prefix: options.jscov_dir_prefix = os.path.abspath( options.jscov_dir_prefix) if not os.path.isdir(options.jscov_dir_prefix): parser.error( "directory %s does not exist as a destination for coverage " "data." % options.jscov_dir_prefix) if options.testingModulesDir is None: # Try to guess the testing modules directory. possible = [os.path.join(here, os.path.pardir, 'modules')] if build_obj: possible.insert( 0, os.path.join(build_obj.topobjdir, '_tests', 'modules')) for p in possible: if os.path.isdir(p): options.testingModulesDir = p break # Paths to specialpowers and mochijar from the tests archive. options.stagedAddons = [ os.path.join(here, 'extensions', 'specialpowers'), os.path.join(here, 'mochijar'), ] if build_obj: objdir_xpi_stage = os.path.join(build_obj.distdir, 'xpi-stage') if os.path.isdir(objdir_xpi_stage): options.stagedAddons = [ os.path.join(objdir_xpi_stage, 'specialpowers'), os.path.join(objdir_xpi_stage, 'mochijar'), ] plugins_dir = os.path.join(build_obj.distdir, 'plugins') if os.path.isdir( plugins_dir ) and plugins_dir not in options.extraProfileFiles: options.extraProfileFiles.append(plugins_dir) # Even if buildbot is updated, we still want this, as the path we pass in # to the app must be absolute and have proper slashes. if options.testingModulesDir is not None: options.testingModulesDir = os.path.normpath( options.testingModulesDir) if not os.path.isabs(options.testingModulesDir): options.testingModulesDir = os.path.abspath( options.testingModulesDir) if not os.path.isdir(options.testingModulesDir): parser.error('--testing-modules-dir not a directory: %s' % options.testingModulesDir) options.testingModulesDir = options.testingModulesDir.replace( '\\', '/') if options.testingModulesDir[-1] != '/': options.testingModulesDir += '/' if options.runUntilFailure: if not options.repeat: options.repeat = 29 if options.dumpOutputDirectory is None: options.dumpOutputDirectory = tempfile.gettempdir() if options.dumpAboutMemoryAfterTest or options.dumpDMDAfterTest: if not os.path.isdir(options.dumpOutputDirectory): parser.error('--dump-output-directory not a directory: %s' % options.dumpOutputDirectory) if options.useTestMediaDevices: if not mozinfo.isLinux: parser.error( '--use-test-media-devices is only supported on Linux currently' ) gst01 = spawn.find_executable("gst-launch-0.1") gst010 = spawn.find_executable("gst-launch-0.10") gst10 = spawn.find_executable("gst-launch-1.0") pactl = spawn.find_executable("pactl") if not (gst01 or gst10 or gst010): parser.error('Missing gst-launch-{0.1,0.10,1.0}, required for ' '--use-test-media-devices') if not pactl: parser.error('Missing binary pactl required for ' '--use-test-media-devices') # The a11y and chrome flavors can't run with e10s. if options.flavor in ('a11y', 'chrome') and options.e10s: parser.error("mochitest-{} does not support e10s, try again with " "--disable-e10s.".format(options.flavor)) if options.enable_fission: options.extraPrefs.append("fission.autostart=true") options.extraPrefs.append( "dom.serviceWorkers.parent_intercept=true") options.extraPrefs.append("browser.tabs.documentchannel=true") options.leakThresholds = { "default": options.defaultLeakThreshold, "tab": options.defaultLeakThreshold, "forkserver": options.defaultLeakThreshold, # GMP rarely gets a log, but when it does, it leaks a little. "gmplugin": 20000, "rdd": 400, } # See the dependencies of bug 1401764. if mozinfo.isWin: options.leakThresholds["tab"] = 1000 # XXX We can't normalize test_paths in the non build_obj case here, # because testRoot depends on the flavor, which is determined by the # mach command and therefore not finalized yet. Conversely, test paths # need to be normalized here for the mach case. if options.test_paths and build_obj: # Normalize test paths so they are relative to test root options.test_paths = [ build_obj._wrap_path_argument(p).relpath() for p in options.test_paths ] return options
def remap_file(self, inFileName, outFileName, variableList=None, overwrite=False, renormalize=None, logger=None): # {{{ ''' Given a source file defining either an MPAS mesh or a lat-lon grid and a destination file or set of arrays defining a lat-lon grid, constructs a mapping file used for interpolation between the source and destination grids. Parameters ---------- inFileName : str The path to the file containing a data set on the source grid outFileName : str The path where the data on the destination grid should be written variableList : list of str, optional A list of variables to be mapped. By default, all variables are mapped overwrite : bool, optional Whether the destination file should be overwritten if it already exists. If `False`, and the destination file is already present, the function does nothing and returns immediately renormalize : float, optional A threshold to use to renormalize the data logger : ``logging.Logger``, optional A logger to which ncclimo output should be redirected Raises ------ OSError If ``ncremap`` is not in the system path. ValueError If ``mappingFileName`` is ``None`` (meaning no remapping is needed). ''' # Authors # ------- # Xylar Asay-Davis if self.mappingFileName is None: raise ValueError('No mapping file was given because remapping is ' 'not necessary. The calling\n' 'code should simply use the constents of {} ' 'directly.'.format(inFileName)) if not overwrite and os.path.exists(outFileName): # a remapped file already exists, so nothing to do return if isinstance(self.sourceDescriptor, ProjectionGridDescriptor): raise TypeError('Source grid is a projection grid, not supported ' 'by ncremap.\n' 'Consider using Remapper.remap') if isinstance(self.destinationDescriptor, ProjectionGridDescriptor): raise TypeError('Destination grid is a projection grid, not ' 'supported by ncremap.\n' 'Consider using Remapper.remap') if find_executable('ncremap') is None: raise OSError('ncremap not found. Make sure the latest nco ' 'package is installed: \n' 'conda install nco\n' 'Note: this presumes use of the conda-forge ' 'channel.') args = ['ncremap', '-i', inFileName, '-m', self.mappingFileName, '--vrb=1', '-o', outFileName] regridArgs = [] if renormalize is not None: regridArgs.append('--renormalize={}'.format(renormalize)) if isinstance(self.sourceDescriptor, LatLonGridDescriptor): regridArgs.extend(['--rgr lat_nm={}'.format( self.sourceDescriptor.latVarName), '--rgr lon_nm={}'.format( self.sourceDescriptor.lonVarName)]) if len(regridArgs) > 0: args.extend(['-R', ' '.join(regridArgs)]) if isinstance(self.sourceDescriptor, MpasMeshDescriptor): # Note: using the -C (climatology) flag for now because otherwise # ncremap tries to add a _FillValue attribute that might # already be present and quits with an error args.extend(['-P', 'mpas', '-C']) if variableList is not None: args.extend(['-v', ','.join(variableList)]) # set an environment variable to make sure we're not using czender's # local version of NCO instead of one we have intentionally loaded env = os.environ.copy() env['NCO_PATH_OVERRIDE'] = 'No' if logger is None: print('running: {}'.format(' '.join(args))) # make sure any output is flushed before we add output from the # subprocess sys.stdout.flush() sys.stderr.flush() subprocess.check_call(args, env=env) else: logger.info('running: {}'.format(' '.join(args))) for handler in logger.handlers: handler.flush() process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) stdout, stderr = process.communicate() if stdout: stdout = stdout.decode('utf-8') for line in stdout.split('\n'): logger.info(line) if stderr: stderr = stderr.decode('utf-8') for line in stderr.split('\n'): logger.error(line) if process.returncode != 0: raise subprocess.CalledProcessError(process.returncode, ' '.join(args))
def __init__(self, path, cache_dir): # pylint: disable=too-many-branches """Initialize ClangFormat.""" self.path = None clang_format_progname_ext = "" if sys.platform == "win32": clang_format_progname_ext += ".exe" # Check the clang-format the user specified if path is not None: if os.path.isfile(path): self.path = path else: print("WARNING: Could not find clang-format %s" % (path)) # Check the environment variable if "MONGO_CLANG_FORMAT" in os.environ: self.path = os.environ["MONGO_CLANG_FORMAT"] if self.path and not self._validate_version(): self.path = None # Check the users' PATH environment variable now if self.path is None: # Check for various versions staring with binaries with version specific suffixes in the # user's path programs = [ CLANG_FORMAT_PROGNAME + "-" + CLANG_FORMAT_VERSION, CLANG_FORMAT_PROGNAME + "-" + CLANG_FORMAT_SHORT_VERSION, CLANG_FORMAT_PROGNAME, ] if sys.platform == "win32": for i, _ in enumerate(programs): programs[i] += '.exe' for program in programs: self.path = spawn.find_executable(program) if self.path: if not self._validate_version(): self.path = None else: break # If Windows, try to grab it from Program Files # Check both native Program Files and WOW64 version if sys.platform == "win32": programfiles = [ os.environ["ProgramFiles"], os.environ["ProgramFiles(x86)"], ] for programfile in programfiles: win32bin = os.path.join(programfile, "LLVM\\bin\\clang-format.exe") if os.path.exists(win32bin): self.path = win32bin break # Have not found it yet, download it from the web if self.path is None: if not os.path.isdir(cache_dir): os.makedirs(cache_dir) self.path = os.path.join( cache_dir, CLANG_FORMAT_PROGNAME + "-" + CLANG_FORMAT_VERSION + clang_format_progname_ext) # Download a new version if the cache is empty or stale if not os.path.isfile(self.path) or not self._validate_version(): if sys.platform.startswith("linux"): get_clang_format_from_linux_cache(self.path) elif sys.platform == "darwin": get_clang_format_from_darwin_cache(self.path) else: print("ERROR: clang-format.py does not support downloading clang-format " + " on this platform, please install clang-format " + CLANG_FORMAT_VERSION) # Validate we have the correct version # We only can fail here if the user specified a clang-format binary and it is the wrong # version if not self._validate_version(): print("ERROR: exiting because of previous warning.") sys.exit(1) self.print_lock = threading.Lock()
import sys import tempfile import warnings from distutils.spawn import find_executable from subprocess import PIPE, Popen import numpy as np from mbuild import clone from mbuild.box import Box from mbuild.compound import Compound from mbuild.exceptions import MBuildError __all__ = ["fill_box", "fill_region", "fill_sphere", "solvate"] PACKMOL = find_executable("packmol") PACKMOL_HEADER = """ tolerance {0:.16f} filetype xyz output {1} seed {2} sidemax {3} """ PACKMOL_SOLUTE = """ structure {0} number 1 center fixed {1:.3f} {2:.3f} {3:.3f} 0. 0. 0. end structure """
def build_mapping_file(self, method='bilinear', additionalArgs=None, logger=None): # {{{ ''' Given a source file defining either an MPAS mesh or a lat-lon grid and a destination file or set of arrays defining a lat-lon grid, constructs a mapping file used for interpolation between the source and destination grids. Parameters ---------- method : {'bilinear', 'neareststod', 'conserve'}, optional The method of interpolation used, see documentation for `ESMF_RegridWeightGen` for details. additionalArgs : list of str, optional A list of additional arguments to ``ESMF_RegridWeightGen`` logger : ``logging.Logger``, optional A logger to which ncclimo output should be redirected Raises ------ OSError If ``ESMF_RegridWeightGen`` is not in the system path. ValueError If sourceDescriptor or destinationDescriptor is of an unknown type ''' # Authors # ------- # Xylar Asay-Davis if self.mappingFileName is None or \ os.path.exists(self.mappingFileName): # a valid weight file already exists, so nothing to do return if find_executable('ESMF_RegridWeightGen') is None: raise OSError('ESMF_RegridWeightGen not found. Make sure esmf ' 'package is installed via\n' 'latest nco: \n' 'conda install nco\n' 'Note: this presumes use of the conda-forge ' 'channel.') # Write source and destination SCRIP files in temporary locations self.sourceDescriptor.to_scrip(_get_temp_path()) self.destinationDescriptor.to_scrip(_get_temp_path()) args = ['ESMF_RegridWeightGen', '--source', self.sourceDescriptor.scripFileName, '--destination', self.destinationDescriptor.scripFileName, '--weight', self.mappingFileName, '--method', method, '--netcdf4', '--no_log'] if self.sourceDescriptor.regional: args.append('--src_regional') if self.destinationDescriptor.regional: args.append('--dst_regional') if self.sourceDescriptor.regional or \ self.destinationDescriptor.regional: args.append('--ignore_unmapped') if additionalArgs is not None: args.extend(additionalArgs) if logger is None: print('running: {}'.format(' '.join(args))) # make sure any output is flushed before we add output from the # subprocess sys.stdout.flush() sys.stderr.flush() # throw out the standard output from ESMF_RegridWeightGen, as it's # rather verbose but keep stderr DEVNULL = open(os.devnull, 'wb') subprocess.check_call(args, stdout=DEVNULL) else: logger.info('running: {}'.format(' '.join(args))) for handler in logger.handlers: handler.flush() process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() # throw out the standard output from ESMF_RegridWeightGen, as it's # rather verbose but keep stderr if stderr: for line in stderr.split('\n'): logger.error(line) if process.returncode != 0: raise subprocess.CalledProcessError(process.returncode, ' '.join(args)) # remove the temporary SCRIP files os.remove(self.sourceDescriptor.scripFileName) os.remove(self.destinationDescriptor.scripFileName)
role_events = [ event for event in r.events if event.get('event_data', {}).get( 'role', '') == "benthomasson.hello_role" ] assert 'runner_on_ok' in [event['event'] for event in role_events] for event in role_events: event_data = event['event_data'] assert not event_data.get( 'warning', False) # role use should not contain warnings if event['event'] == 'runner_on_ok': assert event_data['res']['msg'] == 'Hello world!' finally: shutil.rmtree('test/integration/artifacts') @pytest.mark.skipif(find_executable('cgexec') is None, reason="cgexec not available") @pytest.mark.skipif(LooseVersion( pkg_resources.get_distribution('ansible').version) < LooseVersion('2.8'), reason="Valid only on Ansible 2.8+") def test_profile_data(): tdir = tempfile.mkdtemp() try: r = run(private_data_dir=tdir, inventory="localhost ansible_connection=local", resource_profiling=True, resource_profiling_base_cgroup='ansible-runner', playbook=[{ 'hosts': 'all', 'gather_facts': False, 'tasks': [{
import os import tempfile import warnings import pytest from matplotlib.font_manager import (findfont, FontProperties, fontManager, json_dump, json_load, get_font, get_fontconfig_fonts, is_opentype_cff_font, fontManager as fm) from matplotlib import rc_context if six.PY2: from distutils.spawn import find_executable has_fclist = find_executable('fc-list') is not None else: # py >= 3.3 from shutil import which has_fclist = which('fc-list') is not None def test_font_priority(): with rc_context(rc={'font.sans-serif': ['cmmi10', 'Bitstream Vera Sans']}): font = findfont(FontProperties(family=["sans-serif"])) assert os.path.basename(font) == 'cmmi10.ttf' # Smoketest get_charmap, which isn't used internally anymore font = get_font(font) cmap = font.get_charmap() assert len(cmap) == 131