def run_le_auto(le_auto_path, venv_dir, base_url=None, le_auto_args_str='--version', **kwargs): """Run the prebuilt version of letsencrypt-auto, returning stdout and stderr strings. If the command returns other than 0, raise CalledProcessError. """ env = environ.copy() d = dict(VENV_PATH=venv_dir, NO_CERT_VERIFY='1', **kwargs) if base_url is not None: # URL to PyPI-style JSON that tell us the latest released version # of LE: d['LE_AUTO_JSON_URL'] = base_url + 'certbot/json' # URL to dir containing letsencrypt-auto and letsencrypt-auto.sig: d['LE_AUTO_DIR_TEMPLATE'] = base_url + '%s/' # The public key corresponding to signing.key: d['LE_AUTO_PUBLIC_KEY'] = """-----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsMoSzLYQ7E1sdSOkwelg tzKIh2qi3bpXuYtcfFC0XrvWig071NwIj+dZiT0OLZ2hPispEH0B7ISuuWg1ll7G hFW0VdbxL6JdGzS2ShNWkX9hE9z+j8VqwDPOBn3ZHm03qwpYkBDwQib3KqOdYbTT uUtJmmGcuk3a9Aq/sCT6DdfmTSdP5asdQYwIcaQreDrOosaS84DTWI3IU+UYJVgl LsIVPBuy9IcgHidUQ96hJnoPsDCWsHwX62495QKEarauyKQrJzFes0EY95orDM47 Z5o/NDiQB11m91yNB0MmPYY9QSbnOA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68 iQIDAQAB -----END PUBLIC KEY-----""" env.update(d) return out_and_err( le_auto_path + ' ' + le_auto_args_str, shell=True, env=env)
def get_mkl_dirs_and_libs_like_numpy(): from os.path import dirname, join, isfile from sys import platform from os import environ from subprocess import check_output from glob import glob try: import numpy except ImportError: raise Exception("MKL autodetection failed: NumPy could not " "be imported. Specify MKL location manually") lapack_lite_path = glob(join(dirname(numpy.__file__), 'linalg', 'lapack_lite*.so'))[0] if not isfile(lapack_lite_path): raise Exception("MKL autodetection failed: '"+lapack_lite_path+ "' is not a file. Specify MKL location manually") if platform.startswith('darwin'): otool_output = check_output(['otool','-L',lapack_lite_path]).decode("utf-8") mkl_dirs,mkl_libs = parse_otool_output(otool_output) else: # 'linux' -- we've checked that its 'darwin' or 'linux' before env = environ.copy() # Fix for Canopy 1.0.3: ensure that LD_LIBRARY_PATH contains the # appdata/canopy-*/lib directory if "VENV_LD_LIBRARY_PATH" in env: env["LD_LIBRARY_PATH"] = env["VENV_LD_LIBRARY_PATH"] ldd_output = check_output(['ldd',lapack_lite_path],env=env).decode("utf-8") mkl_dirs,mkl_libs = parse_ldd_output(ldd_output) return set(mkl_dirs), set(mkl_libs)
def run(self): print("Running shell command:", self.command) for k in self.env: print(k, "=", self.env[k]) env = environ.copy() env.update(self.env) check_call(self.command, shell=True, env=env)
def __init__(self, path=None, python=None, cache=None, readonly=False): if path is None: path = get_env_path() if not path: raise VirtualenvPathNotFound('Path for virtualenv is not define or virtualenv is not activate') self.python = python # remove trailing slash so os.path.split() behaves correctly if path[-1] == os.path.sep: path = path[:-1] # Expand path so shell shortcuts may be used such as ~ self.path = os.path.abspath(os.path.expanduser(path)) self.env = environ.copy() if cache is not None: self.env['PIP_DOWNLOAD_CACHE'] = os.path.expanduser(os.path.expandvars(cache)) self.readonly = readonly # True if the virtual environment has been set up through open_or_create() self._ready = False
def run(*cmd, **kwargs): """Simply run a process, tweaked for internal use. Passes current environment! :param silent: Optional *keyword* to suppress any stdout. :param capture: Optional *keyword* to capture stdout. """ silent = kwargs.get('silent') is True capture = kwargs.get('capture') is True env = environ.copy() env.update(kwargs.get('env', dict())) if silent or capture: proc = Popen(cmd, stdout=PIPE, env=env) lines = list() for line in iter(proc.stdout.readline, ''): if not silent: print(line.replace('\n', '').replace('\r', '')) sys.stdout.flush() lines.append(line) return ''.join(lines).strip() else: proc = Popen(cmd, env=env) proc.wait()
def build(argv = ' '.join(argv[1:])): from sys import maxsize from os import environ environ = environ.copy() command = './bin/buildout -c buildout-build.cfg %s' % argv if system() == 'Linux': from platform import dist dist_name = dist()[0].lower() if dist_name == 'ubuntu': command = './bin/buildout -c buildout-build-ubuntu.cfg %s' % argv if dist_name in ['redhat', 'centos'] and maxsize > 2**32: command = './bin/buildout -c buildout-build-redhat-64bit.cfg %s' % argv elif system() == 'Darwin': from platform import mac_ver environ["MACOSX_DEPLOYMENT_TARGET"] = '.'.join(mac_ver()[0].split('.', 2)[:2]) command = './bin/buildout -c buildout-build-osx.cfg %s' % argv elif system() == 'Windows': if maxsize > 2**32: command = './bin/buildout -c buildout-build-windows-64bit.cfg %s' % argv else: command = './bin/buildout -c buildout-build-windows.cfg %s' % argv print 'executing "%s"' % command process = Popen(command.split(), env=environ) stdout, stderr = process.communicate() exit(process.returncode)
def setEnv(self, environment) : self.environment = environ.copy() for key in environment : self.environment[key] = environment[key] return self
def make_test_git(): directory = mkdtemp() path = getcwd() sample = abspath(pjoin(path, 'rtd_tests/fixtures/sample_repo')) directory = pjoin(directory, 'sample_repo') copytree(sample, directory) env = environ.copy() env['GIT_DIR'] = pjoin(directory, '.git') chdir(directory) # Initialize and configure log.info(check_output(['git', 'init'] + [directory], env=env)) log.info(check_output( ['git', 'config', 'user.email', '*****@*****.**'], env=env )) log.info(check_output( ['git', 'config', 'user.name', 'Read the Docs'], env=env )) # Set up the actual repository log.info(check_output(['git', 'add', '.'], env=env)) log.info(check_output(['git', 'commit', '-m"init"'], env=env)) # Add repo itself as submodule log.info(check_output(['git', 'checkout', '-b', 'submodule'], env=env)) log.info(check_output(['git', 'submodule', 'add', '-b', 'master', './', 'submodule'], env=env)) log.info(check_output(['git', 'add', '.'], env=env)) log.info(check_output(['git', 'commit', '-m"Add submodule"'], env=env)) # Checkout to master branch again log.info(check_output(['git', 'checkout', 'master'], env=env)) chdir(path) return directory
def CheckUploadSize(_, target, source, env): # pylint: disable=W0613,W0621 if "BOARD" not in env: return max_size = int(env.BoardConfig().get("upload.maximum_size", 0)) if max_size == 0 or "SIZETOOL" not in env: return sysenv = environ.copy() sysenv['PATH'] = str(env['ENV']['PATH']) cmd = [ env.subst("$SIZETOOL"), "-B", str(source[0] if isinstance(target[0], Alias) else target[0]) ] result = util.exec_command(cmd, env=sysenv) if result['returncode'] != 0: return print result['out'].strip() line = result['out'].strip().splitlines()[1] values = [v.strip() for v in line.split("\t")] used_size = int(values[0]) + int(values[1]) if used_size > max_size: sys.stderr.write("Error: The program size (%d bytes) is greater " "than maximum allowed (%s bytes)\n" % (used_size, max_size)) env.Exit(1)
def __launch_desktop_blocker(self, session_name, user_id, x11_display): print "Launch desktop-blocker to '%s'" % session_name from os import environ env = environ.copy() env["DISPLAY"] = x11_display proclist = gtop.proclist(gtop.PROCLIST_KERN_PROC_UID, int(user_id)) if len(proclist) > 0 : from subprocess import Popen, PIPE lang_var = Popen('cat /proc/%s/environ | tr "\\000" "\\n" | grep ^LANG= ' % proclist[0] , shell=True, stdout=PIPE).stdout.readline().strip("\n") if len(lang_var) > 0 : env["LANG"] = lang_var.replace("LANG=","") pid = Popen('nanny-desktop-blocker', env=env).pid else: pid = Popen('nanny-desktop-blocker', env=env).pid pid_file = "/var/lib/nanny/desktop_blocks_pids/%s.%s" % (user_id, os.path.basename(session_name)) fd = open(pid_file, "w") fd.write(str(pid)) fd.close() pid, ret = os.waitpid(pid, 0) if os.path.exists(pid_file) : os.unlink(pid_file) return session_name, user_id, ret
def run(self, args): self.check_not_docsearch_app_id('run a config manually') run_command = ["python", "scraper/src/index.py"] env = environ.copy() env.update({'CONFIG': args[0], 'INDEX_PREFIX': ""}) return self.exec_shell_command(run_command, env)
def pipe_output(read, write): from os import environ environ = environ.copy() environ['HOME'] = str(Path('.').realpath()) from subprocess import Popen vupdate = Popen( ('venv-update', '--version'), env=environ, stdout=write, close_fds=True, ) from os import close from testing.capture_subprocess import read_all close(write) result = read_all(read) vupdate.wait() result = result.decode('US-ASCII') uncolored = uncolor(result) assert uncolored.startswith('> ') # FIXME: Sometimes this is 'python -m', sometimes 'python2.7 -m'. Weird. assert uncolored.endswith('''\ -m virtualenv virtualenv_run --version 1.11.6 ''') return result, uncolored
def pipe_output(read, write): from os import environ environ = environ.copy() from subprocess import Popen vupdate = Popen( ('venv-update', 'venv=', '--version'), env=environ, stdout=write, close_fds=True, ) from os import close from testing.capture_subprocess import read_all close(write) result = read_all(read) vupdate.wait() result = result.decode('US-ASCII') print(result) uncolored = uncolor(result) assert uncolored.startswith('> ') # FIXME: Sometimes this is 'python -m', sometimes 'python2.7 -m'. Weird. import virtualenv assert uncolored.endswith(''' > virtualenv --version %s ''' % virtualenv.__version__) return result, uncolored
def execSubproc(exelist, envdict=None, shell=True): """ Function to execute commands in various ways. """ from os import environ, path import subprocess # change environ of subprocess if nec. env = environ.copy() if envdict: for var, val in envdict.iteritems(): env[var] = val # execute each command in the given list for exe in exelist: if shell: exe = " ".join(exe) # print "==> Executing command '{}' ...".format(exe) subproc = subprocess.Popen(exe, env=env, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pout, perr = subproc.communicate() if pout: print pout if perr: print perr return False # print "==> Execution complete." # all done executing commands? ok... return True
def delete_git_tag(directory, tag): env = environ.copy() env['GIT_DIR'] = pjoin(directory, '.git') chdir(directory) command = ['git', 'tag', '--delete', tag] check_output(command, env=env)
def runInVT(it, cmd, envm=None, stdin=PIPE): from subprocess import Popen as ppn, STDOUT from shlex import split as shs from os import setsid from psutil import Process as pss if envm is None: from os import environ as env envm = env.copy() # drop run if busy by previous task if type(it.proc) is ppn and(it.proc.poll()== None): return False it.proc = ppn(shs(cmd), env=envm, #stdin=it.fifo_in_fn, stdin=stdin, stdout=it.pty_child_fd, stderr=STDOUT, preexec_fn=setsid) it.proc.children = None try: p = pss(it.proc.pid) it.proc.children = p.get_children(recursive=True) except: pass #it.std_reply = open(it.fifo_in_fn, 'wb') while it.proc.poll()==None: # poll()==None means still running slp(.5) #it.std_reply.close() it.proc = None return True
def run_compiler(self, files="test.proto"): if not isinstance(files, list): files = [files] files = map(lambda name: path.join(self.proto_dir.name, name), files) new_env = environ.copy() new_env["PATH"] += ":" + path.normpath(path.join(path.dirname(__file__), "..", "bin")) if "PYTHONPATH" in new_env: new_env["PYTHONPATH"] += ":" + path.normpath(path.join(path.dirname(__file__), "..")) else: new_env["PYTHONPATH"] = path.normpath(path.join(path.dirname(__file__), "..")) args = ["protoc", "--python3_out=" + self.out_dir.name, "--proto_path=" + self.proto_dir.name] args.extend(files) proc = Popen(args, stderr=PIPE, env=new_env) proc.wait() if proc.returncode: value = proc.stderr.readline() while value: print(value) value = proc.stderr.readline() raise ValueError
def delete_git_branch(directory, branch): env = environ.copy() env['GIT_DIR'] = pjoin(directory, '.git') chdir(directory) command = ['git', 'branch', '-D', branch] check_output(command, env=env)
def test_handler(): tmp = mkdtemp() with open('{}/{}.py'.format(tmp, handler_module), 'w') as out: out.write(handler_code) env = environ.copy() env['PYTHONPATH'] = '{}:{}'.format(tmp, env.get('PYTHONPATH', '')) sock_path = '{}/nuclio.sock'.format(tmp) run_test_server(sock_path) handler = '{}:{}'.format(handler_module, handler_func) py_file = '{}/wrapper.py'.format(here) cmd = [ executable, py_file, '--handler', handler, '--socket-path', sock_path, ] child = Popen(cmd, env=env) try: timeout = 3 # In seconds if not RequestHandler.done.wait(timeout): assert False, 'No reply after {} seconds'.format(timeout) assert len(RequestHandler.messages) == 3, 'Bad number of message' log = RequestHandler.messages[0] assert 'message' in log, 'No message in log' metric = RequestHandler.messages[1] assert 'duration' in metric, 'No duration in metric' out = RequestHandler.messages[-1]['body'] assert out.encode('utf-8') == payload[::-1], 'Bad output' finally: child.kill()
def run(self): errored = False try: cmd = [self.__get_parameter('psql_path', '/usr/bin/psql'), '--no-password'] environment = environ.copy() for name in self.parent.settings.postgres_variables: if name in self.parent.settings and self.parent.settings[name]: self.__try_add_parameter_name_to_environment(environment, name, self.parent.settings.postgres_variables[name]) client_encoding_name = self.parent.settings.postgres_variables['client_encoding'] if client_encoding_name not in environment: environment[client_encoding_name] = self.parent.encoding if (self.file): with open(self.file) as inputfile: psqlprocess = Popen(cmd, stdin=inputfile, stdout=PIPE, stderr=STDOUT, env=environment) stdout, stderr = psqlprocess.communicate() else: psqlprocess = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=STDOUT, env=environment) stdout, stderr = psqlprocess.communicate(bytes(self.query, self.parent.encoding)) output_text = stdout.decode(self.parent.encoding) retcode = psqlprocess.poll() except BaseException as e: errored = True output_text = format_exc() retcode = 1 set_timeout(lambda:self.__output(retcode, output_text), 0)
def run_compiler(self, files='test.proto'): if not isinstance(files, list): files = [files] files = map(lambda name: path.join(self.proto_dir.name, name), files) new_env = environ.copy() new_env['PATH'] += ':' + path.normpath(path.join(path.dirname(__file__), '..', 'bin')) if 'PYTHONPATH' in new_env: new_env['PYTHONPATH'] += ':' + path.normpath(path.join(path.dirname(__file__), '..')) else: new_env['PYTHONPATH'] = path.normpath(path.join(path.dirname(__file__), '..')) args = [ 'protoc', '--python3_out=' + self.out_dir.name, '--proto_path=' + self.proto_dir.name, ] args.extend(files) proc = Popen(args, stderr=PIPE, env=new_env) proc.wait() if proc.returncode: value = proc.stderr.readline() while value: print(value) value = proc.stderr.readline() raise ValueError
def __init__(self, cwd=None, path=None, server_bind=None, verbosity=0, strace_depth=3, export=False): super().__init__() self.cwd = cwd or os.getcwd() self.env = environ.copy() self.server = rts.CraftrRuntimeServer(self) self.server_bind = server_bind self.rts_funcs = {} self.ext_importer = ext.CraftrImporter(self) self.path = [craftr.path.join(craftr.path.dirname(__file__), 'lib')] self.modules = {} self.targets = {} self.var = {} self.verbosity = verbosity self.strace_depth = strace_depth self.export = export if path is not None: self.path.extend(path) self.targets['clean'] = Target( command = 'ninja -t clean', inputs = None, outputs = None, name = 'clean', module = None, explicit = True)
def on_context_enter(self, prev): ''' Called when entering the Session context with :func:`magic.enter_context`. Does the following things: * Sets up the :data`os.environ` with the values from :attr:`Session.env` * Adds the :attr:`Session.ext_importer` to :data:`sys.meta_path` * Starts the Craftr Runtime Server (:attr:`Session.server`) and sets the ``CRAFTR_RTS`` environment variable .. note:: A copy of the original :data:`os.environ` is saved and later restored in :meth:`on_context_leave`. The :data:`os.environ` object can not be replaced by another object, that is why we change its values in-place. ''' if prev is not None: raise RuntimeError('session context can not be nested') # We can not change os.environ effectively, we must update the # dictionary instead. self._old_environ = environ.copy() environ.clear() environ.update(self.env) self.env = environ sys.meta_path.append(self.ext_importer) self.update()
def on_context_leave(self): ''' Called when the context manager entered with :func:`magic.enter_context` is exited. Undos all of the stuff that :meth:`on_context_enter` did and more. * Stop the Craftr Runtime Server * Restore the :data:`os.environ` dictionary * Removes all ``craftr.ext.`` modules from :data:`sys.modules` and ensures they are in :attr:`Session.modules` (they are expected to be put there from the :class:`ext.CraftrImporter`). ''' self._stop_server() # Restore the original values of os.environ. self.env = environ.copy() environ.clear() environ.update(self._old_environ) del self._old_environ sys.meta_path.remove(self.ext_importer) for key, module in list(sys.modules.items()): if key.startswith('craftr.ext.'): name = key[11:] assert name in self.modules and self.modules[name] is module, key del sys.modules[key] try: # Remove the module from the `craftr.ext` modules contents, too. delattr(ext, name.split('.')[0]) except AttributeError: pass
def _rpm_addsign_rewrites_the_file(filepath): from os import environ logger.info("Signing {!r}".format(filepath)) command = ['rpm', '--addsign', filepath] env = environ.copy() env['HOME'] = env.get('HOME', "/root") env['GNUPGHOME'] = path.join(env.get('HOME', "/root"), ".gnupg") log_execute_assert_success('echo | setsid rpm --addsign {}'.format(filepath), env=env, shell=True)
def execute_with_buildout(commandline_or_args, env=None): from os import name, path, environ _env = environ.copy() if env: _env.update(env) args = parse_args(commandline_or_args) execute_assert_success([path.join('bin', 'buildout{}'.format('.exe' if name == 'nt' else ''))] + \ BUILDOUT_PARAMETERS + args, env=_env)
def __init__(self, path, cache=None): # remove trailing slash so os.path.split() behaves correctly if path[-1] == '/': path = path[:-1] self.path = path self.env = environ.copy() if cache is not None: self.env['PIP_DOWNLOAD_CACHE'] = os.path.expanduser(os.path.expandvars(cache))
def execute_reproducer(self, cgroup, inputfile, functionname): menv = environ.copy() menv["ASAN_OPTIONS"] = "detect_leaks=0" infd = open(inputfile, "r") (returncode, stdoutdata, stderrdata) = cgroups_run_timed_subprocess( [self.reproducer, "--fuzz-driver=" + functionname], cgroup=cgroup, stdin=infd, env=menv) ret = AsanResult(stderrdata, inputfile, functionname) infd.close() return ret
def process(self, purge_meta_data=True): """ Calls the external helper scripts to (optionally) purge the meta data and then send the contents of the dropbox via email. """ fs_process = join(self.container.settings["fs_bin_path"], "process.sh") fs_config = join(self.container.settings["fs_bin_path"], "briefkasten.conf") shellenv = environ.copy() shellenv["PATH"] = "%s:%s" % (shellenv["PATH"], self.container.settings["fs_bin_path"]) return call("%s -d %s -c %s" % (fs_process, self.fs_path, fs_config), shell=True, env=shellenv)
def setup_environment(): """Setup test environment for some functions that are tested in this module. In particular this functions stores attributes and other things that we need to stub in some test functions. This needs to be done on a function level and not module level because each testfunction needs a pristine environment. """ global GIVEN_ENV GIVEN_ENV['env'] = env.copy()
def __init__(self, args=None): if args == None: args = [__file__] guifile = "/usr/share/pywinery/gui.glade" localgui = path_join(dirname(args[0]), "gui.glade") if isfile(localgui): guifile = localgui self.killable_threads = [] self.xml = gtk.glade.XML(guifile) self.configfile = expandvars("$HOME/.config/pywinery/prefixes.config") self.configlines = [] self.readConfigFile() self.msi = None self.wineversion = [0] self.autocreateprefix = False self.errors = ErrorManager() self.winebin = getBin("wine") if not bool(self.winebin): self.showError("nowine", "Wine is not detected on your system") self.xml.get_widget("expander1").set_property("visible", False) self.xml.get_widget("button1").set_property("visible", False) self.xml.get_widget("button8").set_property("visible", False) else: self.wineversion = getWineVersion() self.autocreateprefix = self.wineversion > (1, ) self.xml.get_widget("button13").set_property( "visible", checkBin("wine-doors")) self.silent = False self.nodebug = False c = 1 for i in args[1:]: if i[0] != "-": break # Given commands can contains - and -- too elif i in ("-x", "--nogui"): self.silent = True elif i in ("-s", "--silent"): self.nodebug = True c += 1 self.zig = args[c:] self.favprefix = None self.path = None if self.zig: path = realpath(self.zig[0]) self.path = dirname(path) sp = self.path.split(sep) for i in ("drive_c", "c:"): if i in sp: self.favprefix = sep.join(sp[:sp.index(i)]) if not self.favprefix in self.configlines: self.unknowndir() break if isfile(self.zig[0]) and guess_type(realpath( self.zig[0]))[0].lower() == "application/x-msi": self.msi = realpath(args[1]) self.xml.get_widget("hbuttonbox1").set_property("visible", True) self.xml.get_widget("hbox2").set_property("visible", False) self.xml.get_widget("button1").set_property( "visible", not bool(self.msi)) self.xml.get_widget("button8").set_property( "visible", bool(self.msi)) else: self.xml.get_widget("expander1").set_expanded(True) self.xml.get_widget("hbuttonbox1").set_property("visible", False) self.xml.get_widget("hbox2").set_property("visible", True) for i in (11, 1, 3, 4, 5, 6, 7): self.xml.get_widget("label%d" % i).set_property("visible", False) self.xml.get_widget("vbox10").set_property("visible", False) self.comboInit() dic = { "on_window1_destroy": self.__quit, "on_button12_clicked": self.__quit, "on_combobox1_changed": self.combochange, "on_button1_clicked": None, "on_button6_clicked": self.adddir, "on_button7_clicked": self.removeprefix, "on_button1_clicked": self.runAndExit, "on_button8_clicked": lambda x: (self.execute([self.winebin, "msiexec", "/i", self.msi]), self.__quit()), "on_button2_clicked": lambda x: self.execute("winecfg"), "on_button4_clicked": lambda x: self.execute("winefile"), "on_button5_clicked": lambda x: self.execute([self.winebin, "uninstaller"]), "on_button3_clicked": lambda x: self.execute(["xdg-open", self.getComboValue()]), "on_button10_clicked": self.createPrefix, "on_button11_clicked": lambda x: self.execute(["xterm", "-e", "wine", "cmd"]), "on_button13_clicked": lambda x: self.execute("wine-doors"), "on_dialog1_delete_event": returnFalse, } self.xml.signal_autoconnect(dic) self.env = environ.copy() if self.nodebug: self.env["WINEDEBUG"] = "-all"
from time import sleep from subprocess import Popen, call """ Component test fixtures. This module makes the following assumptions: * py.test is invoked from the same directory as this module is located * start_outside_service.py is located in the same directory * dbus-proxy is found in a build/ directory one level above, i.e. "../build/dbus-proxy" """ OUTSIDE_SOCKET = "/tmp/dbus_proxy_outside_socket" INSIDE_SOCKET = "/tmp/dbus_proxy_inside_socket" # Setup an environment for the fixtures to share so the bus address is the same for all environment = environ.copy() environment["DBUS_SESSION_BUS_ADDRESS"] = "unix:path=" + OUTSIDE_SOCKET @pytest.fixture(scope="function") def session_bus(request): """ Create a session bus. The dbus-deamon will be torn down at the end of the test. """ # TODO: Parametrize the socket path. dbus_daemon = None # The 'exec' part is a workaround to make the whole process group be killed # later when kill() is caled and not just the shell. This is only needed when # 'shell' is set to True like in the later Popen() call below.
objWord.ActiveDocument.Close End Sub """ code = code_template.format( f0=f0, f1=f1, ) vbs = open(unix_vbs, "w") vbs.write(code) vbs.close() # https://stackoverflow.com/questions/2231227/python-subprocess-popen-with-a-modified-environment e = environ.copy() e["WINEPREFIX"] = WINEPREFIX win_vbs = winpath(unix_vbs) # https://leereid.wordpress.com/2011/08/03/how-to-run-vbscript/ """ cscript = Popen( [ "wine", "C:\\windows\\command\\start.exe", "/Unix", WINEPREFIX + "/dosdevices/c:/Windows/system32/cscript", win_vbs, ], env = e
def get_recipe_env(self, arch=None, with_flags_in_cc=True): if self.from_crystax: return super(GuestPythonRecipe, self).get_recipe_env( arch=arch, with_flags_in_cc=with_flags_in_cc) env = environ.copy() android_host = env['HOSTARCH'] = arch.command_prefix toolchain = '{toolchain_prefix}-{toolchain_version}'.format( toolchain_prefix=self.ctx.toolchain_prefix, toolchain_version=self.ctx.toolchain_version) toolchain = join(self.ctx.ndk_dir, 'toolchains', toolchain, 'prebuilt', build_platform) env['CC'] = ( '{clang} -target {target} -gcc-toolchain {toolchain}').format( clang=join(self.ctx.ndk_dir, 'toolchains', 'llvm', 'prebuilt', build_platform, 'bin', 'clang'), target=arch.target, toolchain=toolchain) env['AR'] = join(toolchain, 'bin', android_host) + '-ar' env['LD'] = join(toolchain, 'bin', android_host) + '-ld' env['RANLIB'] = join(toolchain, 'bin', android_host) + '-ranlib' env['READELF'] = join(toolchain, 'bin', android_host) + '-readelf' env['STRIP'] = join(toolchain, 'bin', android_host) + '-strip' env['STRIP'] += ' --strip-debug --strip-unneeded' env['PATH'] = ('{hostpython_dir}:{old_path}').format( hostpython_dir=self.get_recipe('host' + self.name, self.ctx).get_path_to_python(), old_path=env['PATH']) ndk_flags = ( '-fPIC --sysroot={ndk_sysroot} -D__ANDROID_API__={android_api} ' '-isystem {ndk_android_host} -I{ndk_include}').format( ndk_sysroot=join(self.ctx.ndk_dir, 'sysroot'), android_api=self.ctx.ndk_api, ndk_android_host=join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include', android_host), ndk_include=join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include')) sysroot = self.ctx.ndk_platform env['CFLAGS'] = env.get('CFLAGS', '') + ' ' + ndk_flags env['CPPFLAGS'] = env.get('CPPFLAGS', '') + ' ' + ndk_flags env['LDFLAGS'] = env.get('LDFLAGS', '') + ' --sysroot={} -L{}'.format( sysroot, join(sysroot, 'usr', 'lib')) # Manually add the libs directory, and copy some object # files to the current directory otherwise they aren't # picked up. This seems necessary because the --sysroot # setting in LDFLAGS is overridden by the other flags. # TODO: Work out why this doesn't happen in the original # bpo-30386 Makefile system. logger.warning('Doing some hacky stuff to link properly') lib_dir = join(sysroot, 'usr', 'lib') if arch.arch == 'x86_64': lib_dir = join(sysroot, 'usr', 'lib64') env['LDFLAGS'] += ' -L{}'.format(lib_dir) shprint(sh.cp, join(lib_dir, 'crtbegin_so.o'), './') shprint(sh.cp, join(lib_dir, 'crtend_so.o'), './') env['SYSROOT'] = sysroot if sh.which('lld') is not None: # Note: The -L. is to fix a bug in python 3.7. # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409 env["LDFLAGS"] += ' -L. -fuse-ld=lld' else: logger.warning('lld not found, linking without it. ' + 'Consider installing lld if linker errors occur.') return env
def cmd(self, command, **kwargs): # prepare the environ, based on the system + our own env env = environ.copy() env.update(self.environ) # prepare the process kwargs.setdefault('env', env) kwargs.setdefault('stdout', PIPE) kwargs.setdefault('stderr', PIPE) kwargs.setdefault('close_fds', True) kwargs.setdefault('shell', True) kwargs.setdefault('show_output', self.log_level > 1) show_output = kwargs.pop('show_output') get_stdout = kwargs.pop('get_stdout', False) get_stderr = kwargs.pop('get_stderr', False) break_on_error = kwargs.pop('break_on_error', True) sensible = kwargs.pop('sensible', False) run_condition = kwargs.pop('run_condition', None) quiet = kwargs.pop('quiet', False) if not quiet: if not sensible: self.debug('Run {0!r}'.format(command)) else: if isinstance(command, (list, tuple)): self.debug('Run {0!r} ...'.format(command[0])) else: self.debug('Run {0!r} ...'.format(command.split()[0])) self.debug('Cwd {}'.format(kwargs.get('cwd'))) # open the process if sys.platform == 'win32': kwargs.pop('close_fds', None) process = Popen(command, **kwargs) # prepare fds fd_stdout = process.stdout.fileno() fd_stderr = process.stderr.fileno() if fcntl: fcntl.fcntl(fd_stdout, fcntl.F_SETFL, fcntl.fcntl(fd_stdout, fcntl.F_GETFL) | os.O_NONBLOCK) fcntl.fcntl(fd_stderr, fcntl.F_SETFL, fcntl.fcntl(fd_stderr, fcntl.F_GETFL) | os.O_NONBLOCK) ret_stdout = [] if get_stdout else None ret_stderr = [] if get_stderr else None while not run_condition or run_condition(): try: readx = select.select([fd_stdout, fd_stderr], [], [], 1)[0] except select.error: break if fd_stdout in readx: chunk = process.stdout.read() if not chunk: break if get_stdout: ret_stdout.append(chunk) if show_output: stdout.write(chunk.decode('utf-8', 'replace')) if fd_stderr in readx: chunk = process.stderr.read() if not chunk: break if get_stderr: ret_stderr.append(chunk) if show_output: stderr.write(chunk.decode('utf-8', 'replace')) stdout.flush() stderr.flush() try: process.communicate( timeout=(1 if run_condition and not run_condition() else None)) except TimeoutExpired: pass if process.returncode != 0 and break_on_error: self.error('Command failed: {0}'.format(command)) self.log_env(self.ERROR, kwargs['env']) self.error('') self.error('Buildozer failed to execute the last command') if self.log_level <= self.INFO: self.error( 'If the error is not obvious, please raise the log_level to 2' ) self.error('and retry the latest command.') else: self.error( 'The error might be hidden in the log above this error') self.error( 'Please read the full log, and search for it before') self.error('raising an issue with buildozer itself.') self.error( 'In case of a bug report, please add a full log with log_level = 2' ) raise BuildozerCommandException() if ret_stdout: ret_stdout = b''.join(ret_stdout) if ret_stderr: ret_stderr = b''.join(ret_stderr) return (ret_stdout.decode('utf-8', 'ignore') if ret_stdout else None, ret_stderr.decode('utf-8') if ret_stderr else None, process.returncode)
class Context(object): '''A build context. If anything will be built, an instance this class will be instantiated and used to hold all the build state.''' env = environ.copy() root_dir = None # the filepath of toolchain.py storage_dir = None # the root dir where builds and dists will be stored build_dir = None # in which bootstraps are copied for building # and recipes are built dist_dir = None # the Android project folder where everything ends up libs_dir = None # where Android libs are cached after build but # before being placed in dists aars_dir = None ccache = None # whether to use ccache cython = None # the cython interpreter name ndk_platform = None # the ndk platform directory dist_name = None # should be deprecated in favour of self.dist.dist_name bootstrap = None bootstrap_build_dir = None recipe_build_order = None # Will hold the list of all built recipes symlink_java_src = False # If True, will symlink instead of copying during build java_build_tool = 'auto' @property def packages_path(self): '''Where packages are downloaded before being unpacked''' return join(self.storage_dir, 'packages') @property def templates_dir(self): return join(self.root_dir, 'templates') @property def libs_dir(self): # Was previously hardcoded as self.build_dir/libs dir = join(self.build_dir, 'libs_collections', self.bootstrap.distribution.name) ensure_dir(dir) return dir @property def javaclass_dir(self): # Was previously hardcoded as self.build_dir/java dir = join(self.build_dir, 'javaclasses', self.bootstrap.distribution.name) ensure_dir(dir) return dir @property def aars_dir(self): dir = join(self.build_dir, 'aars', self.bootstrap.distribution.name) ensure_dir(dir) return dir @property def python_installs_dir(self): dir = join(self.build_dir, 'python-installs') ensure_dir(dir) return dir def get_python_install_dir(self): dir = join(self.python_installs_dir, self.bootstrap.distribution.name) return dir def setup_dirs(self, storage_dir): '''Calculates all the storage and build dirs, and makes sure the directories exist where necessary.''' self.storage_dir = expanduser(storage_dir) if ' ' in self.storage_dir: raise ValueError('storage dir path cannot contain spaces, please ' 'specify a path with --storage-dir') self.build_dir = join(self.storage_dir, 'build') self.dist_dir = join(self.storage_dir, 'dists') def ensure_dirs(self): ensure_dir(self.storage_dir) ensure_dir(self.build_dir) ensure_dir(self.dist_dir) ensure_dir(join(self.build_dir, 'bootstrap_builds')) ensure_dir(join(self.build_dir, 'other_builds')) @property def android_api(self): '''The Android API being targeted.''' if self._android_api is None: raise ValueError('Tried to access android_api but it has not ' 'been set - this should not happen, something ' 'went wrong!') return self._android_api @android_api.setter def android_api(self, value): self._android_api = value @property def ndk_ver(self): '''The version of the NDK being used for compilation.''' if self._ndk_ver is None: raise ValueError('Tried to access ndk_ver but it has not ' 'been set - this should not happen, something ' 'went wrong!') return self._ndk_ver @ndk_ver.setter def ndk_ver(self, value): self._ndk_ver = value @property def sdk_dir(self): '''The path to the Android SDK.''' if self._sdk_dir is None: raise ValueError('Tried to access sdk_dir but it has not ' 'been set - this should not happen, something ' 'went wrong!') return self._sdk_dir @sdk_dir.setter def sdk_dir(self, value): self._sdk_dir = value @property def ndk_dir(self): '''The path to the Android NDK.''' if self._ndk_dir is None: raise ValueError('Tried to access ndk_dir but it has not ' 'been set - this should not happen, something ' 'went wrong!') return self._ndk_dir @ndk_dir.setter def ndk_dir(self, value): self._ndk_dir = value def prepare_build_environment(self, user_sdk_dir, user_ndk_dir, user_android_api, user_ndk_ver): '''Checks that build dependencies exist and sets internal variables for the Android SDK etc. ..warning:: This *must* be called before trying any build stuff ''' self.ensure_dirs() if self._build_env_prepared: return # AND: This needs revamping to carefully check each dependency # in turn ok = True # Work out where the Android SDK is sdk_dir = None if user_sdk_dir: sdk_dir = user_sdk_dir if sdk_dir is None: # This is the old P4A-specific var sdk_dir = environ.get('ANDROIDSDK', None) if sdk_dir is None: # This seems used more conventionally sdk_dir = environ.get('ANDROID_HOME', None) if sdk_dir is None: # Checks in the buildozer SDK dir, useful # # for debug tests of p4a possible_dirs = glob.glob(expanduser(join( '~', '.buildozer', 'android', 'platform', 'android-sdk-*'))) possible_dirs = [d for d in possible_dirs if not (d.endswith('.bz2') or d.endswith('.gz'))] if possible_dirs: info('Found possible SDK dirs in buildozer dir: {}'.format( ', '.join([d.split(os.sep)[-1] for d in possible_dirs]))) info('Will attempt to use SDK at {}'.format(possible_dirs[0])) warning('This SDK lookup is intended for debug only, if you ' 'use python-for-android much you should probably ' 'maintain your own SDK download.') sdk_dir = possible_dirs[0] if sdk_dir is None: warning('Android SDK dir was not specified, exiting.') exit(1) self.sdk_dir = realpath(sdk_dir) # Check what Android API we're using android_api = None if user_android_api: android_api = user_android_api if android_api is not None: info('Getting Android API version from user argument') if android_api is None: android_api = environ.get('ANDROIDAPI', None) if android_api is not None: info('Found Android API target in $ANDROIDAPI') if android_api is None: info('Android API target was not set manually, using ' 'the default of {}'.format(DEFAULT_ANDROID_API)) android_api = DEFAULT_ANDROID_API android_api = int(android_api) self.android_api = android_api if self.android_api >= 21 and self.archs[0].arch == 'armeabi': error('Asked to build for armeabi architecture with API ' '{}, but API 21 or greater does not support armeabi'.format( self.android_api)) error('You probably want to build with --arch=armeabi-v7a instead') exit(1) if exists(join(sdk_dir, 'tools', 'bin', 'avdmanager')): avdmanager = sh.Command(join(sdk_dir, 'tools', 'bin', 'avdmanager')) targets = avdmanager('list', 'target').stdout.decode('utf-8').split('\n') elif exists(join(sdk_dir, 'tools', 'android')): android = sh.Command(join(sdk_dir, 'tools', 'android')) targets = android('list').stdout.decode('utf-8').split('\n') else: error('Could not find `android` or `sdkmanager` binaries in ' 'Android SDK. Exiting.') apis = [s for s in targets if re.match(r'^ *API level: ', s)] apis = [re.findall(r'[0-9]+', s) for s in apis] apis = [int(s[0]) for s in apis if s] info('Available Android APIs are ({})'.format( ', '.join(map(str, apis)))) if android_api in apis: info(('Requested API target {} is available, ' 'continuing.').format(android_api)) else: warning(('Requested API target {} is not available, install ' 'it with the SDK android tool.').format(android_api)) warning('Exiting.') exit(1) # Find the Android NDK # Could also use ANDROID_NDK, but doesn't look like many tools use this ndk_dir = None if user_ndk_dir: ndk_dir = user_ndk_dir if ndk_dir is not None: info('Getting NDK dir from from user argument') if ndk_dir is None: # The old P4A-specific dir ndk_dir = environ.get('ANDROIDNDK', None) if ndk_dir is not None: info('Found NDK dir in $ANDROIDNDK') if ndk_dir is None: # Apparently the most common convention ndk_dir = environ.get('NDK_HOME', None) if ndk_dir is not None: info('Found NDK dir in $NDK_HOME') if ndk_dir is None: # Another convention (with maven?) ndk_dir = environ.get('ANDROID_NDK_HOME', None) if ndk_dir is not None: info('Found NDK dir in $ANDROID_NDK_HOME') if ndk_dir is None: # Checks in the buildozer NDK dir, useful # # for debug tests of p4a possible_dirs = glob.glob(expanduser(join( '~', '.buildozer', 'android', 'platform', 'android-ndk-r*'))) if possible_dirs: info('Found possible NDK dirs in buildozer dir: {}'.format( ', '.join([d.split(os.sep)[-1] for d in possible_dirs]))) info('Will attempt to use NDK at {}'.format(possible_dirs[0])) warning('This NDK lookup is intended for debug only, if you ' 'use python-for-android much you should probably ' 'maintain your own NDK download.') ndk_dir = possible_dirs[0] if ndk_dir is None: warning('Android NDK dir was not specified, exiting.') exit(1) self.ndk_dir = realpath(ndk_dir) # Find the NDK version, and check it against what the NDK dir # seems to report ndk_ver = None if user_ndk_ver: ndk_ver = user_ndk_ver if ndk_dir is not None: info('Got NDK version from from user argument') if ndk_ver is None: ndk_ver = environ.get('ANDROIDNDKVER', None) if ndk_dir is not None: info('Got NDK version from $ANDROIDNDKVER') self.ndk = 'google' try: with open(join(ndk_dir, 'RELEASE.TXT')) as fileh: reported_ndk_ver = fileh.read().split(' ')[0].strip() except IOError: pass else: if reported_ndk_ver.startswith('crystax-ndk-'): reported_ndk_ver = reported_ndk_ver[12:] self.ndk = 'crystax' if ndk_ver is None: ndk_ver = reported_ndk_ver info(('Got Android NDK version from the NDK dir: ' 'it is {}').format(ndk_ver)) else: if ndk_ver != reported_ndk_ver: warning('NDK version was set as {}, but checking ' 'the NDK dir claims it is {}.'.format( ndk_ver, reported_ndk_ver)) warning('The build will try to continue, but it may ' 'fail and you should check ' 'that your setting is correct.') warning('If the NDK dir result is correct, you don\'t ' 'need to manually set the NDK ver.') if ndk_ver is None: warning('Android NDK version could not be found, exiting.') exit(1) self.ndk_ver = ndk_ver info('Using {} NDK {}'.format(self.ndk.capitalize(), self.ndk_ver)) virtualenv = None if virtualenv is None: virtualenv = sh.which('virtualenv2') if virtualenv is None: virtualenv = sh.which('virtualenv-2.7') if virtualenv is None: virtualenv = sh.which('virtualenv') if virtualenv is None: raise IOError('Couldn\'t find a virtualenv executable, ' 'you must install this to use p4a.') self.virtualenv = virtualenv info('Found virtualenv at {}'.format(virtualenv)) # path to some tools self.ccache = sh.which("ccache") if not self.ccache: info('ccache is missing, the build will not be optimized in the ' 'future.') for cython_fn in ("cython2", "cython-2.7", "cython"): cython = sh.which(cython_fn) if cython: self.cython = cython break else: error('No cython binary found. Exiting.') exit(1) if not self.cython: ok = False warning("Missing requirement: cython is not installed") # AND: need to change if supporting multiple archs at once arch = self.archs[0] platform_dir = arch.platform_dir toolchain_prefix = arch.toolchain_prefix toolchain_version = None self.ndk_platform = join( self.ndk_dir, 'platforms', 'android-{}'.format(self.android_api), platform_dir) if not exists(self.ndk_platform): warning('ndk_platform doesn\'t exist: {}'.format( self.ndk_platform)) ok = False py_platform = sys.platform if py_platform in ['linux2', 'linux3']: py_platform = 'linux' toolchain_versions = [] toolchain_path = join(self.ndk_dir, 'toolchains') if os.path.isdir(toolchain_path): toolchain_contents = glob.glob('{}/{}-*'.format(toolchain_path, toolchain_prefix)) toolchain_versions = [split(path)[-1][len(toolchain_prefix) + 1:] for path in toolchain_contents] else: warning('Could not find toolchain subdirectory!') ok = False toolchain_versions.sort() toolchain_versions_gcc = [] for toolchain_version in toolchain_versions: if toolchain_version[0].isdigit(): # GCC toolchains begin with a number toolchain_versions_gcc.append(toolchain_version) if toolchain_versions: info('Found the following toolchain versions: {}'.format( toolchain_versions)) info('Picking the latest gcc toolchain, here {}'.format( toolchain_versions_gcc[-1])) toolchain_version = toolchain_versions_gcc[-1] else: warning('Could not find any toolchain for {}!'.format( toolchain_prefix)) ok = False self.toolchain_prefix = toolchain_prefix self.toolchain_version = toolchain_version # Modify the path so that sh finds modules appropriately environ['PATH'] = ( '{ndk_dir}/toolchains/{toolchain_prefix}-{toolchain_version}/' 'prebuilt/{py_platform}-x86/bin/:{ndk_dir}/toolchains/' '{toolchain_prefix}-{toolchain_version}/prebuilt/' '{py_platform}-x86_64/bin/:{ndk_dir}:{sdk_dir}/' 'tools:{path}').format( sdk_dir=self.sdk_dir, ndk_dir=self.ndk_dir, toolchain_prefix=toolchain_prefix, toolchain_version=toolchain_version, py_platform=py_platform, path=environ.get('PATH')) for executable in ("pkg-config", "autoconf", "automake", "libtoolize", "tar", "bzip2", "unzip", "make", "gcc", "g++"): if not sh.which(executable): warning("Missing executable: {} is not installed".format( executable)) if not ok: error('{}python-for-android cannot continue; aborting{}'.format( Err_Fore.RED, Err_Fore.RESET)) sys.exit(1) def __init__(self): super(Context, self).__init__() self.include_dirs = [] self._build_env_prepared = False self._sdk_dir = None self._ndk_dir = None self._android_api = None self._ndk_ver = None self.ndk = None self.toolchain_prefix = None self.toolchain_version = None self.local_recipes = None self.copy_libs = False # this list should contain all Archs, it is pruned later self.archs = ( ArchARM(self), ArchARMv7_a(self), Archx86(self), ArchAarch_64(self), ) self.root_dir = realpath(dirname(__file__)) # remove the most obvious flags that can break the compilation self.env.pop("LDFLAGS", None) self.env.pop("ARCHFLAGS", None) self.env.pop("CFLAGS", None) self.python_recipe = None # Set by TargetPythonRecipe def set_archs(self, arch_names): all_archs = self.archs new_archs = set() for name in arch_names: matching = [arch for arch in all_archs if arch.arch == name] for match in matching: new_archs.add(match) self.archs = list(new_archs) if not self.archs: warning('Asked to compile for no Archs, so failing.') exit(1) info('Will compile for the following archs: {}'.format( ', '.join([arch.arch for arch in self.archs]))) def prepare_bootstrap(self, bs): bs.ctx = self self.bootstrap = bs self.bootstrap.prepare_build_dir() self.bootstrap_build_dir = self.bootstrap.build_dir def prepare_dist(self, name): self.dist_name = name self.bootstrap.prepare_dist_dir(self.dist_name) def get_site_packages_dir(self, arch=None): '''Returns the location of site-packages in the python-install build dir. ''' # AND: This *must* be replaced with something more general in # order to support multiple python versions and/or multiple # archs. if self.python_recipe.from_crystax: return self.get_python_install_dir() return join(self.get_python_install_dir(), 'lib', 'python2.7', 'site-packages') def get_libs_dir(self, arch): '''The libs dir for a given arch.''' ensure_dir(join(self.libs_dir, arch)) return join(self.libs_dir, arch) def has_lib(self, arch, lib): return exists(join(self.get_libs_dir(arch), lib)) def has_package(self, name, arch=None): try: recipe = Recipe.get_recipe(name, self) except IOError: pass else: name = getattr(recipe, 'site_packages_name', None) or name name = name.replace('.', '/') site_packages_dir = self.get_site_packages_dir(arch) return (exists(join(site_packages_dir, name)) or exists(join(site_packages_dir, name + '.py')) or exists(join(site_packages_dir, name + '.pyc')) or exists(join(site_packages_dir, name + '.pyo')) or exists(join(site_packages_dir, name + '.so')) or glob.glob(join(site_packages_dir, name + '-*.egg'))) def not_has_package(self, name, arch=None): return not self.has_package(name, arch)
def build(): from sys import maxsize from os import environ environ = environ.copy() buildout_file = 'buildout-build.cfg' if system() == 'Linux': from platform import dist, linux_distribution _, version, distid = linux_distribution() dist_name = dist()[0].lower() if dist_name == 'ubuntu': if version == '16.04': buildout_file = 'buildout-build-ubuntu-16.04.cfg' else: buildout_file = 'buildout-build-ubuntu.cfg' if dist_name in ['redhat', 'centos']: if maxsize > 2**32: arch = execute_assert_success(["uname", "-i"]).get_stdout().lower() if 'ppc64le' in arch: buildout_file = 'buildout-build-redhat-ppc64le.cfg' elif 'ppc64' in arch: buildout_file = 'buildout-build-redhat-ppc64.cfg' else: if version.startswith('4.'): buildout_file = 'buildout-build-redhat-4-64bit.cfg' else: buildout_file = 'buildout-build-redhat-64bit.cfg' else: if version.startswith('4.'): buildout_file = 'buildout-build-redhat-4-32bit.cfg' else: buildout_file = 'buildout-build-redhat-32bit.cfg' if dist_name in ['suse']: if version in ['10']: buildout_file = 'buildout-build-suse-10.cfg' else: arch = execute_assert_success(["uname", "-i"]).get_stdout().lower() if 'ppc64le' in arch: buildout_file = 'buildout-build-suse-ppc64le.cfg' elif 'ppc64' in arch: buildout_file = 'buildout-build-suse-ppc64.cfg' elif system() == 'Darwin': from platform import mac_ver environ["MACOSX_DEPLOYMENT_TARGET"] = '.'.join(mac_ver()[0].split('.', 2)[:2]) gcc_version = execute_assert_success(["gcc", "--version"]).get_stdout() if 'version 5.' in gcc_version: buildout_file = 'buildout-build-osx-xcode-5.cfg' elif 'version 6.' in gcc_version: buildout_file = 'buildout-build-osx-xcode-6.cfg' elif 'version 7.' in gcc_version: buildout_file = 'buildout-build-osx-xcode-7.cfg' elif 'version 8.' in gcc_version: buildout_file = 'buildout-build-osx-xcode-8.cfg' elif 'version 9.' in gcc_version: buildout_file = 'buildout-build-osx-xcode-8.cfg' else: buildout_file = 'buildout-build-osx.cfg' elif system() == 'Windows': if maxsize > 2**32: buildout_file = 'buildout-build-windows-64bit.cfg' else: buildout_file = 'buildout-build-windows.cfg' elif system() == "SunOS": if 'sparc' in execute_assert_success(["isainfo"]).get_stdout().lower(): buildout_file = 'buildout-build-solaris-sparc.cfg' elif '64' in execute_assert_success(["isainfo", "-b"]).get_stdout(): buildout_file = 'buildout-build-solaris-64bit.cfg' else: pass # TODO support 32 bit elif system() == "AIX": from os import uname aix_version = "{0[3]}.{0[2]}".format(uname()) if aix_version == "7.1": buildout_file = 'buildout-build-aix.cfg' elif aix_version == "7.2": buildout_file = 'buildout-build-aix-7.2.cfg' execte_buildout(buildout_file, environ)
def get_hostrecipe_env(self, arch): env = environ.copy() env['PYTHONPATH'] = join(dirname(self.real_hostpython_location), 'Lib', 'site-packages') return env
def _script_io() -> None: from sys import argv from os import environ from urllib.request import build_opener main(argv[:], environ.copy(), logging.basicConfig, build_opener())
# get first (and only) profile in json file device_id = next(iter(image_info["profiles"].keys())) if device_id not in output["profiles"]: output["profiles"].update(image_info["profiles"]) else: output["profiles"][device_id]["images"].append( image_info["profiles"][device_id]["images"][0]) default_packages, output["arch_packages"] = run( [ "make", "--no-print-directory", "-C", f"target/linux/{output['target'].split('/')[0]}", "val.DEFAULT_PACKAGES", "val.ARCH_PACKAGES", ], capture_output=True, check=True, env=environ.copy().update({"TOPDIR": Path().cwd()}), text=True, ).stdout.splitlines() output["default_packages"] = default_packages.split() if output: output_path.write_text( json.dumps(output, sort_keys=True, separators=(",", ":"))) else: print("JSON info file script could not find any JSON files for target")
class Context(object): env = environ.copy() root_dir = None cache_dir = None build_dir = None dist_dir = None install_dir = None ccache = None cython = None sdkver = None sdksimver = None def __init__(self): super(Context, self).__init__() self.include_dirs = [] ok = True sdks = sh.xcodebuild("-showsdks").splitlines() # get the latest iphoneos iphoneos = [x for x in sdks if "iphoneos" in x] if not iphoneos: print("No iphone SDK installed") ok = False else: iphoneos = iphoneos[0].split()[-1].replace("iphoneos", "") self.sdkver = iphoneos # get the latest iphonesimulator version iphonesim = [x for x in sdks if "iphonesimulator" in x] if not iphonesim: ok = False print("Error: No iphonesimulator SDK installed") else: iphonesim = iphonesim[0].split()[-1].replace("iphonesimulator", "") self.sdksimver = iphonesim # get the path for Developer self.devroot = "{}/Platforms/iPhoneOS.platform/Developer".format( sh.xcode_select("-print-path").strip()) # path to the iOS SDK self.iossdkroot = "{}/SDKs/iPhoneOS{}.sdk".format( self.devroot, self.sdkver) # root of the toolchain self.root_dir = realpath(dirname(__file__)) self.build_dir = "{}/build".format(self.root_dir) self.cache_dir = "{}/.cache".format(self.root_dir) self.dist_dir = "{}/dist".format(self.root_dir) self.install_dir = "{}/dist/root".format(self.root_dir) self.include_dir = "{}/dist/include".format(self.root_dir) self.archs = (ArchSimulator(self), Arch64Simulator(self), ArchIOS(self), Arch64IOS(self)) # path to some tools self.ccache = sh.which("ccache") if not self.ccache: #print("ccache is missing, the build will not be optimized in the future.") pass for cython_fn in ("cython-2.7", "cython"): cython = sh.which(cython_fn) if cython: self.cython = cython break if not self.cython: ok = False print("Missing requirement: cython is not installed") # check the basic tools for tool in ("pkg-config", "autoconf", "automake", "libtool"): if not sh.which(tool): print("Missing requirement: {} is not installed".format(tool)) if not ok: sys.exit(1) ensure_dir(self.root_dir) ensure_dir(self.build_dir) ensure_dir(self.cache_dir) ensure_dir(self.dist_dir) ensure_dir(self.install_dir) ensure_dir(self.include_dir) ensure_dir(join(self.include_dir, "common")) # remove the most obvious flags that can break the compilation self.env.pop("MACOSX_DEPLOYMENT_TARGET", None) self.env.pop("PYTHONDONTWRITEBYTECODE", None) self.env.pop("ARCHFLAGS", None) self.env.pop("CFLAGS", None) self.env.pop("LDFLAGS", None) # set the state self.state = JsonStore(join(self.dist_dir, "state.db"))
def build_arch(self, arch): if self.ctx.ndk_api < self.MIN_NDK_API: error( 'Target ndk-api is {}, but the python3 recipe supports only {}+' .format(self.ctx.ndk_api, self.MIN_NDK_API)) exit(1) recipe_build_dir = self.get_build_dir(arch.arch) # Create a subdirectory to actually perform the build build_dir = join(recipe_build_dir, 'android-build') ensure_dir(build_dir) # TODO: Get these dynamically, like bpo-30386 does sys_prefix = '/usr/local' sys_exec_prefix = '/usr/local' # Skipping "Ensure that nl_langinfo is broken" from the original bpo-30386 platform_name = 'android-{}'.format(self.ctx.ndk_api) with current_directory(build_dir): env = environ.copy() # TODO: Get this information from p4a's arch system android_host = 'arm-linux-androideabi' android_build = sh.Command( join(recipe_build_dir, 'config.guess'))().stdout.strip().decode('utf-8') platform_dir = join(self.ctx.ndk_dir, 'platforms', platform_name, 'arch-arm') toolchain = '{android_host}-4.9'.format(android_host=android_host) toolchain = join(self.ctx.ndk_dir, 'toolchains', toolchain, 'prebuilt', 'linux-x86_64') CC = '{clang} -target {target} -gcc-toolchain {toolchain}'.format( clang=join(self.ctx.ndk_dir, 'toolchains', 'llvm', 'prebuilt', 'linux-x86_64', 'bin', 'clang'), target='armv7-none-linux-androideabi', toolchain=toolchain) AR = join(toolchain, 'bin', android_host) + '-ar' LD = join(toolchain, 'bin', android_host) + '-ld' RANLIB = join(toolchain, 'bin', android_host) + '-ranlib' READELF = join(toolchain, 'bin', android_host) + '-readelf' STRIP = join( toolchain, 'bin', android_host) + '-strip --strip-debug --strip-unneeded' env['CC'] = CC env['AR'] = AR env['LD'] = LD env['RANLIB'] = RANLIB env['READELF'] = READELF env['STRIP'] = STRIP env['PATH'] = '{hostpython_dir}:{old_path}'.format( hostpython_dir=self.get_recipe('hostpython3', self.ctx).get_path_to_python(), old_path=env['PATH']) ndk_flags = ( '--sysroot={ndk_sysroot} -D__ANDROID_API__={android_api} ' '-isystem {ndk_android_host}').format( ndk_sysroot=join(self.ctx.ndk_dir, 'sysroot'), android_api=self.ctx.ndk_api, ndk_android_host=join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include', android_host)) sysroot = join(self.ctx.ndk_dir, 'platforms', platform_name, 'arch-arm') env['CFLAGS'] = env.get('CFLAGS', '') + ' ' + ndk_flags env['CPPFLAGS'] = env.get('CPPFLAGS', '') + ' ' + ndk_flags env['LDFLAGS'] = env.get('LDFLAGS', '') + ' --sysroot={} -L{}'.format( sysroot, join(sysroot, 'usr', 'lib')) # Manually add the libs directory, and copy some object # files to the current directory otherwise they aren't # picked up. This seems necessary because the --sysroot # setting in LDFLAGS is overridden by the other flags. # TODO: Work out why this doesn't happen in the original # bpo-30386 Makefile system. logger.warning('Doing some hacky stuff to link properly') lib_dir = join(sysroot, 'usr', 'lib') env['LDFLAGS'] += ' -L{}'.format(lib_dir) shprint(sh.cp, join(lib_dir, 'crtbegin_so.o'), './') shprint(sh.cp, join(lib_dir, 'crtend_so.o'), './') env['SYSROOT'] = sysroot if not exists('config.status'): shprint(sh.Command(join(recipe_build_dir, 'configure')), *(' '.join( ('--host={android_host}', '--build={android_build}', '--enable-shared', '--disable-ipv6', 'ac_cv_file__dev_ptmx=yes', 'ac_cv_file__dev_ptc=no', '--without-ensurepip', 'ac_cv_little_endian_double=yes', '--prefix={prefix}', '--exec-prefix={exec_prefix}')).format( android_host=android_host, android_build=android_build, prefix=sys_prefix, exec_prefix=sys_exec_prefix)).split(' '), _env=env) if not exists('python'): shprint(sh.make, 'all', _env=env) # TODO: Look into passing the path to pyconfig.h in a # better way, although this is probably acceptable sh.cp('pyconfig.h', join(recipe_build_dir, 'Include'))
if IS_WIN: natives_dir = '..\\native\\build\\' else: natives_dir = '../native/build/' if IS_WIN: prefix = dirname(executable) else: prefix = dirname(dirname(executable)) print("$PREFIX = %s" % prefix) if IS_WIN: # inherenting from the environment allows native code to see MKL common_env_vars = environ.copy() common_env_vars.update({'REPS': '12'}) params_1d = {'N': '1200000'} params_2d = {'N1': '1200', 'N2': '1200'} params_3d = {'N1': '113', 'N2': '114', 'N3': '115'} else: common_env_vars = update_envs( environ, { 'LD_LIBRARY_PATH': prefix + '/lib' + ':' + environ.get('LD_LIBRARY_PATH', default=''), 'REPS': '16' }) params_1d = {'N': '5000000'} params_2d = {'N1': '2500', 'N2': '2500'} params_3d = {'N1': '113', 'N2': '214', 'N3': '315'}
def run(self): def onExperimentSucceeded(_): msg("experiment suceeded") reactor.stop() def onExperimentFailed(failure): err("Experiment execution failed, exiting with error.") err(failure) if reactor.running: reactor.stop() reactor.addSystemEventTrigger('after', 'shutdown', sys.exit, 1) chdir(self._workspace_dir) # Step 1: # Inject all the config options as env variables to give sub-processes easy acces to them. self.local_env = environ.copy() self.local_env.update(configToEnv(self._cfg)) self.local_env['LOCAL_RUN'] = 'True' # Step 2: # Clear output dir before starting. output_dir = path.join(self._workspace_dir, 'output') if path.exists(output_dir): rmtree(output_dir) # Step 3: # Sync the working dir with the head nodes d = Deferred() d.addCallback(lambda _: self.copyWorkspaceToHeadNodes()) # Step 4: # Run the set up script, both locally and in the head nodes d.addCallback(lambda _: self.runSetupScripts()) # Step 5: # Start the tracker, either locally or on the first head node of the list. d.addCallback(lambda _: self.startTracker()) # Step 6: # Start the config server, always locally if running instances locally as the head nodes are firewalled and # can only be reached from the outside trough SSH. d.addCallback(lambda _: self.startExperimentServer()) # Step 7: # Spawn both local and remote instance runner scripts, which will connect to the config server and wait for all # of them to be ready before starting the experiment. d.addCallback(lambda _: self.startInstances()) # Step 8: # Collect all the data from the remote head nodes. d.addCallback(lambda _: self.collectOutputFromHeadNodes()) # Step 9: # Extract the data and graph stuff d.addCallback(lambda _: self.runPostProcess()) # TODO: From here onwards reactor.callLater(0, d.callback, None) # reactor.callLater(60, reactor.stop) return d.addCallbacks(onExperimentSucceeded, onExperimentFailed)
if tensor.has_field("velocity"): self.velocities.append(tensor['velocity']) self.__updateJSON() self.frames_processed += 1 def __updateJSON(self): with open(self._result_path, "w") as write_file: json.dump(self.velocities, write_file, indent=4, sort_keys=True) def __dump_data(self): with open(self._result_path, "a") as write_file: write_file.write("{} \n".format(self.velocities)) if __name__ == "__main__": svclassifier = pickle.load(open(SVM_PATH, 'rb')) for file_name in listdir(DATASET_PATH): if file_name.endswith(".mp4"): video_path = join(DATASET_PATH, file_name) pipeline_str = CLASSIFY_PIPELINE_TEMPLATE.format( video_path, MODEL_PATH, ALPHA, ALPHA_HW, realpath(__file__), join(DATASET_PATH, file_name.replace('.mp4', '.json'))) print(pipeline_str) proc = subprocess.run(shlex.split(pipeline_str), env=environ.copy()) if proc.returncode != 0: print("Error while running pipeline") exit(-1)
for v in ("JSTESTS_EXTRA_ARGS", "JITTEST_EXTRA_ARGS"): env[v] = "--args='--dll %s' %s" % (injector_lib, env.get(v, "")) # Always run all enabled tests, even if earlier ones failed. But return the # first failed status. results = [("(make-nonempty)", 0)] if "checks" in test_suites: results.append(("make check", run_test_command([MAKE, "check"]))) if "jittest" in test_suites: results.append( ("make check-jit-test", run_test_command([MAKE, "check-jit-test"]))) if "jsapitests" in test_suites: jsapi_test_binary = os.path.join(OBJDIR, "dist", "bin", "jsapi-tests") test_env = env.copy() test_env["TOPSRCDIR"] = DIR.source if use_minidump and platform.system() == "Linux": test_env["LD_PRELOAD"] = injector_lib st = run_test_command([jsapi_test_binary], env=test_env) if st < 0: print("PROCESS-CRASH | jsapi-tests | application crashed") print("Return code: {}".format(st)) results.append(("jsapi-tests", st)) if "jstests" in test_suites: results.append(("jstests", run_test_command([MAKE, "check-jstests"]))) if "gdb" in test_suites: test_script = os.path.join(DIR.js_src, "gdb", "run-tests.py") auto_args = ["-s", "-o", "--no-progress"] if AUTOMATION else [] extra_args = env.get("GDBTEST_EXTRA_ARGS", "").split(" ") results.append((
def run(self): """ Launch GameThread instance This function start a new processus with Popen and wait until it stop. When it finish, GameThread emit a signal to main interface. """ # Call game-started signal on main window self.parent.emit("game-started", self.game) started = datetime.now() self.logger.info("Launch %s" % self.game.name) try: command = self.game.command(self.fullscreen) self.logger.debug("Command: %s" % ' '.join(command)) # ------------------------------------ # Check environment # ------------------------------------ # Get a copy of current environment environment = environ.copy() # Check if current game has specific environment variable for key, value in self.game.environment.items(): environment[key] = value # ------------------------------------ # Start process # ------------------------------------ self.logger.info("Log to %s" % self.path) # Logging process output with open(self.path, 'w') as pipe: self.proc = Popen(command, stdin=PIPE, stdout=pipe, stderr=pipe, env=environment, start_new_session=True, universal_newlines=True) self.proc.communicate() self.logger.info("Close %s" % self.game.name) self.proc.terminate() # ------------------------------------ # Play time # ------------------------------------ self.delta = (datetime.now() - started) except OSError as error: self.logger.error("Cannot access to game: %s" % str(error)) self.error = True except MemoryError as error: self.logger.error("A memory error occur: %s" % str(error)) self.error = True except KeyboardInterrupt: self.logger.info("Terminate by keyboard interrupt") except Exception as error: self.logger.info("An exception error occur: %s" % str(error)) self.error = True # Call game-terminate signal on main window self.parent.emit("game-terminate", self)
def getEnvironment(self): return environ.copy()
def _scheduleStep(self, stepName, steps, simulate, scriptArgs, skip, startAfter): """ Schedule a single execution step. @param step: A C{dict} with a job specification. @param simulate: If C{True}, this step should be simulated. The step script is still run, but with SP_SIMULATE=1 in its environment. Else, SP_SIMULATE=0 will be in the environment. @param scriptArgs: A C{list} of C{str} arguments that should be put on the command line of all steps that have no dependencies. @param skip: If C{True}, the step should be skipped, which will be indicated to the script by SP_SKIP=1 in its environment. SP_SKIP will be 0 in non-skipped steps. It is up to the script, which is run in either case, to decide how to behave. @param startAfter: A C{list} of C{int} job ids that must complete (either successully or unsuccessully, it doesn't matter) before steps in the current specification may start. If C{None}, steps in the current specification may start immediately. """ step = steps[stepName] step['tasks'] = defaultdict(set) step['simulate'] = simulate step['skip'] = skip scriptArgsStr = ' '.join(map(str, scriptArgs)) if scriptArgs else '' if scriptArgs: # Single quote each script arg so shell metacharacters and # whitespace are preserved. args = [] for arg in map(str, scriptArgs): if "'" in arg: raise SchedulingError( 'Script argument "%s" contains a single quote, which ' 'is currently not supported.' % arg) else: args.append("'%s'" % arg) scriptArgsStr = ' '.join(args) else: scriptArgsStr = '' if step.get('error step', False): separator = '?' after = 'afternotok' else: separator = ',' after = 'afterok' # taskDependencies is keyed by task name. These are the tasks # started by the steps that the current step depends on. Its # values are sets of SLURM job ids the tasks that step started and # which this step therefore depends on. step['taskDependencies'] = taskDependencies = defaultdict(set) for stepName in step.get('dependencies', ()): for taskName, jobIds in steps[stepName]['tasks'].items(): taskDependencies[taskName].update(jobIds) if taskDependencies: if 'collect' in step: # This step is a 'collector'. I.e., it is dependent on all # tasks from all its dependent steps and cannot run until # they have all finished. We will only run the script once, # and tell it about all job ids for all tasks that are # depended on. env = environ.copy() env['SP_ORIGINAL_ARGS'] = scriptArgsStr env['SP_SIMULATE'] = str(int(simulate)) env['SP_SKIP'] = str(int(skip)) dependencies = separator.join( sorted(('%s:%d' % (after, jobId)) for jobIds in taskDependencies.values() for jobId in jobIds)) env['SP_DEPENDENCY_ARG'] = '--dependency=' + dependencies self._runStepScript(step, sorted(taskDependencies), env) else: # The script for this step gets run once for each task in the # steps it depends on. for taskName in sorted(taskDependencies): env = environ.copy() env['SP_ORIGINAL_ARGS'] = scriptArgsStr env['SP_SIMULATE'] = str(int(simulate)) env['SP_SKIP'] = str(int(skip)) jobIds = steps[stepName]['tasks'][taskName] dependencies = separator.join( sorted(('%s:%d' % (after, jobId)) for jobId in jobIds)) if dependencies: env['SP_DEPENDENCY_ARG'] = ('--dependency=' + dependencies) else: env.pop('SP_DEPENDENCY_ARG', None) self._runStepScript(step, [taskName], env) else: # Either this step has no dependencies or the steps it is # dependent on did not start any tasks. env = environ.copy() if startAfter: dependencies = separator.join( sorted(('%s:%d' % (after, jobId)) for jobId in startAfter)) env['SP_DEPENDENCY_ARG'] = '--dependency=' + dependencies else: env.pop('SP_DEPENDENCY_ARG', None) if 'dependencies' in step: # The step has dependencies, but the dependent steps did # not start any tasks. Run the step as though there were no # dependencies. args = [] else: # The step has no dependencies. Run it with the original # command line arguments and put any --startAfter job ids # into the SP_DEPENDENCY_ARG environment variable. args = [] if scriptArgs is None else list(map(str, scriptArgs)) env['SP_ORIGINAL_ARGS'] = scriptArgsStr env['SP_SIMULATE'] = str(int(simulate)) env['SP_SKIP'] = str(int(skip)) self._runStepScript(step, args, env) step['scheduledAt'] = time.time()
def setUp(self): IntegrationTest.setUp(self) self.environment = environ.copy() self.environment['PYTHONPATH'] = 'src'
# folder; it might be created already, in which case scriptest bombs. A simple rmtree # will work in *nix, but Windows will often hang on to files in a folder while it is # being cleaned up, so we need to have a retry mechanism. count = 0 while True: try: rmtree(script_test_path) break except OSError: count += 1 if count == 10: raise time.sleep(0.1) apps_path = path.join(here, 'apps') base_environ = environ.copy() base_environ['PYTHONPATH'] = apps_path if not is_win: class BWTestFileEnvironment(TestFileEnvironment): def clear(self, force=True): super(BWTestFileEnvironment, self).clear(force=force) else: # Windows throws scripttest some curveballs when it comes to cleaning up folders. Use force # on clear to prevent scripttest checking to see if it created the directory (since we know # what path we are working with, beware if you put things there that don't go there). Then, # we would still get WindowsError exceptions, so have a retry with a short delay to resolve. class BWTestFileEnvironment(TestFileEnvironment): def clear(self): count = 0
def build(): from sys import maxsize from os import environ from platform import version environ = environ.copy() buildout_file = 'buildout-build.cfg' if system() == 'Linux': from distro import linux_distribution dist_name, version, distid = linux_distribution( full_distribution_name=False) dist_name = dist_name.replace('rhel', 'redhat').replace( 'sles', 'suse').replace('enterpriseenterpriseserver', 'oracle') if dist_name == 'ubuntu': if version >= '16.04': buildout_file = 'buildout-build-ubuntu-16.04.cfg' else: buildout_file = 'buildout-build-ubuntu.cfg' if dist_name in ['redhat', 'centos', 'oracle', 'suse']: arch = execute_assert_success(["uname", "-i"]).get_stdout().lower() if 'ppc64le' in arch: buildout_file = 'buildout-build-redhat-ppc64le.cfg' elif 'ppc64' in arch: buildout_file = 'buildout-build-redhat-ppc64.cfg' elif 'i386' in arch: buildout_file = 'buildout-build-redhat-32bit.cfg' else: if version.startswith('8') or version.startswith('15'): buildout_file = 'buildout-build-redhat-8-64bit.cfg' else: buildout_file = 'buildout-build-redhat-64bit.cfg' elif system() == 'Darwin': from platform import mac_ver environ["MACOSX_DEPLOYMENT_TARGET"] = '.'.join(mac_ver()[0].split( '.', 2)[:2]) gcc_version = execute_assert_success(["gcc", "--version" ]).get_stdout().decode() if 'version 5.' in gcc_version: buildout_file = 'buildout-build-osx-xcode-5.cfg' elif 'version 6.' in gcc_version: buildout_file = 'buildout-build-osx-xcode-6.cfg' elif 'version 7.' in gcc_version: buildout_file = 'buildout-build-osx-xcode-7.cfg' elif 'version 8.' in gcc_version: buildout_file = 'buildout-build-osx-xcode-8.cfg' elif 'version 9.' in gcc_version: buildout_file = 'buildout-build-osx-xcode-8.cfg' elif 'version 10.' in gcc_version: buildout_file = 'buildout-build-osx-xcode-8.cfg' elif 'version 11.' in gcc_version: buildout_file = 'buildout-build-osx-xcode-8.cfg' else: buildout_file = 'buildout-build-osx.cfg' elif system() == 'Windows': if maxsize > 2**32: buildout_file = 'buildout-build-windows-64bit.cfg' else: buildout_file = 'buildout-build-windows.cfg' elif system() == "SunOS": if 'sparc' in execute_assert_success(["isainfo"]).get_stdout().lower(): buildout_file = 'buildout-build-solaris-sparc.cfg' if '11.4' in version(): buildout_file = 'buildout-build-solaris-11.4-sparc.cfg' elif '64' in execute_assert_success(["isainfo", "-b"]).get_stdout(): buildout_file = 'buildout-build-solaris-64bit.cfg' if '11.4' in version(): buildout_file = 'buildout-build-solaris-11.4-64bit.cfg' else: pass # TODO support 32 bit elif system() == "AIX": buildout_file = 'buildout-build-aix.cfg' execte_buildout(buildout_file, environ)
# Copyright (c) JeremÃas Casteglione <*****@*****.**> # See LICENSE file. from os import environ from _sadm.utils.cmd import call, callCheck from .check import check __all__ = ['deploy'] _cmdenv = environ.copy() _cmdenv['DEBIAN_FRONTEND'] = 'noninteractive' def deploy(env): if env.settings.getboolean('os.pkg', 'update', fallback = False): env.log('update') _update() for diff in check(env, action = 'remove'): opt, pkg = diff env.log("%s %s" % (opt, pkg)) _remove(pkg) for diff in check(env, action = 'install'): opt, pkg = diff env.log("%s %s" % (opt, pkg)) _install(pkg) for diff in check(env, action = 'prune'): opt, pkg = diff env.log("%s %s" % (opt, pkg)) _prune(pkg)
def __call__(self, *args, **kwargs): self.args = args self.env = environ.copy() return MagicMock()
from setuptools import setup, find_packages from setuptools.extension import Extension from scalene.scalene_version import scalene_version from os import path, environ import platform import sys if sys.platform == 'darwin': import sysconfig mdt = 'MACOSX_DEPLOYMENT_TARGET' target = environ[mdt] if mdt in environ else sysconfig.get_config_var(mdt) # target >= 10.9 is required for gcc/clang to find libstdc++ headers if [int(n) for n in target.split('.')] < [10, 9]: from os import execve newenv = environ.copy() newenv[mdt] = '10.9' execve(sys.executable, [sys.executable] + sys.argv, newenv) def clang_version(): import re pat = re.compile('Clang ([0-9]+)') match = pat.search(platform.python_compiler()) version = int(match.group(1)) return version def multiarch_args(): """Returns args requesting multi-architecture support, if applicable.""" # On MacOS we build "universal2" packages, for both x86_64 and arm64/M1 if sys.platform == 'darwin':
def tor_network(reactor, temp_dir, chutney, request): # this is the actual "chutney" script at the root of a chutney checkout chutney_dir = chutney chut = join(chutney_dir, 'chutney') # now, as per Chutney's README, we have to create the network # ./chutney configure networks/basic # ./chutney start networks/basic env = environ.copy() env.update({"PYTHONPATH": join(chutney_dir, "lib")}) proto = _DumpOutputProtocol(None) reactor.spawnProcess( proto, sys.executable, ( sys.executable, '-m', 'chutney.TorNet', 'configure', join(chutney_dir, 'networks', 'basic'), ), path=join(chutney_dir), env=env, ) pytest_twisted.blockon(proto.done) proto = _DumpOutputProtocol(None) reactor.spawnProcess( proto, sys.executable, ( sys.executable, '-m', 'chutney.TorNet', 'start', join(chutney_dir, 'networks', 'basic'), ), path=join(chutney_dir), env=env, ) pytest_twisted.blockon(proto.done) # print some useful stuff proto = _CollectOutputProtocol() reactor.spawnProcess( proto, sys.executable, ( sys.executable, '-m', 'chutney.TorNet', 'status', join(chutney_dir, 'networks', 'basic'), ), path=join(chutney_dir), env=env, ) try: pytest_twisted.blockon(proto.done) except ProcessTerminated: print("Chutney.TorNet status failed (continuing):") print(proto.output.getvalue()) def cleanup(): print("Tearing down Chutney Tor network") proto = _CollectOutputProtocol() reactor.spawnProcess( proto, sys.executable, ( sys.executable, '-m', 'chutney.TorNet', 'stop', join(chutney_dir, 'networks', 'basic'), ), path=join(chutney_dir), env=env, ) pytest_twisted.blockon(proto.done) request.addfinalizer(cleanup) return chut
vel_msg.angular.z = -MAX_ANGULAR_RATE vel_msg.linear.x = 0.0 print('Turn in place') # Go straight forward in barrel in midle of view else: vel_msg.angular.z = 0.0 vel_msg.linear.x = MAX_LINEAR_RATE print('Go forward') control_publisher.publish(vel_msg) rate.sleep() if __name__ == '__main__': fastai_env = environ.copy() fastai_env['PATH'] = '/home/dipto/anaconda3/bin:' + fastai_env['PATH'] fastai_env['PYTHONPATH'] = '' rospy.init_node('Driver', anonymous=True) rospy.Subscriber('/lab06_radeeb/camera1/image_raw', Image, camera_callback, queue_size=1) image_converter = CvBridge() run( 'source activate fastai-cpu && /home/dipto/ros_workspaces/csc790_labs/src/lab06_radeeb/lab06_radeeb_vision/Scripts/fastai_helper.py', fastai_env)
def make_test_git(): directory = mkdtemp() path = get_readthedocs_app_path() sample = abspath(pjoin(path, 'rtd_tests/fixtures/sample_repo')) directory = pjoin(directory, 'sample_repo') copytree(sample, directory) env = environ.copy() env['GIT_DIR'] = pjoin(directory, '.git') chdir(directory) # Initialize and configure check_output(['git', 'init'] + [directory], env=env) check_output(['git', 'config', 'user.email', '*****@*****.**'], env=env) check_output(['git', 'config', 'user.name', 'Read the Docs'], env=env) # Set up the actual repository check_output(['git', 'add', '.'], env=env) check_output(['git', 'commit', '-m"init"'], env=env) # Add fake repo as submodule. We need to fake this here because local path # URL are not allowed and using a real URL will require Internet to clone # the repo check_output(['git', 'checkout', '-b', 'submodule', 'master'], env=env) # https://stackoverflow.com/a/37378302/2187091 mkdir(pjoin(directory, 'foobar')) gitmodules_path = pjoin(directory, '.gitmodules') with open(gitmodules_path, 'w') as fh: fh.write( '''[submodule "foobar"]\n\tpath = foobar\n\turl = https://foobar.com/git\n''' ) check_output( [ 'git', 'update-index', '--add', '--cacheinfo', '160000', '233febf4846d7a0aeb95b6c28962e06e21d13688', 'foobar', ], env=env, ) check_output(['git', 'add', '.'], env=env) check_output(['git', 'commit', '-m"Add submodule"'], env=env) # Add a relative submodule URL in the relativesubmodule branch check_output(['git', 'checkout', '-b', 'relativesubmodule', 'master'], env=env) check_output( ['git', 'submodule', 'add', '-b', 'master', './', 'relativesubmodule'], env=env) check_output(['git', 'add', '.'], env=env) check_output(['git', 'commit', '-m"Add relative submodule"'], env=env) # Add an invalid submodule URL in the invalidsubmodule branch check_output(['git', 'checkout', '-b', 'invalidsubmodule', 'master'], env=env) check_output( ['git', 'submodule', 'add', '-b', 'master', './', 'invalidsubmodule'], env=env, ) check_output(['git', 'add', '.'], env=env) check_output(['git', 'commit', '-m"Add invalid submodule"'], env=env) # Checkout to master branch again check_output(['git', 'checkout', 'master'], env=env) return directory
env.Depends(web_server_static_files, header_file) env.Depends(web_server_static, env.Command(web_server_static_files, source, make_static)) # # Generate Web app resources # #print("PYTHON ENV:{}\n".format(env.Dump())) if npm_installed: headers_src = join(env.subst("$PROJECTSRC_DIR"), "web_static") gui_dir = join(env.subst("$PROJECTSRC_DIR"), "node") dist_dir = join(env.subst("$PROJECT_DIR"), "data") node_modules = join(env.subst("$PROJECT_DIR"), "node_modules") my_env = environ.copy() my_flags = env.ParseFlags(env['BUILD_FLAGS']) for item in my_flags.get("CPPDEFINES"): if isinstance(item,list): # print(item[0],item[1]) my_env["DEFINE_"+item[0]] = item[1] elif isinstance(item,str): # print(item,'True') my_env["DEFINE_"+item] = 'True' else: print(item) my_env["PROGNAME"] = env["PROGNAME"] if "UPLOAD_PORT" in env : my_env["UPLOAD_PORT"] = env["UPLOAD_PORT"]
class Context: """A build context. If anything will be built, an instance this class will be instantiated and used to hold all the build state.""" # Whether to make a debug or release build build_as_debuggable = False # Whether to strip debug symbols in `.so` files with_debug_symbols = False env = environ.copy() # the filepath of toolchain.py root_dir = None # the root dir where builds and dists will be stored storage_dir = None # in which bootstraps are copied for building # and recipes are built build_dir = None distribution = None """The Distribution object representing the current build target location.""" # the Android project folder where everything ends up dist_dir = None # where Android libs are cached after build # but before being placed in dists libs_dir = None aars_dir = None # Whether setup.py or similar should be used if present: use_setup_py = False ccache = None # whether to use ccache ndk_platform = None # the ndk platform directory bootstrap = None bootstrap_build_dir = None recipe_build_order = None # Will hold the list of all built recipes symlink_bootstrap_files = ( False # If True, will symlink instead of copying during build ) java_build_tool = "auto" @property def packages_path(self): """Where packages are downloaded before being unpacked""" return join(self.storage_dir, "packages") @property def templates_dir(self): return join(self.root_dir, "templates") @property def libs_dir(self): # Was previously hardcoded as self.build_dir/libs directory = join(self.build_dir, "libs_collections", self.bootstrap.distribution.name) ensure_dir(directory) return directory @property def javaclass_dir(self): # Was previously hardcoded as self.build_dir/java directory = join(self.build_dir, "javaclasses", self.bootstrap.distribution.name) ensure_dir(directory) return directory @property def aars_dir(self): directory = join(self.build_dir, "aars", self.bootstrap.distribution.name) ensure_dir(directory) return directory @property def python_installs_dir(self): directory = join(self.build_dir, "python-installs") ensure_dir(directory) return directory def get_python_install_dir(self): return join(self.python_installs_dir, self.bootstrap.distribution.name) def setup_dirs(self, storage_dir): """Calculates all the storage and build dirs, and makes sure the directories exist where necessary.""" self.storage_dir = expanduser(storage_dir) if " " in self.storage_dir: raise ValueError("storage dir path cannot contain spaces, please " "specify a path with --storage-dir") self.build_dir = join(self.storage_dir, "build") self.dist_dir = join(self.storage_dir, "dists") def ensure_dirs(self): ensure_dir(self.storage_dir) ensure_dir(self.build_dir) ensure_dir(self.dist_dir) ensure_dir(join(self.build_dir, "bootstrap_builds")) ensure_dir(join(self.build_dir, "other_builds")) @property def android_api(self): """The Android API being targeted.""" if self._android_api is None: raise ValueError("Tried to access android_api but it has not " "been set - this should not happen, something " "went wrong!") return self._android_api @android_api.setter def android_api(self, value): self._android_api = value @property def ndk_api(self): """The API number compile against""" if self._ndk_api is None: raise ValueError("Tried to access ndk_api but it has not " "been set - this should not happen, something " "went wrong!") return self._ndk_api @ndk_api.setter def ndk_api(self, value): self._ndk_api = value @property def sdk_dir(self): """The path to the Android SDK.""" if self._sdk_dir is None: raise ValueError("Tried to access sdk_dir but it has not " "been set - this should not happen, something " "went wrong!") return self._sdk_dir @sdk_dir.setter def sdk_dir(self, value): self._sdk_dir = value @property def ndk_dir(self): """The path to the Android NDK.""" if self._ndk_dir is None: raise ValueError("Tried to access ndk_dir but it has not " "been set - this should not happen, something " "went wrong!") return self._ndk_dir @ndk_dir.setter def ndk_dir(self, value): self._ndk_dir = value def prepare_build_environment(self, user_sdk_dir, user_ndk_dir, user_android_api, user_ndk_api): """Checks that build dependencies exist and sets internal variables for the Android SDK etc. ..warning:: This *must* be called before trying any build stuff """ self.ensure_dirs() if self._build_env_prepared: return ok = True # Work out where the Android SDK is sdk_dir = None if user_sdk_dir: sdk_dir = user_sdk_dir # This is the old P4A-specific var if sdk_dir is None: sdk_dir = environ.get("ANDROIDSDK", None) # This seems used more conventionally if sdk_dir is None: sdk_dir = environ.get("ANDROID_HOME", None) # Checks in the buildozer SDK dir, useful for debug tests of p4a if sdk_dir is None: possible_dirs = glob.glob( expanduser( join("~", ".buildozer", "android", "platform", "android-sdk-*"))) possible_dirs = [ d for d in possible_dirs if not d.endswith((".bz2", ".gz")) ] if possible_dirs: info("Found possible SDK dirs in buildozer dir: {}".format( ", ".join([d.split(os.sep)[-1] for d in possible_dirs]))) info("Will attempt to use SDK at {}".format(possible_dirs[0])) warning("This SDK lookup is intended for debug only, if you " "use python-for-android much you should probably " "maintain your own SDK download.") sdk_dir = possible_dirs[0] if sdk_dir is None: raise BuildInterruptingException( "Android SDK dir was not specified, exiting.") self.sdk_dir = realpath(sdk_dir) # Check what Android API we're using android_api = None if user_android_api: android_api = user_android_api info("Getting Android API version from user argument: {}".format( android_api)) elif "ANDROIDAPI" in environ: android_api = environ["ANDROIDAPI"] info("Found Android API target in $ANDROIDAPI: {}".format( android_api)) else: info("Android API target was not set manually, using " "the default of {}".format(RECOMMENDED_TARGET_API)) android_api = RECOMMENDED_TARGET_API android_api = int(android_api) self.android_api = android_api check_target_api(android_api, self.archs[0].arch) apis = get_available_apis(self.sdk_dir) info("Available Android APIs are ({})".format(", ".join(map(str, apis)))) if android_api in apis: info(("Requested API target {} is available, " "continuing.").format(android_api)) else: raise BuildInterruptingException( ("Requested API target {} is not available, install " "it with the SDK android tool.").format(android_api)) # Find the Android NDK # Could also use ANDROID_NDK, but doesn't look like many tools use this ndk_dir = None if user_ndk_dir: ndk_dir = user_ndk_dir info("Getting NDK dir from from user argument") if ndk_dir is None: # The old P4A-specific dir ndk_dir = environ.get("ANDROIDNDK", None) if ndk_dir is not None: info("Found NDK dir in $ANDROIDNDK: {}".format(ndk_dir)) if ndk_dir is None: # Apparently the most common convention ndk_dir = environ.get("NDK_HOME", None) if ndk_dir is not None: info("Found NDK dir in $NDK_HOME: {}".format(ndk_dir)) if ndk_dir is None: # Another convention (with maven?) ndk_dir = environ.get("ANDROID_NDK_HOME", None) if ndk_dir is not None: info("Found NDK dir in $ANDROID_NDK_HOME: {}".format(ndk_dir)) if ndk_dir is None: # Checks in the buildozer NDK dir, useful # # for debug tests of p4a possible_dirs = glob.glob( expanduser( join("~", ".buildozer", "android", "platform", "android-ndk-r*"))) if possible_dirs: info("Found possible NDK dirs in buildozer dir: {}".format( ", ".join([d.split(os.sep)[-1] for d in possible_dirs]))) info("Will attempt to use NDK at {}".format(possible_dirs[0])) warning("This NDK lookup is intended for debug only, if you " "use python-for-android much you should probably " "maintain your own NDK download.") ndk_dir = possible_dirs[0] if ndk_dir is None: raise BuildInterruptingException( "Android NDK dir was not specified") self.ndk_dir = realpath(ndk_dir) check_ndk_version(ndk_dir) ndk_api = None if user_ndk_api: ndk_api = user_ndk_api info( "Getting NDK API version (i.e. minimum supported API) from user argument" ) elif "NDKAPI" in environ: ndk_api = environ.get("NDKAPI", None) info("Found Android API target in $NDKAPI") else: ndk_api = min(self.android_api, RECOMMENDED_NDK_API) warning( "NDK API target was not set manually, using " "the default of {} = min(android-api={}, default ndk-api={})". format(ndk_api, self.android_api, RECOMMENDED_NDK_API)) ndk_api = int(ndk_api) self.ndk_api = ndk_api check_ndk_api(ndk_api, self.android_api) # path to some tools self.ccache = sh.which("ccache") if not self.ccache: info("ccache is missing, the build will not be optimized in the " "future.") try: subprocess.check_output(["python3", "-m", "cython", "--help"]) except subprocess.CalledProcessError: warning("Cython for python3 missing. If you are building for " " a python 3 target (which is the default)" " then THINGS WILL BREAK.") # This would need to be changed if supporting multiarch APKs arch = self.archs[0] toolchain_prefix = arch.toolchain_prefix self.ndk_platform, ndk_platform_dir_exists = get_ndk_platform_dir( self.ndk_dir, self.ndk_api, arch) ok = ok and ndk_platform_dir_exists py_platform = sys.platform if py_platform in ["linux2", "linux3"]: py_platform = "linux" toolchain_versions, toolchain_path_exists = get_toolchain_versions( self.ndk_dir, arch) ok = ok and toolchain_path_exists toolchain_versions.sort() toolchain_versions_gcc = [] for toolchain_version in toolchain_versions: if toolchain_version[0].isdigit(): # GCC toolchains begin with a number toolchain_versions_gcc.append(toolchain_version) if toolchain_versions: info("Found the following toolchain versions: {}".format( toolchain_versions)) info("Picking the latest gcc toolchain, here {}".format( toolchain_versions_gcc[-1])) toolchain_version = toolchain_versions_gcc[-1] else: warning("Could not find any toolchain for {}!".format( toolchain_prefix)) ok = False self.toolchain_prefix = toolchain_prefix self.toolchain_version = toolchain_version # Modify the path so that sh finds modules appropriately environ["PATH"] = ( "{ndk_dir}/toolchains/{toolchain_prefix}-{toolchain_version}/" "prebuilt/{py_platform}-x86/bin/:{ndk_dir}/toolchains/" "{toolchain_prefix}-{toolchain_version}/prebuilt/" "{py_platform}-x86_64/bin/:{ndk_dir}:{sdk_dir}/" "tools:{path}").format( sdk_dir=self.sdk_dir, ndk_dir=self.ndk_dir, toolchain_prefix=toolchain_prefix, toolchain_version=toolchain_version, py_platform=py_platform, path=environ.get("PATH"), ) for executable in ( "pkg-config", "autoconf", "automake", "libtoolize", "tar", "bzip2", "unzip", "make", "gcc", "g++", ): if not sh.which(executable): warning(f"Missing executable: {executable} is not installed") if not ok: raise BuildInterruptingException( "python-for-android cannot continue due to the missing executables above" ) def __init__(self): self.include_dirs = [] self._build_env_prepared = False self._sdk_dir = None self._ndk_dir = None self._android_api = None self._ndk_api = None self.ndk = None self.toolchain_prefix = None self.toolchain_version = None self.local_recipes = None self.copy_libs = False # this list should contain all Archs, it is pruned later self.archs = ( ArchARM(self), ArchARMv7_a(self), Archx86(self), Archx86_64(self), ArchAarch_64(self), ) self.root_dir = realpath(dirname(__file__)) # remove the most obvious flags that can break the compilation self.env.pop("LDFLAGS", None) self.env.pop("ARCHFLAGS", None) self.env.pop("CFLAGS", None) self.python_recipe = None # Set by TargetPythonRecipe def set_archs(self, arch_names): all_archs = self.archs new_archs = set() for name in arch_names: matching = [arch for arch in all_archs if arch.arch == name] for match in matching: new_archs.add(match) self.archs = list(new_archs) if not self.archs: raise BuildInterruptingException( "Asked to compile for no Archs, so failing.") info("Will compile for the following archs: {}".format(", ".join( [arch.arch for arch in self.archs]))) def prepare_bootstrap(self, bootstrap): if not bootstrap: raise TypeError("None is not allowed for bootstrap") bootstrap.ctx = self self.bootstrap = bootstrap self.bootstrap.prepare_build_dir() self.bootstrap_build_dir = self.bootstrap.build_dir def prepare_dist(self): self.bootstrap.prepare_dist_dir() def get_site_packages_dir(self, arch=None): """Returns the location of site-packages in the python-install build dir. """ return self.get_python_install_dir() def get_libs_dir(self, arch): """The libs dir for a given arch.""" ensure_dir(join(self.libs_dir, arch)) return join(self.libs_dir, arch) def has_lib(self, arch, lib): return exists(join(self.get_libs_dir(arch), lib)) def has_package(self, name, arch=None): # If this is a file path, it'll need special handling: if (name.find("/") >= 0 or name.find("\\") >= 0 ) and name.find("://") < 0: # (:// would indicate an url) if not os.path.exists(name): # Non-existing dir, cannot look this up. return False try: name = get_package_name(os.path.abspath(name)) except ValueError: # Failed to look up any meaningful name. return False # Try to look up recipe by name: try: recipe = Recipe.get_recipe(name, self) except ValueError: pass else: name = getattr(recipe, "site_packages_name", None) or name name = name.replace(".", "/") site_packages_dir = self.get_site_packages_dir(arch) return (exists(join(site_packages_dir, name)) or exists(join(site_packages_dir, name + ".py")) or exists(join(site_packages_dir, name + ".pyc")) or exists(join(site_packages_dir, name + ".pyo")) or exists(join(site_packages_dir, name + ".so")) or glob.glob(join(site_packages_dir, name + "-*.egg"))) def not_has_package(self, name, arch=None): return not self.has_package(name, arch)