def main(): my_config = get_config() new_synapse_config = generate_configuration( my_config, get_zookeeper_topology(), get_all_namespaces() ) with tempfile.NamedTemporaryFile() as tmp_file: new_synapse_config_path = tmp_file.name with open(new_synapse_config_path, 'w') as fp: json.dump(new_synapse_config, fp, sort_keys=True, indent=4, separators=(',', ': ')) # Match permissions that puppet expects os.chmod(new_synapse_config_path, 0644) # Restart synapse if the config files differ should_restart = not filecmp.cmp(new_synapse_config_path, my_config['config_file']) # Always swap new config file into place. Our monitoring system # checks the config['config_file'] file age to ensure that it is # continually being updated. shutil.copy(new_synapse_config_path, my_config['config_file']) if should_restart: subprocess.check_call(SYNAPSE_RESTART_COMMAND)
def sh(cmd): msg_trace('sh(%r)' % cmd) try: subprocess.check_call(cmd, shell=True) except: msg_trace('FAILED!') raise
def shell(self, name, *args): self.parser.set_usage(self.envshell_usage) self.parser.prog = 'envshell' options, args = self.parser.parse_args(list(args)) if len(args) == 0: raise Response("%s\nError: incorrect number of arguments" % (self.parser.get_usage()), 2) sys.stdout.write("Launching envshell for %s. " "Type 'exit' or 'Ctrl+D' to return.\n" % self.path(args[0])) sys.stdout.flush() self.open(args[0], 2) shell = os.environ['SHELL'] try: subprocess.check_call([shell], universal_newlines=True, bufsize=0, close_fds=not is_windows, **params) except OSError as err: if err.errno == 2: raise Response("Unable to find shell %s" % shell, err.errno) else: raise Response("An error occurred: %s" % err, status=err.errno) raise Response()
def MakeCoreSdist(self): os.chdir(args.grr_src) subprocess.check_call([self.PYTHON_BIN64, "setup.py", "sdist", "--dist-dir=%s" % self.BUILDDIR, "--no-make-docs", "--no-make-ui-files", "--no-sync-artifacts"]) return glob.glob(os.path.join(self.BUILDDIR, "grr-response-core-*.zip")).pop()
def initEnKFrun(callback=None,basedir=None,site=None,siteparam=None): if not site: raise "site is a required parameter" if siteparam: param = ast.literal_eval(siteparam) else: raise "Parameters are required. Please submit Paramerers with task submission" if not basedir: raise "Basedir is required" #Create working directory newDir = basedir + "celery_data/" + str(initEnKFrun.request.id) check_call(["mkdir",newDir]) os.chdir(newDir) #copy matlab code to working directory codeDir =basedir + 'enfk_matlab/' for file in glob.glob(codeDir + '*'): shutil.copy(file, newDir) #check_call(["cp","-r",codeDir + '*',newDir]) #set inital Paramters files setup_param(newDir,param) if callback: result=subtask(callback).delay(task_id=str(initEnKFrun.request.id),wkdir=newDir) return {'task_id':result.task_id,'task_name':result.task_name} else: return newDir
def run(self): if subprocess.call(["git", "--version"]) != 0: print("ERROR:\n\tPlease install git.") exit(1) status_lines = subprocess.check_output(["git", "status"]).splitlines() current_branch = status_lines[0].strip().split()[-1].decode() print("On branch {}.".format(current_branch)) if current_branch != self.branch: print("ERROR:\n\tNew tags can only be made from branch \"{}\"." "".format(self.branch)) print("\tYou can use \"git checkout {}\" to switch the branch." "".format(self.branch)) exit(1) tags_output = subprocess.check_output(["git", "tag"]) tags = [tag.strip().decode() for tag in tags_output.splitlines()] tag = "v" + __version__ if tag in tags: print("Warning: \n\tTag {} already exists.".format(tag)) print("\tEdit the version information in {}".format( os.path.join(HERE, PACKAGE_NAME, "__init__.py") )) else: print("Creating tag \"{}\".".format(tag)) subprocess.check_call(["git", "tag", tag]) print("Pushing tag \"{}\" to remote \"{}\".".format(tag, self.remote)) subprocess.check_call(["git", "push", self.remote, tag])
def archive(conn, root, source, **kwargs): source = path.abspath(source) dir = path.dirname(source) base = path.basename(source) cursor = conn.cursor() cursor.execute("INSERT OR IGNORE INTO versions VALUES (:name, 0)", {"name": base}) cursor.execute("UPDATE versions SET version = version + 1 WHERE name = :name", {"name": base}) cursor.execute("SELECT version FROM versions WHERE name = :name", {"name": base}) version = cursor.fetchone()["version"] conn.commit() archive = "{base}-{version}.tar.gz".format(**locals()) fullPath = path.join(root, archive) commands = ["tar", "--checkpoint=.10000", "-C", dir, "-czf", fullPath, base] log(Level.Info, "Running command ", " ".join(commands)) subprocess.check_call(commands) print cursor.execute( """INSERT INTO items VALUES ( :name, :version, CURRENT_TIMESTAMP, 'Archived', :source, :archive); """, { "name": base, "version": version, "source": source, "archive": archive, }) conn.commit()
def generate_skeleton(package, path): """ Use conda skeleton pypi to generate a recipe for a package and save it to path. Parameters ---------- package: Package The package for which a recipe is to be generated. path: str Path to which the recipe should be written. """ additional_arguments = ['--version', str(package.required_version), '--output-dir', path] if package.setup_options: additional_arguments.extend(['--setup-options={}'.format(package.setup_options)]) if package.numpy_compiled_extensions: additional_arguments.append('--pin-numpy') subprocess.check_call(["conda", "skeleton", "pypi", package.pypi_name] + additional_arguments)
def build_dist(self): for sdir in self.staging_dirs: if os.path.exists(sdir): shutil.rmtree(sdir) main_stage, ninja_stage = self.staging_dirs modules = [os.path.splitext(os.path.split(x)[1])[0] for x in glob(os.path.join('mesonbuild/modules/*'))] modules = ['mesonbuild.modules.' + x for x in modules if not x.startswith('_')] modules += ['distutils.version'] modulestr = ','.join(modules) python = shutil.which('python') cxfreeze = os.path.join(os.path.dirname(python), "Scripts", "cxfreeze") if not os.path.isfile(cxfreeze): print("ERROR: This script requires cx_freeze module") sys.exit(1) subprocess.check_call([python, cxfreeze, '--target-dir', main_stage, '--include-modules', modulestr, 'meson.py']) if not os.path.exists(os.path.join(main_stage, 'meson.exe')): sys.exit('Meson exe missing from staging dir.') os.mkdir(ninja_stage) shutil.copy(shutil.which('ninja'), ninja_stage) if not os.path.exists(os.path.join(ninja_stage, 'ninja.exe')): sys.exit('Ninja exe missing from staging dir.')
def BuildYcmdLibs( args ): build_dir = mkdtemp( prefix = 'ycm_build.' ) try: full_cmake_args = [ '-G', GetGenerator( args ) ] if OnMac(): full_cmake_args.extend( CustomPythonCmakeArgs() ) full_cmake_args.extend( GetCmakeArgs( args ) ) full_cmake_args.append( p.join( DIR_OF_THIS_SCRIPT, 'cpp' ) ) os.chdir( build_dir ) subprocess.check_call( [ 'cmake' ] + full_cmake_args ) build_target = ( 'ycm_support_libs' if 'YCM_TESTRUN' not in os.environ else 'ycm_core_tests' ) build_command = [ 'cmake', '--build', '.', '--target', build_target ] if OnWindows(): build_command.extend( [ '--config', 'Release' ] ) else: build_command.extend( [ '--', '-j', str( NumCores() ) ] ) subprocess.check_call( build_command ) if 'YCM_TESTRUN' in os.environ: RunYcmdTests( build_dir ) finally: os.chdir( DIR_OF_THIS_SCRIPT ) rmtree( build_dir, ignore_errors = OnTravisOrAppVeyor() )
def create_profile_F(self): """Function to create the SSHTunnel Firefox profile All firefox instances must be closed (or they can be left open, but the function will killall firefox) """ subprocess.call(["killall", "firefox"]) #all firefox instance must be killed subprocess.check_call(["firefox","-CreateProfile","SSHTunnel"]) #Create a new Profile named SSHTunnel #Navigate to the profile folder: os.chdir('/home/'+ str(LOGNAME) +'/.mozilla/firefox/') list_of_items = os.listdir(os.getcwd()) for folder in list_of_items: if 'SSHTunnel' in folder: os.chdir(folder) break else: raise Exception("Create new profile for firefox failed") write_prefs_js() self.launch_firefox_instance()
def sendCommand(self, jbossHome, controller, user, password, cluster): self.fillParameters(jbossHome, controller, user, password) startCommand = self._clisg+cluster+":start-servers" print("eseguo: "+self._complPath+" "+self._cliconn+" "+self._complContr+" "+self._complUser+" "+self._complPwd+" "+startCommand) subprocess.check_call([self._complPath,self._cliconn,self._complContr,self._complUser,self._complPwd,startCommand])
def compare_review(review_spec, branch, remote, rebase=False): new_ps = None # none means latest if '-' in review_spec: review_spec, new_ps = review_spec.split('-') review, old_ps = parse_review_number(review_spec) if old_ps is None or old_ps == new_ps: raise InvalidPatchsetsToCompare(old_ps, new_ps) old_review = build_review_number(review, old_ps) new_review = build_review_number(review, new_ps) old_branch = fetch_review(old_review, branch, remote) checkout_review(old_branch) if rebase: print('Rebasing %s' % old_branch) rebase = rebase_changes(branch, remote, False) if not rebase: print('Skipping rebase because of conflicts') run_command_exc(CommandFailed, 'git', 'rebase', '--abort') new_branch = fetch_review(new_review, branch, remote) checkout_review(new_branch) if rebase: print('Rebasing also %s' % new_branch) if not rebase_changes(branch, remote, False): print("Rebasing of the new branch failed, " "diff can be messed up (use -R to not rebase at all)!") run_command_exc(CommandFailed, 'git', 'rebase', '--abort') subprocess.check_call(['git', 'diff', old_branch])
def start_dcvoip_process(): try: logger.info("Starting dcvoip process - Thanks Jonas Karlsson!") with open(os.devnull, 'wb') as devnull: subprocess.check_call(["sudo", "service", "dcvoip", "start"], stdout=devnull) except (subprocess.CalledProcessError, IOError): logging.warning("Unable to start the dcvoip process")
def stop_dcvoip_process(): try: logger.info("Stopping dcvoip process") with open(os.devnull, 'wb') as devnull: subprocess.check_call(["sudo", "service", "dcvoip", "stop"], stdout=devnull) except (subprocess.CalledProcessError, IOError): logging.warning("Unable to stop the dcvoip process")
def create_pool(service, name, replicas=2): ''' Create a new RADOS pool ''' if pool_exists(service, name): log("Ceph pool {} already exists, skipping creation".format(name), level=WARNING) return # Calculate the number of placement groups based # on upstream recommended best practices. osds = get_osds(service) if osds: pgnum = (len(osds) * 100 / replicas) else: # NOTE(james-page): Default to 200 for older ceph versions # which don't support OSD query from cli pgnum = 200 cmd = [ 'ceph', '--id', service, 'osd', 'pool', 'create', name, str(pgnum) ] check_call(cmd) cmd = [ 'ceph', '--id', service, 'osd', 'pool', 'set', name, 'size', str(replicas) ] check_call(cmd)
def archive_logs(log_dir): """Compress log files in given log_dir using gzip.""" log_files = [] for r, ds, fs in os.walk(log_dir): log_files.extend(os.path.join(r, f) for f in fs if is_log(f)) if log_files: subprocess.check_call(['gzip', '--best', '-f'] + log_files)
def do_build(package): print("Build started for %s" % package.name) sphinx_args = [package.path, package.build_path] copy_env = os.environ.copy() if package.devhelp: sphinx_args = ["-b", "devhelpfork"] + sphinx_args copy_env["PGIDOCGEN_TARGET_PREFIX"] = DEVHELP_PREFIX else: sphinx_args = ["-b", "html"] + sphinx_args copy_env["PGIDOCGEN_TARGET_PREFIX"] = "" copy_env["PGIDOCGEN_TARGET_BASE_PATH"] = \ os.path.dirname(package.build_path) subprocess.check_call( [sys.executable, "-m", "sphinx", "-n", "-q", "-a", "-E"] + sphinx_args, env=copy_env) # we don't rebuild, remove all caches shutil.rmtree(os.path.join(package.build_path, ".doctrees")) os.remove(os.path.join(package.build_path, ".buildinfo")) # remove some pages we don't need os.remove(os.path.join(package.build_path, "genindex.html")) os.remove(os.path.join(package.build_path, "search.html")) if os.name != "nt": for d in ["structs", "unions", "interfaces", "iface-structs", "class-structs"]: os.symlink("classes", os.path.join(package.build_path, d)) return package
def snapshot(source, destination, name=None): """Snapshot one directory to another. Specify names to snapshot small, named differences.""" source = source + sep destination = destination + sep if not path.isdir(source): raise RuntimeError("source is not a directory") if path.exists(destination): if not path.isdir(destination): raise RuntimeError("destination is not a directory") if name is None: raise RuntimeError("can't snapshot base snapshot if destination exists") snapdir = path.join(destination, ".snapdir") if path.exists(path.join(source, ".snapdir")): raise RuntimeError("snapdir exists in source directory") if name is None: check_call(["rsync", "--del", "-av", source, destination]) makedirs(snapdir) else: if not path.exists(snapdir): raise RuntimeError("No snapdir in destination directory") check_call(["rsync", "--del", "-av", "--only-write-batch={}".format(path.join(snapdir, name)), source, destination])
def __exit__(self, exc_type, _value, _traceback): for k in self._config_dict: if self._previous_values.get(k): subprocess.check_call( ['git', 'config', '--local', k, self._previous_values[k]]) else: subprocess.check_call(['git', 'config', '--local', '--unset', k])
def _exec(app, args, work_dir, extra_env, save_output): all_args = [app] if args: all_args.extend(args) all_env = os.environ.copy() if extra_env: all_env.update(extra_env) cmd = ' '.join(all_args) if _output_file and (not save_output): with open(_output_file, 'a') as f: f.write('\n') if extra_env: for var in sorted(extra_env.iterkeys()): f.write('%s = %s\n' % (var, extra_env[var])) f.write('\n') f.write('%s\n\n' % cmd) cmd = '%s >>%s 2>&1' % (cmd, _output_file) if save_output: return subprocess.check_output(cmd, cwd=work_dir, env=all_env, shell=True) else: subprocess.check_call(cmd, cwd=work_dir, env=all_env, shell=True)
def text_mof(pdf_part, pdf_year, pdf_number, pdf_name): s3_url = "https://mgax-mof.s3.amazonaws.com" pdf_url = s3_url + "/" + pdf_name with temp_dir() as tmp: pdf_local_path = tmp / pdf_name text_path = tmp / 'plain.txt' with pdf_local_path.open('wb') as f: resp = requests.get(pdf_url, stream=True) assert resp.status_code == 200 for chunk in FileWrapper(resp.raw): f.write(chunk) subprocess.check_call(['pdftotext', pdf_local_path, text_path]) with text_path.open('r') as f: raw_text = f.read() json = dict([('part', int(pdf_part)), ('year', int(pdf_year)), ('number', int(pdf_number)), ('slug', pdf_name.split('.')[0]), ('text', raw_text)]) resp = requests.put(flask.current_app.config['ELASTIC_SEARCH_URL'] + pdf_name.split('.')[0], data=flask.json.dumps(json)) assert 200 <= resp.status_code < 300, repr(resp)
def get_compressor(ctools=COMPRESSING_TOOLS): global UPTO am_dir_pattern = "/usr/share/automake-*" am_files_pattern = am_dir_pattern + "/am/*.am" if len(glob.glob(am_dir_pattern)) == 0: # automake is not installed. UPTO = STEP_PRECONFIGURE logging.warn("Automake not found. The process will go up to the step: " + UPTO) return ctools[-1] # fallback to the default (gzip). for ct in ctools: # NOTE: bzip2 tries compressing input from stdin even it is invoked # with --version option. So give null input (/dev/null) to it. c_exists = 0 == subprocess.check_call( "%(command)s --version > /dev/null 2> /dev/null < /dev/null" % ct, shell=True ) if c_exists: am_support_c = 0 == subprocess.check_call( "grep -q -e '^dist-%s:' %s 2> /dev/null" % (ct.command, am_files_pattern), shell=True ) if am_support_c: return ct # gzip must exist at least and not reached here: raise RuntimeError("No compressor found! Aborting...")
def run_script(prefix, dist, action='post-link', env_prefix=None): """ call the post-link (or pre-unlink) script, and return True on success, False on failure """ path = join(prefix, 'Scripts' if on_win else 'bin', '.%s-%s.%s' % ( name_dist(dist), action, 'bat' if on_win else 'sh')) if not isfile(path): return True if on_win: try: args = [os.environ['COMSPEC'], '/c', path] except KeyError: return False else: args = ['/bin/bash', path] env = os.environ env['PREFIX'] = env_prefix or prefix env['PKG_NAME'], env['PKG_VERSION'], env['PKG_BUILDNUM'] = str(dist).rsplit('-', 2) try: subprocess.check_call(args, env=env) except subprocess.CalledProcessError: return False return True
def pull_file(self, device_path, destination_path, remove_original=False): """Copies or moves the specified file on the device to the host.""" subprocess.check_call(self._adb_command([ 'pull', device_path, destination_path])) if remove_original: subprocess.check_call(self._adb_command([ 'shell', 'rm', device_path]))
def set_versions(base_version, build_number, label): print("mvers: " + check_string_output(["agvtool", "mvers", "-terse1"])) print("vers: " + check_string_output(["agvtool", "vers", "-terse"])) marketing_version = "%s.%s %s" % (base_version, build_number, label) build_version = "%s.%s" % (base_version, build_number) subprocess.check_call(["agvtool", "new-marketing-version", marketing_version]) subprocess.check_call(["agvtool", "new-version", "-all", build_version])
def prepare_release(build_number, image_source_path): release_dir = "release" try: os.makedirs(release_dir) except FileExistsError: pass tag = 'builds/%s/%s' % (base_version, build_number) subprocess.check_call(['git', 'tag', tag]) import appcast appcast_text = appcast.generate_appcast(image_source_path, updates_template_file, build_number, updates_signing_key_file) with open(os.path.join(release_dir, updates_appcast_file), 'w') as appcast_file: appcast_file.write(appcast_text) import shutil copied_image = os.path.join(release_dir, os.path.basename(image_source_path)) unversioned_image = os.path.join(release_dir, artifact_prefix + ".dmg") shutil.copyfile(image_source_path, copied_image) shutil.copyfile(image_source_path, unversioned_image) publish_release_notes_file = os.path.join(release_dir, os.path.basename(release_notes_file)) shutil.copyfile(release_notes_file, publish_release_notes_file) publish_release_notes_filebase, publish_release_notes_ext = os.path.splitext(publish_release_notes_file) publish_release_notes_version_file = "%s-%s%s" % (publish_release_notes_filebase, build_number, publish_release_notes_ext) shutil.copyfile(release_notes_file, publish_release_notes_version_file)
def sklearn2pmml(estimator, mapper, pmml, with_repr = False, verbose = False): if(not isinstance(estimator, BaseEstimator)): raise TypeError("The estimator object is not an instance of " + BaseEstimator.__name__) if((mapper is not None) and (not isinstance(mapper, DataFrameMapper))): raise TypeError("The mapper object is not an instance of " + DataFrameMapper.__name__) cmd = ["java", "-cp", os.pathsep.join(_classpath()), "org.jpmml.sklearn.Main"] dumps = [] try: estimator_pkl = _dump(estimator) cmd.extend(["--pkl-estimator-input", estimator_pkl]) dumps.append(estimator_pkl) if(with_repr): estimator_repr = repr(estimator) cmd.extend(["--repr-estimator", estimator_repr]) if(mapper): mapper_pkl = _dump(mapper) cmd.extend(["--pkl-mapper-input", mapper_pkl]) dumps.append(mapper_pkl) if(with_repr): mapper_repr = repr(mapper) cmd.extend(["--repr-mapper", mapper_repr]) cmd.extend(["--pmml-output", pmml]) if(verbose): print(cmd) subprocess.check_call(cmd) finally: for dump in dumps: os.remove(dump)
def _commit_and_push(self): """ Commit all the files and push. """ deploy = self._deploy_branch source = self._source_branch remote = self._remote_name source_commit = uni_check_output(['git', 'rev-parse', source]) commit_message = ( 'Nikola auto commit.\n\n' 'Source commit: %s' 'Nikola version: %s' % (source_commit, __version__) ) commands = [ ['git', 'add', '-A'], ['git', 'commit', '-m', commit_message], ['git', 'push', '-f', remote, '%s:%s' % (deploy, deploy)], ['git', 'checkout', source], ] for command in commands: self.logger.info("==> {0}".format(command)) try: subprocess.check_call(command) except subprocess.CalledProcessError as e: self.logger.error( 'Failed GitHub deployment — command {0} ' 'returned {1}'.format(e.cmd, e.returncode) ) sys.exit(e.returncode)
def sendpfast(x, pps=None, mbps=None, realtime=None, loop=0, iface=None): """Send packets at layer 2 using tcpreplay for performance pps: packets per second mpbs: MBits per second realtime: use packet's timestamp, bending time with realtime value loop: number of times to process the packet list iface: output interface """ if iface is None: iface = conf.iface argv = [conf.prog.tcpreplay, "--intf1=%s" % iface ] if pps is not None: argv.append("--pps=%i" % pps) elif mbps is not None: argv.append("--mbps=%i" % mbps) elif realtime is not None: argv.append("--multiplier=%i" % realtime) else: argv.append("--topspeed") if loop: argv.append("--loop=%i" % loop) f = get_temp_file() argv.append(f) wrpcap(f, x) try: subprocess.check_call(argv) except KeyboardInterrupt: log_interactive.info("Interrupted by user") except Exception,e: log_interactive.error("while trying to exec [%s]: %s" % (argv[0],e))
def generate_and_upload_docs(): old_cwd = os.getcwd() tmp = tempfile.TemporaryDirectory() os.chdir(tmp.name) log_info('Entered temporary directory', os.getcwd(), 'cloning repository now') clone_url = f'https://{username}:{access_token}@github.com/{repository}.git' subprocess.check_call(['git', 'clone', '--single-branch', '--branch', 'gh-pages', clone_url, '.']) subprocess.check_call(['git', 'config', 'user.name', username]) subprocess.check_call(['git', 'config', 'user.email', f'{username}@users.noreply.github.com']) for project in changed_projects: project_docs_out = os.path.join(tmp.name, project.cmake_project_name) project_docs_in = project.generate_docs() shutil.rmtree(project_docs_out, ignore_errors=True) copy_tree(project_docs_in, project_docs_out) if subprocess.check_output(['git', 'status', '-s']).decode(sys.getdefaultencoding()): with LogGroup('Uploading'): log_info('Creating a commit') subprocess.check_call(['git', 'add', '.']) subprocess.check_call(['git', 'status']) subprocess.check_call(['git', 'commit', '-a', '-m', f'Generated docs for {sha[0:8]}']) log_info('Pushing') subprocess.check_call(['git', 'push', clone_url, 'gh-pages']) else: log_info('There are no changes to documentation') os.chdir(old_cwd)
def toeps(counter, ext): global dirname subprocess.check_call(['convert',\ '%s/plot-%s.%s' % (dirname, counter, ext), \ '%s/plot-%s.eps' % (dirname, counter)])
def build_extension(self, ext): # self.parallel = 4 extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) # required for auto-detection & inclusion of auxiliary "native" libs if not extdir.endswith(os.path.sep): extdir += os.path.sep debug = int(os.environ.get("DEBUG", 0)) if self.debug is None else self.debug cfg = "Debug" if debug else "Release" # CMake lets you override the generator - we need to check this. # Can be set with Conda-Build, for example. cmake_generator = os.environ.get("CMAKE_GENERATOR", "") # Set Python_EXECUTABLE instead if you use PYBIND11_FINDPYTHON cmake_args = [ "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}".format(extdir), "-DPYTHON_EXECUTABLE={}".format(sys.executable), "-DCMAKE_BUILD_TYPE={}".format(cfg), # not used on MSVC, but no harm "-DCNIS_WITH_PYTHON_API=ON", ] build_args = [] # Adding CMake arguments set as environment variable # (needed e.g. to build for ARM OSx on conda-forge) if "CMAKE_ARGS" in os.environ: cmake_args += [item for item in os.environ["CMAKE_ARGS"].split(" ") if item] if self.compiler.compiler_type != "msvc": # Using Ninja-build since it a) is available as a wheel and b) # multithreads automatically. MSVC would require all variables be # exported for Ninja to pick it up, which is a little tricky to do. # Users can override the generator with CMAKE_GENERATOR in CMake # 3.15+. if not cmake_generator: try: import ninja # noqa: F401 cmake_args += ["-GNinja"] except ImportError: pass else: # Single config generators are handled "normally" single_config = any(x in cmake_generator for x in {"NMake", "Ninja"}) # CMake allows an arch-in-generator style for backward compatibility contains_arch = any(x in cmake_generator for x in {"ARM", "Win64"}) # Specify the arch if using MSVC generator, but only if it doesn't # contain a backward-compatibility arch spec already in the # generator name. if not single_config and not contains_arch: cmake_args += ["-A", PLAT_TO_CMAKE[self.plat_name]] # Multi-config generators have a different way to specify configs if not single_config: cmake_args += [ "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}".format(cfg.upper(), extdir) ] build_args += ["--config", cfg] if sys.platform.startswith("darwin"): # Cross-compile support for macOS - respect ARCHFLAGS if set archs = re.findall(r"-arch (\S+)", os.environ.get("ARCHFLAGS", "")) if archs: cmake_args += ["-DCMAKE_OSX_ARCHITECTURES={}".format(";".join(archs))] # Set CMAKE_BUILD_PARALLEL_LEVEL to control the parallel build level # across all generators. if "CMAKE_BUILD_PARALLEL_LEVEL" not in os.environ: # self.parallel is a Python 3 only way to set parallel jobs by hand # using -j in the build_ext call, not supported by pip or PyPA-build. if hasattr(self, "parallel") and self.parallel: # CMake 3.12+ only. build_args += ["-j{}".format(self.parallel)] if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) subprocess.check_call( ["cmake", ext.sourcedir] + cmake_args, cwd=self.build_temp ) subprocess.check_call( ["cmake", "--build", "."] + build_args, cwd=self.build_temp )
def main(): # Read arguments args = argparser.parse_args() architecture = args.architecture particle_diameter = args.particle_diameter trainging_dir = args.training_dir annot_dir = args.annot_dir input_size = args.input_size output_dir = args.output_directory if subprocess.os.path.exists(output_dir): sp_global_def.ERROR("Output folder already exists. Stop execution.") else: subprocess.os.makedirs(output_dir) sp_global_def.write_command(output_dir) num_patches = args.num_patches overlap_patches = args.overlap_patches train_times = args.train_times saved_weights_name = args.saved_weights_name saved_weights_name = subprocess.os.path.join(output_dir, saved_weights_name) pretrained_weights_name = args.pretrained_weights_name batch_size = args.batch_size learning_rate = args.learning_rate np_epoch = args.np_epoch object_scale = args.object_scale no_object_scale = args.no_object_scale coord_scale = args.coord_scale valid_image_dir = args.valid_image_dir valid_annot_dir = args.valid_annot_dir fine_tune = args.fine_tune gpu_fraction = args.gpu_fraction num_cpu = args.num_cpu cryolo_train_path = args.cryolo_train_path skiplowpass = args.skiplowpass cutoff = args.cutoff filtered_dir = args.filtered_dir warmup = args.warmup early_stop = int(args.early) str_gpus = [str(entry).strip() for entry in args.gpu.split(",")] arg_gpu = " ".join(str_gpus) # Create config file model_dict = { "architecture": architecture, "input_size": input_size, "anchors": [particle_diameter, particle_diameter], "overlap_patches": overlap_patches, "max_box_per_image": 1000, "num_patches": num_patches, } if not skiplowpass: model_dict["filter"] = [cutoff, filtered_dir] else: if args.usejanni: model_dict["filter"] = [ args.janni_model, args.janni_overlap, args.janni_batches, filtered_dir, ] train_dict = { "train_image_folder": trainging_dir, "train_annot_folder": annot_dir, "train_times": train_times, "pretrained_weights": pretrained_weights_name, "batch_size": batch_size, "learning_rate": learning_rate, "nb_epoch": np_epoch, "warmup_epochs": 0, "object_scale": object_scale, "no_object_scale": no_object_scale, "coord_scale": coord_scale, "class_scale": 1.0, "saved_weights_name": saved_weights_name, "debug": True, "log_path": "cryolo_logs/", } valid_dict = { "valid_image_folder": valid_image_dir, "valid_annot_folder": valid_annot_dir, "valid_times": 1, } dict = {"model": model_dict, "train": train_dict, "valid": valid_dict} path = subprocess.os.path.join(output_dir, "config_yolo.json") with open(path, "w") as f: json.dump(dict, f, ensure_ascii=False, indent=4) # Run the training config_argument = "-c=" + path warmup_argument = "-w=" + str(warmup) gpu_argument = "-g " + arg_gpu early_stop = "-e=" + str(early_stop) fine_tune_argument = "" if fine_tune: fine_tune_argument = "--fine_tune" gpu_fraction_arg = "--gpu_fraction=1.0" if gpu_fraction < 1.0 and gpu_fraction > 0.0: gpu_fraction_arg = "--gpu_fraction=" + str(gpu_fraction) num_cpu_arg = "" if num_cpu != -1: num_cpu_arg = "--num_cpu=" + str(num_cpu) cryolo_ex_pth = "cryolo_train.py" if cryolo_train_path: cryolo_ex_pth = cryolo_train_path """Multiline Comment0""" if fine_tune: warmup_argument = "-w=0" command = [ cryolo_ex_pth, config_argument, warmup_argument, gpu_argument, early_stop, gpu_fraction_arg, ] if fine_tune: command.append(fine_tune_argument) if num_cpu != -1: command.append(num_cpu_arg) subprocess.check_call(command)
def test1_01_version(self): subprocess.check_call(['gcat_workflow_cloud', '--version'])
def test_simple(self): platforms = [] # Xilinx Spartan6 platforms += [("official", "minispartan6")] platforms += [("community", "sp605")] # Xilinx Artix7 platforms += [("official", "arty")] platforms += [("official", "nexys4ddr")] platforms += [("official", "nexys_video")] platforms += [("partner", "netv2")] platforms += [("community", "ac701")] # Xilinx Kintex7 platforms += [("official", "kc705")] platforms += [("official", "genesys2")] platforms += [("community", "kx2")] # Xilinx Kintex Ultrascale platforms += [("official", "kcu105")] # Intel Cyclone4 platforms += [("official", "de0nano")] platforms += [("community", "de2_115")] # Intel Cyclone5 platforms += [("community", "de1soc")] # Intel Max10 platforms += [("community", "de10lite")] # Lattice iCE40 platforms += [("partner", "tinyfpga_bx")] platforms += [("partner", "fomu_evt")] platforms += [("partner", "fomu_hacker")] platforms += [("partner", "fomu_pvt")] # Lattice MachXO2 platforms += [("official", "machxo3")] # Lattice ECP3 platforms += [("official", "versa_ecp3")] # Lattice ECP5 platforms += [("official", "versa_ecp5")] platforms += [("partner", "ulx3s")] platforms += [("partner", "trellisboard")] platforms += [("community", "ecp5_evn")] # Microsemi PolarFire platforms += [("official", "avalanche")] for s, p in platforms: with self.subTest(platform=p): cmd = """\ litex_boards/official/targets/simple.py litex_boards.{s}.platforms.{p} \ --cpu-type=vexriscv \ --no-compile-software \ --no-compile-gateware \ --uart-name="stub" \ """.format(s=s, p=p) subprocess.check_call(cmd, shell=True)
H0 uniform 0 %e\n\ PHI0 uniform 0 %f\n\ COSIOTA uniform -1 1\n\ PSI uniform 0 %f" % (h0ul, np.pi, np.pi/2.) priorf = 'test.prior' f = open(priorf, 'w') f.write(priorfile) f.close() # run code commandline="\ %s --detectors %s --par-file %s --input-files %s --outfile %s --prior-file %s --Nlive %s --Nmcmcinitial %s --sampleprior %s" \ % (execu, dets, parf, datafile, outfile, priorf, Nlive, Nmcmcinitial, priorsamples) sp.check_call(commandline, shell=True) # read in prior samples f = h5py.File(outfile, 'r') a = f['lalinference'] h0samps = a['lalinference_nest']['nested_samples']['H0'][:] # get normed histogram of samples [n, nedges] = np.histogram(h0samps, bins=20, range=(0., h0ul), density=True) nc = np.cumsum(n)*(nedges[1]-nedges[0]) stat, p = ss.kstest(nc, 'uniform') print "K-S test p-value for upper range of %e = %f" % (h0ul, p) if p < 0.005:
return 1 def check_stab_of_linear_props(rounds_start, rounds_end, samples_num): for rounds in range(rounds_start, rounds_end + 1): if check_linear_props(rounds, rounds, samples_num) != 0: print("Transformation is linear at {0}-th round".format(rounds)) return 1 return 0 if __name__ == '__main__': rounds_start = 1 rounds_end = 100 samples_num = 15 # create directory samples/ create_samples_dir() # creating samples in dir samples/ generate_samples_for_checking_linear_props(samples_num) # encrypting and decrypting check_call(["clefia_exec.exe", str(rounds_start), str(rounds_end)]) # check linear properties check_linear_props(rounds_start, rounds_end, samples_num) # check stability of linear properties #check_stab_of_linear_props(rounds_start, rounds_end, samples_num)
def main(): # Read config generated by configure. config = configparser.ConfigParser() configfile = os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "config.ini") config.read_file(open(configfile)) src_dir = config["environment"]["SRCDIR"] build_dir = config["environment"]["BUILDDIR"] tests_dir = os.path.join(src_dir, 'test', 'functional') # Parse arguments and pass through unrecognised args parser = argparse.ArgumentParser( add_help=False, usage='%(prog)s [test_runner.py options] [script options] [scripts]', description=__doc__, epilog=''' Help text and arguments for individual test script:''', formatter_class=argparse.RawTextHelpFormatter) parser.add_argument( '--coverage', action='store_true', help='generate a basic coverage report for the RPC interface') parser.add_argument( '--exclude', '-x', help= 'specify a comma-separated-list of scripts to exclude. Do not include the .py extension in the name.' ) parser.add_argument( '--extended', action='store_true', help='run the extended test suite in addition to the basic tests') parser.add_argument('--cutoff', type=int, default=DEFAULT_EXTENDED_CUTOFF, help='set the cutoff runtime for what tests get run') parser.add_argument( '--force', '-f', action='store_true', help= 'run tests even on platforms where they are disabled by default (e.g. windows).' ) parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit') parser.add_argument( '--jobs', '-j', type=int, default=DEFAULT_JOBS, help='how many test scripts to run in parallel. Default=4.') parser.add_argument( '--keepcache', '-k', action='store_true', help= 'the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.' ) parser.add_argument('--quiet', '-q', action='store_true', help='only print results summary and failure logs') parser.add_argument('--tmpdirprefix', '-t', default=tempfile.gettempdir(), help="Root directory for datadirs") parser.add_argument( '--junitouput', '-ju', default=os.path.join(build_dir, 'junit_results.xml'), help="file that will store JUnit formatted test results.") args, unknown_args = parser.parse_known_args() # Create a set to store arguments and create the passon string tests = set(arg for arg in unknown_args if arg[:2] != "--") passon_args = [arg for arg in unknown_args if arg[:2] == "--"] passon_args.append("--configfile={}".format(configfile)) # Set up logging logging_level = logging.INFO if args.quiet else logging.DEBUG logging.basicConfig(format='%(message)s', level=logging_level) # Create base test directory tmpdir = os.path.join("{}", "bitcoin_test_runner_{:%Y%m%d_%H%M%S}").format( args.tmpdirprefix, datetime.datetime.now()) os.makedirs(tmpdir) logging.debug("Temporary test directory at {}".format(tmpdir)) enable_wallet = config["components"].getboolean("ENABLE_WALLET") enable_utils = config["components"].getboolean("ENABLE_UTILS") enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND") if config["environment"]["EXEEXT"] == ".exe" and not args.force: # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964 print( "Tests currently disabled on Windows by default. Use --force option to enable" ) sys.exit(0) if not (enable_wallet and enable_utils and enable_bitcoind): print( "No functional tests to run. Wallet, utils, and bitcoind must all be enabled" ) print( "Rerun `configure` with -enable-wallet, -with-utils and -with-daemon and rerun make" ) sys.exit(0) # Build list of tests all_scripts = get_all_scripts_from_disk(tests_dir, NON_SCRIPTS) # Check all tests with parameters actually exist for test in TEST_PARAMS: if not test in all_scripts: print("ERROR: Test with parameter {} does not exist, check it has " "not been renamed or deleted".format(test)) sys.exit(1) if tests: # Individual tests have been specified. Run specified tests that exist # in the all_scripts list. Accept the name with or without .py # extension. test_list = [ t for t in all_scripts if (t in tests or re.sub(".py$", "", t) in tests) ] # Allow for wildcard at the end of the name, so a single input can # match multiple tests for test in tests: if test.endswith('*'): test_list.extend( [t for t in all_scripts if t.startswith(test[:-1])]) # Make the list unique test_list = list(set(test_list)) # do not cut off explicitly specified tests cutoff = sys.maxsize else: # No individual tests have been specified. # Run all tests that do not exceed test_list = all_scripts cutoff = args.cutoff if args.extended: cutoff = sys.maxsize # Remove the test cases that the user has explicitly asked to exclude. if args.exclude: for exclude_test in args.exclude.split(','): if exclude_test + ".py" in test_list: test_list.remove(exclude_test + ".py") # Use and update timings from build_dir only if separate # build directory is used. We do not want to pollute source directory. build_timings = None if (src_dir != build_dir): build_timings = Timings(os.path.join(build_dir, 'timing.json')) # Always use timings from scr_dir if present src_timings = Timings( os.path.join(src_dir, "test", "functional", 'timing.json')) # Add test parameters and remove long running tests if needed test_list = get_tests_to_run(test_list, TEST_PARAMS, cutoff, src_timings, build_timings) if not test_list: print( "No valid test scripts specified. Check that your test is in one " "of the test lists in test_runner.py, or run test_runner.py with no arguments to run all tests" ) sys.exit(0) if args.help: # Print help for test_runner.py, then print help of the first script # and exit. parser.print_help() subprocess.check_call([os.path.join(tests_dir, test_list[0]), '-h']) sys.exit(0) if not args.keepcache: shutil.rmtree(os.path.join(build_dir, "test", "cache"), ignore_errors=True) run_tests(test_list, build_dir, tests_dir, args.junitouput, config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args, build_timings)
def sendSig(self, interruptSignal): hit = 0 # try signalling the process group if not hit and self.useProcGroup and runtime.platformType == "posix": sig = getattr(signal, "SIG" + interruptSignal, None) if sig is None: log.msg("signal module is missing SIG{0}".format(interruptSignal)) elif not hasattr(os, "kill"): log.msg("os module is missing the 'kill' function") elif self.process.pgid is None: log.msg("self.process has no pgid") else: log.msg("trying to kill process group {0}".format( self.process.pgid)) try: os.killpg(self.process.pgid, sig) log.msg(" signal {0} sent successfully".format(sig)) self.process.pgid = None hit = 1 except OSError: log.msg('failed to kill process group (ignored): {0}'.format( (sys.exc_info()[1]))) # probably no-such-process, maybe because there is no process # group pass elif runtime.platformType == "win32": if interruptSignal is None: log.msg("interruptSignal==None, only pretending to kill child") elif self.process.pid is not None: if interruptSignal == "TERM": log.msg("using TASKKILL PID /T to kill pid {0}".format( self.process.pid)) subprocess.check_call( "TASKKILL /PID {0} /T".format(self.process.pid)) log.msg("taskkill'd pid {0}".format(self.process.pid)) hit = 1 elif interruptSignal == "KILL": log.msg("using TASKKILL PID /F /T to kill pid {0}".format( self.process.pid)) subprocess.check_call( "TASKKILL /F /PID {0} /T".format(self.process.pid)) log.msg("taskkill'd pid {0}".format(self.process.pid)) hit = 1 # try signalling the process itself (works on Windows too, sorta) if not hit: try: log.msg("trying process.signalProcess('{0}')".format( interruptSignal)) self.process.signalProcess(interruptSignal) log.msg(" signal {0} sent successfully".format(interruptSignal)) hit = 1 except OSError: log.err("from process.signalProcess:") # could be no-such-process, because they finished very recently pass except error.ProcessExitedAlready: log.msg("Process exited already - can't kill") # the process has already exited, and likely finished() has # been called already or will be called shortly pass return hit
def test_import(self): subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
t_r_ok = T_R_COMPILER.findall(filename) relax_ok = RELAX_COMPILER.findall(filename) e_t_ok = E_T_COMPILER.findall(filename) acc_data_path = os.path.join(acc_input_dir, filename) emg_data_path = os.path.join(emg_input_dir, filename) if not os.access(acc_data_path, os.R_OK) or not os.access( emg_data_path, os.R_OK): print("error", filename) continue output_name = re.sub(".txt", ".json", filename) output_path = os.path.join(output_dir, output_name) if len(extra_ok) > 0: log_name = re.sub(".txt", "_log.csv", filename) log_path = os.path.join(log_input_dir, log_name) subprocess.check_call([ "python", "mix_reader.py", '2', acc_data_path, emg_data_path, '--csv', log_path, '--output_dir', output_path ]) elif len(t_r_ok) > 0: log_name = re.sub(".txt", "_log.csv", filename) log_path = os.path.join(log_input_dir, log_name) subprocess.check_call([ "python", "mix_reader.py", '1', acc_data_path, emg_data_path, '--csv', log_path, '--output_dir', output_path ]) elif len(e_t_ok) > 0: log_name = re.sub(".txt", "_log.csv", filename) log_path = os.path.join(log_input_dir, log_name) subprocess.check_call([ "python", "mix_reader.py", '3', acc_data_path, emg_data_path, '--csv', log_path, '--output_dir', output_path ])
def main(argv=sys.argv): parser = argparse.ArgumentParser( description='Generate and setup Grunt infrastructure, ' 'then compile JS/LESS bundles for Plone.', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('-i', '--instance', dest='instance', help='path to instance executable. If not provided, ' 'will look in bin this was executed from for ' 'instance or client1') parser.add_argument( '-s', '--site-id', dest='site_id', default='Plone', help='ID of the Plone site to fetch the configuration from. ' 'Used only while Gruntfile generation.') parser.add_argument( '-b', '--bundle', dest='bundle', default='all', help='Name of bundle to compile. Used while compile step.') parser.add_argument( '--compile-dir', dest='compile_dir', default='', help='Output directory for the compiled bundle files. ' 'Used only while Gruntfile generation. ' 'If not given the directory is looked up from Plone registry. ') parser.add_argument('-d', '--base-dir', dest='base_dir', default='.', help='Base directory for this script ' '(by default current working directory).') parser.add_argument('-G', '--skip-generate-gruntfile', dest='skip_gruntfile', action='store_true', help='Skip generation of Gruntfile.js') parser.add_argument( '-I', '--skip-npm-install', dest='skip_npminstall', action='store_true', help='Skip npm install step', ) parser.add_argument( '-C', '--skip-compile', dest='skip_compile', action='store_true', help='Skip compile step (running grunt)', ) args = parser.parse_args() base_path = args.base_dir if base_path == '.': base_path = os.getcwd() # generates only if not already there generate_package_json(base_path) if not args.skip_npminstall: cmd = [NPM_CMD, 'install'] print('Setup npm env') print('Running command: %s' % ' '.join(cmd)) subprocess.check_call(cmd) # Generate Gruntfile grunt = os.path.join(base_path, 'node_modules', 'grunt-cli', 'bin', 'grunt') if not args.skip_gruntfile: generate_gruntfile(base_path, args.instance, args.site_id, args.compile_dir) gruntfile = os.path.join(base_path, 'Gruntfile.js') if not os.path.exists(gruntfile): print("Error, no Gruntfile.js generated at {0} where expected".format( gruntfile)) if not args.skip_compile: print('Compile {0}'.format(args.bundle)) cmd = [ grunt, '--gruntfile={0}'.format(gruntfile), 'compile-{0}'.format(args.bundle) ] print('Running command: %s' % ' '.join(cmd)) subprocess.check_call(cmd) print('Done.')
def test_main_exec(self): subprocess.check_call( [sys.executable, 'youtube_dl/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL)
def hgadd(dest, directories): """Inform hg of the files in |directories|.""" print("hg addremoving...") for d in directories: subprocess.check_call(["hg", "addremove", "%s/%s" % (dest, d)])
f.write(r.content) os.execl(sys.executable, sys.executable, *sys.argv) except: pass # Repositories cloning if "init" in sys.argv[1:]: for name in repos.keys(): os.chdir(os.path.join(current_path)) if not os.path.exists(name): url, need_recursive, need_develop, sha1 = repos[name] # clone repo (recursive if needed) print("[cloning " + name + "]...") full_url = url + name opts = "--recursive" if need_recursive else "" subprocess.check_call("git clone " + full_url + " " + opts, shell=True) if sha1 is not None: os.chdir(os.path.join(current_path, name)) os.system("git checkout {:7x}".format(sha1)) # Repositories update if "update" in sys.argv[1:]: for name in repos.keys(): os.chdir(os.path.join(current_path)) url, need_recursive, need_develop, sha1 = repos[name] print(url) if not os.path.exists(name): raise Exception("{} not initialized, please (re)-run init and install first.".format(name)) # update print("[updating " + name + "]...") os.chdir(os.path.join(current_path, name))
def makeTempDirectory(): tmpDirectory = getDataLocation('tmp') check_call(['mkdir', '-p', tmpDirectory])
def run_script(args): fnull = open(os.devnull, 'w') subprocess.check_call(args, stdout=fnull, stderr=fnull)
def real_main(): (opts, action, cluster_name) = parse_args() # Input parameter validation if opts.wait is not None: # NOTE: DeprecationWarnings are silent in 2.7+ by default. # To show them, run Python with the -Wdefault switch. # See: https://docs.python.org/3.5/whatsnew/2.7.html warnings.warn( "This option is deprecated and has no effect. " "spark-ec2 automatically waits as long as necessary for clusters to startup.", DeprecationWarning) if opts.ebs_vol_num > 8: print >> stderr, "ebs-vol-num cannot be greater than 8" sys.exit(1) try: conn = ec2.connect_to_region(opts.region) except Exception as e: print >> stderr, (e) sys.exit(1) # Select an AZ at random if it was not specified. if opts.zone == "": opts.zone = random.choice(conn.get_all_zones()).name if action == "launch": if opts.slaves <= 0: print >> sys.stderr, "ERROR: You have to start at least 1 slave" sys.exit(1) if opts.resume: (master_nodes, slave_nodes) = get_existing_cluster(conn, opts, cluster_name) else: (master_nodes, slave_nodes) = launch_cluster(conn, opts, cluster_name) wait_for_cluster_state(cluster_instances=(master_nodes + slave_nodes), cluster_state='ssh-ready', opts=opts) setup_cluster(conn, master_nodes, slave_nodes, opts, True) elif action == "destroy": print "Are you sure you want to destroy the cluster %s?" % cluster_name print "The following instances will be terminated:" (master_nodes, slave_nodes) = get_existing_cluster(conn, opts, cluster_name, die_on_error=False) for inst in master_nodes + slave_nodes: print "> %s" % inst.public_dns_name msg = "ALL DATA ON ALL NODES WILL BE LOST!!\nDestroy cluster %s (y/N): " % cluster_name response = raw_input(msg) if response == "y": print "Terminating master..." for inst in master_nodes: inst.terminate() print "Terminating slaves..." for inst in slave_nodes: inst.terminate() # Delete security groups as well if opts.delete_groups: print "Deleting security groups (this will take some time)..." group_names = [ cluster_name + "-master", cluster_name + "-slaves" ] wait_for_cluster_state(cluster_instances=(master_nodes + slave_nodes), cluster_state='terminated', opts=opts) attempt = 1 while attempt <= 3: print "Attempt %d" % attempt groups = [ g for g in conn.get_all_security_groups() if g.name in group_names ] success = True # Delete individual rules in all groups before deleting groups to # remove dependencies between them for group in groups: print "Deleting rules in security group " + group.name for rule in group.rules: for grant in rule.grants: success &= group.revoke( ip_protocol=rule.ip_protocol, from_port=rule.from_port, to_port=rule.to_port, src_group=grant) # Sleep for AWS eventual-consistency to catch up, and for instances # to terminate time.sleep(30) # Yes, it does have to be this long :-( for group in groups: try: conn.delete_security_group(group.name) print "Deleted security group " + group.name except boto.exception.EC2ResponseError: success = False print "Failed to delete security group " + group.name # Unfortunately, group.revoke() returns True even if a rule was not # deleted, so this needs to be rerun if something fails if success: break attempt += 1 if not success: print "Failed to delete all security groups after 3 tries." print "Try re-running in a few minutes." elif action == "login": (master_nodes, slave_nodes) = get_existing_cluster(conn, opts, cluster_name) master = master_nodes[0].public_dns_name print "Logging into master " + master + "..." proxy_opt = [] if opts.proxy_port is not None: proxy_opt = ['-D', opts.proxy_port] subprocess.check_call( ssh_command(opts) + proxy_opt + ['-t', '-t', "%s@%s" % (opts.user, master)]) elif action == "reboot-slaves": response = raw_input("Are you sure you want to reboot the cluster " + cluster_name + " slaves?\n" + "Reboot cluster slaves " + cluster_name + " (y/N): ") if response == "y": (master_nodes, slave_nodes) = get_existing_cluster(conn, opts, cluster_name, die_on_error=False) print "Rebooting slaves..." for inst in slave_nodes: if inst.state not in ["shutting-down", "terminated"]: print "Rebooting " + inst.id inst.reboot() elif action == "get-master": (master_nodes, slave_nodes) = get_existing_cluster(conn, opts, cluster_name) print master_nodes[0].public_dns_name elif action == "stop": response = raw_input( "Are you sure you want to stop the cluster " + cluster_name + "?\nDATA ON EPHEMERAL DISKS WILL BE LOST, " + "BUT THE CLUSTER WILL KEEP USING SPACE ON\n" + "AMAZON EBS IF IT IS EBS-BACKED!!\n" + "All data on spot-instance slaves will be lost.\n" + "Stop cluster " + cluster_name + " (y/N): ") if response == "y": (master_nodes, slave_nodes) = get_existing_cluster(conn, opts, cluster_name, die_on_error=False) print "Stopping master..." for inst in master_nodes: if inst.state not in ["shutting-down", "terminated"]: inst.stop() print "Stopping slaves..." for inst in slave_nodes: if inst.state not in ["shutting-down", "terminated"]: if inst.spot_instance_request_id: inst.terminate() else: inst.stop() elif action == "start": (master_nodes, slave_nodes) = get_existing_cluster(conn, opts, cluster_name) print "Starting slaves..." for inst in slave_nodes: if inst.state not in ["shutting-down", "terminated"]: inst.start() print "Starting master..." for inst in master_nodes: if inst.state not in ["shutting-down", "terminated"]: inst.start() wait_for_cluster_state(cluster_instances=(master_nodes + slave_nodes), cluster_state='ssh-ready', opts=opts) setup_cluster(conn, master_nodes, slave_nodes, opts, False) else: print >> stderr, "Invalid action: %s" % action sys.exit(1)
def do_install(self, name, data): if name in data: utils.makedirs(self.output_dir) LOGGER.notice('Downloading: ' + data[name]) zip_file = BytesIO() zip_file.write(requests.get(data[name]).content) LOGGER.notice('Extracting: {0} into plugins'.format(name)) utils.extract_all(zip_file, 'plugins') dest_path = os.path.join('plugins', name) else: try: plugin_path = utils.get_plugin_path(name) except: LOGGER.error("Can't find plugin " + name) return False utils.makedirs(self.output_dir) dest_path = os.path.join(self.output_dir, name) if os.path.exists(dest_path): LOGGER.error("{0} is already installed".format(name)) return False LOGGER.notice('Copying {0} into plugins'.format(plugin_path)) shutil.copytree(plugin_path, dest_path) reqpath = os.path.join(dest_path, 'requirements.txt') print(reqpath) if os.path.exists(reqpath): LOGGER.notice('This plugin has Python dependencies.') LOGGER.notice('Installing dependencies with pip...') try: subprocess.check_call(('pip', 'install', '-r', reqpath)) except subprocess.CalledProcessError: LOGGER.error('Could not install the dependencies.') print('Contents of the requirements.txt file:\n') with codecs.open(reqpath, 'rb', 'utf-8') as fh: print(indent(fh.read(), 4 * ' ')) print('You have to install those yourself or through a ' 'package manager.') else: LOGGER.notice('Dependency installation succeeded.') reqnpypath = os.path.join(dest_path, 'requirements-nonpy.txt') if os.path.exists(reqnpypath): LOGGER.notice('This plugin has third-party ' 'dependencies you need to install ' 'manually.') print('Contents of the requirements-nonpy.txt file:\n') with codecs.open(reqnpypath, 'rb', 'utf-8') as fh: for l in fh.readlines(): i, j = l.split('::') print(indent(i.strip(), 4 * ' ')) print(indent(j.strip(), 8 * ' ')) print() print('You have to install those yourself or through a package ' 'manager.') confpypath = os.path.join(dest_path, 'conf.py.sample') if os.path.exists(confpypath): LOGGER.notice('This plugin has a sample config file.') print('Contents of the conf.py.sample file:\n') with codecs.open(confpypath, 'rb', 'utf-8') as fh: print( indent( pygments.highlight(fh.read(), PythonLexer(), TerminalFormatter()), 4 * ' ')) return True
def run(self): """Run a command to compile and build assets.""" subprocess.check_call('sh ./asreview/webapp/compile_assets.sh', shell=True)
def _call(args): DEVNULL = open(os.devnull, 'w') subprocess.check_call(" ".join(args), shell=True, stdout=DEVNULL, stderr=DEVNULL)
def systemctl(action: str, *units: str) -> int: """Invokes systemctl on the respective units.""" systemctl_ = CONFIG.get('MollyGuard', 'systemctl', fallback=SYSTEMCTL) return check_call((systemctl_, action, *units))
def deploy_files(conn, root_dir, opts, master_nodes, slave_nodes, modules): active_master = master_nodes[0].public_dns_name num_disks = get_num_disks(opts.instance_type) hdfs_data_dirs = "/mnt/ephemeral-hdfs/data" mapred_local_dirs = "/mnt/hadoop/mrlocal" spark_local_dirs = "/mnt/spark" if num_disks > 1: for i in range(2, num_disks + 1): hdfs_data_dirs += ",/mnt%d/ephemeral-hdfs/data" % i mapred_local_dirs += ",/mnt%d/hadoop/mrlocal" % i spark_local_dirs += ",/mnt%d/spark" % i cluster_url = "%s:7077" % active_master if "." in opts.spark_version: # Pre-built spark & shark deploy (spark_v, shark_v) = get_spark_shark_version(opts) else: # Spark-only custom deploy spark_v = "%s|%s" % (opts.spark_git_repo, opts.spark_version) shark_v = "" modules = filter(lambda x: x != "shark", modules) template_vars = { "master_list": '\n'.join([i.public_dns_name for i in master_nodes]), "active_master": active_master, "slave_list": '\n'.join([i.public_dns_name for i in slave_nodes]), "cluster_url": cluster_url, "hdfs_data_dirs": hdfs_data_dirs, "mapred_local_dirs": mapred_local_dirs, "spark_local_dirs": spark_local_dirs, "swap": str(opts.swap), "modules": '\n'.join(modules), "spark_version": spark_v, "shark_version": shark_v, "hadoop_major_version": opts.hadoop_major_version, "spark_worker_instances": "%d" % opts.worker_instances, "spark_master_opts": opts.master_opts } if opts.copy_aws_credentials: template_vars["aws_access_key_id"] = conn.aws_access_key_id template_vars["aws_secret_access_key"] = conn.aws_secret_access_key else: template_vars["aws_access_key_id"] = "" template_vars["aws_secret_access_key"] = "" # Create a temp directory in which we will place all the files to be # deployed after we substitue template parameters in them tmp_dir = tempfile.mkdtemp() for path, dirs, files in os.walk(root_dir): if path.find(".svn") == -1: dest_dir = os.path.join('/', path[len(root_dir):]) local_dir = tmp_dir + dest_dir if not os.path.exists(local_dir): os.makedirs(local_dir) for filename in files: if filename[0] not in '#.~' and filename[-1] != '~': dest_file = os.path.join(dest_dir, filename) local_file = tmp_dir + dest_file with open(os.path.join(path, filename)) as src: with open(local_file, "w") as dest: text = src.read() for key in template_vars: text = text.replace("{{" + key + "}}", template_vars[key]) dest.write(text) dest.close() # rsync the whole directory over to the master machine command = [ 'rsync', '-rv', '-e', stringify_command(ssh_command(opts)), "%s/" % tmp_dir, "%s@%s:/" % (opts.user, active_master) ] subprocess.check_call(command) # Remove the temp directory we created above shutil.rmtree(tmp_dir)
def run(self): # noqa """Run a command to compile and build assets.""" subprocess.check_call('./airflow/www/compile_assets.sh')
def cryptsetup(action: str, *args: str) -> int: """Runs cryptsetup.""" cryptsetup_ = CONFIG.get('MollyGuard', 'cryptsetup', fallback=CRYPTSETUP) return check_call((cryptsetup_, action, *args))
#08.04.2020 - Holoflow 0.1. import subprocess import argparse #Argument parsing parser = argparse.ArgumentParser(description='Runs holoflow pipeline.') parser.add_argument('-hostrg', help="host reference genome", dest="host_ref_gen", required=True) parser.add_argument('-ibam', help="all bam file", dest="all_bam", required=True) parser.add_argument('-1', help="path1", dest="read1", required=True) parser.add_argument('-2', help="path2", dest="read2", required=True) parser.add_argument('-obam', help="host bam file", dest="host_bam", required=True) args = parser.parse_args() all_bam=args.all_bam host_ref_gen=args.host_ref_gen host_bam=args.host_bam read1=args.read1 read2=args.read2 # Run hostbam1Cmd = 'module load tools samtools/1.9 && samtools view -T'+host_ref_gen+' -b -F12 '+all_bam+' > '+host_bam+'' subprocess.check_call(hostbam1Cmd, shell=True) hostbam2Cmd = 'module load tools samtools/1.9 && samtools view -T '+host_ref_gen+' -b -f12 '+all_bam+' | samtools fastq -1 '+read1+' -2 '+read2+' -' subprocess.check_call(hostbam2Cmd, shell=True) rmAllbamCmd = 'rm'+all_bam+'' subprocess.check_call(rmAllbamCmd, shell=True)
def setUpClass(cls): """Run before all tests: Creates an auth configuration""" cls.port = QGIS_POSTGRES_SERVER_PORT cls.username = '******' cls.password = '******' cls.dbname = 'test_password' cls.tempfolder = QGIS_PG_TEST_PATH cls.certsdata_path = os.path.join(unitTestDataPath('auth_system'), 'certs_keys') cls.hostname = 'localhost' cls.data_path = os.path.join(cls.tempfolder, 'data') os.mkdir(cls.data_path) # Disable SSL verification for setup operations env = dict(os.environ) env['PGSSLMODE'] = 'disable' cls.setUpAuth() subprocess.check_call([ os.path.join(QGIS_POSTGRES_EXECUTABLE_PATH, 'initdb'), '-D', cls.data_path ]) cls.server = subprocess.Popen([ os.path.join(QGIS_POSTGRES_EXECUTABLE_PATH, 'postgres'), '-D', cls.data_path, '-c', "config_file=%s" % cls.pg_conf ], env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # Wait max 10 secs for the server to start end = time.time() + 10 while True: line = cls.server.stderr.readline() print(line) if line.find(b"database system is ready to accept") != -1: break if time.time() > end: raise Exception("Timeout connecting to PostgreSQL") # Create a DB subprocess.check_call([ os.path.join(QGIS_POSTGRES_EXECUTABLE_PATH, 'createdb'), '-h', 'localhost', '-p', cls.port, 'test_password' ], env=env) # Inject test SQL from test path test_sql = os.path.join(unitTestDataPath('provider'), 'testdata_pg.sql') subprocess.check_call([ os.path.join(QGIS_POSTGRES_EXECUTABLE_PATH, 'psql'), '-h', 'localhost', '-p', cls.port, '-f', test_sql, cls.dbname ], env=env) # Create a role subprocess.check_call([ os.path.join(QGIS_POSTGRES_EXECUTABLE_PATH, 'psql'), '-h', 'localhost', '-p', cls.port, '-c', 'CREATE ROLE "%s" WITH SUPERUSER LOGIN PASSWORD \'%s\'' % (cls.username, cls.password), cls.dbname ], env=env)
def run(self): """Run twine to upload to test.pypi.org.""" subprocess.check_call( # nosec ("twine upload --repository-url https://test.pypi.org/legacy/" + " --verbose dist/*").split())
def twitter_send(data): dt_now = datetime.datetime.now() date = dt_now.strftime('%m月%d日 %H時%M分%S秒') img_ok = True send_status = "動作なし\n" img_path = "start.jpg" rospy.loginfo("status : %d",data.status) if data.status == 1: send_status = "ただいまスタートゾーンから出発しました。\n" img_path = "start.jpg" elif data.status == 5: img_ok = False if data.towel1: send_status = "バスタオル1枚目を干しに行きます。\n" else: send_status = "バスタオル2枚目を干しに行きます。\n" elif data.status == 10: img_ok = False send_status = "シーツを干しに行きます。\n" elif data.status == 7: img_path = "bathtowel.jpg" if data.towel1: send_status = "バスタオル1枚目を干し終わりました。\n" else: send_status = "バスタオル2枚目を干し終わりました。\n" elif data.status == 12: img_ok = False send_status = "シーツを干し中です。\n" elif data.status == 15: img_path = "sheets.jpg" send_status = "シーツを干し終わりました。\n" elif data.status == 18: img_ok = False send_status = "スタートゾーンに戻ります。\n" elif data.status == 21: img_path = "arrival.jpg" send_status = "スタートゾーンに到着しました。\n" else: return text = date + "\n明石高専自動機からお伝えします。\n練習中\n" + coat_msg[data.coat] + fight_msg[data.fight] + send_status + "#ロボコン\n" + "#明石高専\n" + "#AkashiSuperDry" print(text) # 生の投稿データの出力 media_id = 0 param = {'status' : text} start_time = time.time() while not os.path.isfile(img_path) and img_ok: if time.time() - start_time > 2: print("画像ファイルなし") img_ok = False break if img_ok: img_data = {'media' : open(img_path, 'rb')} req_media = twitter.post(url_media, files = img_data) if req_media.status_code != 200: print("画像アップロード失敗 : "+ req_media.text) else: print("画像アップロード成功 : "+ req_media.text) media_id = json.loads(req_media.text)['media_id'] print('media_id:%d'% media_id) param['media_ids'] = [media_id] req_media = twitter.post(url_text, params = param) args = ['rm','-f',img_path] try: res = subprocess.check_call(args) print(res) except: print("cannot remove : " + img_path)