def un_deb(src, dst): ar = proc.get_program("ar", "extract .deb files") log.mkdirs_if_needed(dst) proc.run([ar, 'x', os.path.abspath(src)], cwd=dst) for entry_base in os.listdir(dst): entry = os.path.join(dst, entry_base) result = extract_if_needed(entry, os.path.dirname(entry), keep_basename=True) if result: log.rmfile(entry)
def un_tar_lzma(src, dst): import corepkgs # NOTE: the python lzma extractor is not working right #with tarfile.open(src, "r:xz") as file: # file.extractall(path=dst) tar = corepkgs.get_tar_bootstrap() #tar = proc.get_program("tar", "extract .tar.xz files") log.mkdirs_if_needed(dst) proc.run([tar, "-xf", src, "-C", dst])
def un_7z(src, dst): _7z = r'C:\Program Files\7-Zip\7z.exe' if os.path.exists(_7z): log.log("!!!!!!!!!!!!!!!!!!!!!!!!!!!! WARNING !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") log.log("!!!!!!!!!!!!!!!!!!!!!!!!!!!! WARNING !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") log.log("Using '{}' without declaring a dependency on it!".format(_7z)) log.log("!!!!!!!!!!!!!!!!!!!!!!!!!!!! WARNING !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") log.log("!!!!!!!!!!!!!!!!!!!!!!!!!!!! WARNING !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") else: _7z = proc.get_program("7z", "extract .7z files") log.mkdirs_if_needed(dst) proc.run([_7z, "x", src, "-o" + dst])
def runOp(executor, op, bootstrap): cmd = op['cmd'] # TODO: keep the same environment accross all ops # TODO: pull in the environment from all the inputs? # maybe some inputs we want to pull in the environment and some we don't? # if so, I could just select which ones I want to pull in with an op. # it would be similar to addPackageLinks but it would't be adding them to # the @tmpout directory but instead the stage directory. env = op.get('env') with ScopedEnv(executor) as env: proc_env = make_env(executor, bootstrap, env) #log.log("[DEBUG] proc_env:") #import pprint #pprint(proc_env) proc.run(cmd, env=proc_env)
def runner_curry(): cmdstr = '%s %s' % (shell, wrapped) logging.info('Executing shim script via wrapper: %s' % cmdstr) rc = proc.run(cmdstr) if rc != 0: msg = 'Failed to run %s of %s/%s' % (wrapped, self.name, self.version) raise RuntimeError, msg
def package_shim_dependencies(script, env): ''' Return the package dependencies by running the first "dependencies" shim script found in <pathlist> in the given <env> environment. Return a list of (<package>, <optional constraint>) tuples. ''' # for debugging pv = '%s/%s' % (env['ORCH_PACKAGE_NAME'], env['ORCH_PACKAGE_VERSION']) if not script: logging.debug('no dependencies shim for package %s' % pv) return [] fd, fn = tempfile.mkstemp() cmdstr = ' '.join([shell, script, fn]) rc = proc.run(cmdstr, env=env) if rc != 0: raise RuntimeError, '%s dependencies command returned non-zero error %d: %s' % \ (pv, rc, cmdstr) #os.close(fd) fd = open(fn) ret = [] for line in fd.readlines(): line = line.strip() #logging.debug('%s dependency line: "%s"' % (pv, line)) if not line: continue line = line.split(' ',1) ret.append((line[0], line[1:] or '')) os.remove(fn) logging.debug('dependencies for package %s: %s from %s' % (pv, str(ret), script)) return ret
def decode_speech(self, acoustic_model_directory, language_model_file, dictionary_file, wavfile): self.record_to_file(self.wavfile) # FIXME: replace with meaningful event self.bus.send(">ears>PROMPT TYPE 3") try: status = '' stdout = '' stderr = '' try: last_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "pocketsphinx-decoder.py") stdout, stderr, status = proc.run(" ".join([ last_path, acoustic_model_directory, language_model_file, dictionary_file, wavfile, "2>","/dev/null" ]), timeout=8) except proc.Timeout: print "TIMED OUT: "+status+" "+stdout+" "+stderr os.remove(self.wavfile) return stdout except KeyboardInterrupt: sys.exit(0)
def create_set(self, name, base="base"): "Create a new set of disks, and switch to it" disks = {"hd-root": {}} for disk_name, info in disks.iteritems(): info["base"] = self.disk_get_path(disk_name) self.set_disk_set(name) os.mkdir(self.disk_get_dir()) for disk_name, info in disks.iteritems(): new = self.disk_get_path(disk_name) run("qemu-img create -f qcow2 -o backing_file={0} {1}".format( info["base"], new))
def logrun(command, timeout=None): command_str = " ".join(command) + "\n" if timeout: logmsg(command_str + (" TIMEOUT=%d" % timeout)) (out, err, ret) = run(command, timeout) else: logmsg(command_str) (out, err, ret) = run(command) if out: logmsg("stdout: " + out) if err: logmsg("stderr: " + err) if ret: logmsg("Returned: %d" % ret) return (out, err, ret)
def git(url, targetdir, final=None): '''Make a "clone" of a git repository at <url> in to targetdir. If <final> is not given it will be the last part of the <url>'s path. if <final> ends in '.git' it a bare clone will be made. The URL options can be "tag=<tag>" to specify a tag to checkout. If a "branch=<branch>" is also given it will name the branch hold the tagged checkout. If no branch is given it defaults to the tag name. ''' urlp = urlparse.urlparse(url) query = query2dict(urlp.query) tag = query.get('tag') branch = query.get('branch') url_no_query = urlparse.ParseResult(urlp.scheme, urlp.netloc, urlp.path, '', '', '').geturl() if not final: final = os.path.basename(urlp.path) bare = "" if final.endswith('.git'): bare = '--bare' fullpath = os.path.join(targetdir, final) if os.path.exists(fullpath): return 0 util.assuredir(targetdir) rc = proc.run("git clone %s %s %s" % (bare, url_no_query, fullpath)) if rc: return rc if not tag: return 0 if not branch: branch = tag rc = proc.run('git --work-tree=%s --git-dir=%s/.git checkout -b %s %s' % (fullpath, fullpath, branch, tag)) return rc
def git(url, targetdir, final = None): '''Make a "clone" of a git repository at <url> in to targetdir. If <final> is not given it will be the last part of the <url>'s path. if <final> ends in '.git' it a bare clone will be made. The URL options can be "tag=<tag>" to specify a tag to checkout. If a "branch=<branch>" is also given it will name the branch hold the tagged checkout. If no branch is given it defaults to the tag name. ''' urlp = urlparse.urlparse(url) query = query2dict(urlp.query) tag = query.get('tag') branch = query.get('branch') url_no_query = urlparse.ParseResult(urlp.scheme, urlp.netloc, urlp.path, '', '', '').geturl() if not final: final = os.path.basename(urlp.path) bare = "" if final.endswith('.git'): bare = '--bare' fullpath = os.path.join(targetdir, final) if os.path.exists(fullpath): return 0 util.assuredir(targetdir) rc = proc.run("git clone %s %s %s" % (bare, url_no_query, fullpath)) if rc: return rc if not tag: return 0 if not branch: branch = tag rc = proc.run('git --work-tree=%s --git-dir=%s/.git checkout -b %s %s' % (fullpath, fullpath, branch, tag)) return rc
def svn(url, targetdir, final=None): ''' Do an svn checkout of <url> to <targetdir>. ''' print 'in svn: %s' % str(url) urlp = urlparse.urlparse(url) util.assuredir(targetdir) if not final: final = os.path.splitext(os.path.basename(urlp.path))[0] fullpath = os.path.join(targetdir, final) if os.path.exists(fullpath): return 0 rc = proc.run('svn checkout %s %s' % (url, fullpath)) return rc
def find_bad_repositories(list_of_repos): bad_repos = [] for repo in list_of_repos: out, err, exit_code = proc.run( ['/usr/bin/svn', 'status', '%s' % repo], timeout=10) # Match L as 3rd character followed by some whitespace # then dir name (non-whitespace) match = re.search(r"\s\sL\s+\S+", out) if match: bad_repos.append(repo) print("%s is broken" % repo) else: print("%s is OK" % repo) return bad_repos
def fixElf(op): import corepkgs files = op['files'] interp = op.get('interp') rpath = op.get('rpath') if not interp and not rpath: log.log( "Error: the 'fixElf' operation requires either 'interp' or 'rpath'" ) sys.exit() for file in files: log.log("FIXELF {}".format(file)) args = [corepkgs.get_patchelf_bootstrap()] if interp: args += ['--set-interpreter', interp] #elf.change_interpreter(file, interp.encode("ascii")) if rpath: #log.log("[DEBUG] chrpath is in '{}'".format(chrpath64)) #proc.run(chrpath64 #sys.exit("not impl") args += ['--set-rpath', rpath] proc.run(args + [file])
def ungit(src, dst, creates, treeish='HEAD'): '''Produce an archive from a local git repository <src> and unpack it to <dst>. Unlike the other un*() functions in this module the <creates> argument must be given and it is interpreted as the sub-directory under <dst> that should be created. It will usually be chosen be the <package_name> or <package_name>-<package_version>. An optional git "treeish" can be specified, else HEAD is used. ''' if not prepare(src, dst, creates): return cmd = 'git --git-dir={src} archive --format=tar --prefix={creates}/ {treeish} | tar -xf- -C {dst}'.format(**locals()) print cmd return proc.run(cmd)
def guess_method(url): '''Guess the function to use to download a given URL. Return a tuple containing the function (one of "web", "git" or "svn") and the URL modified to be what is expected by that method ''' urlp = urlparse.urlparse(url) scheme = urlp.scheme.split('+') # we could instead do this with an eval() s2f = {'git': git, 'svn': svn, 'web': web} # caller follows the rules and provides <method>+<actual_scheme> if len(scheme) == 2: if scheme == ('svn', 'ssh'): # svn directly accepts svn+ssh:// return (svn, url) # but not others like svn+http:// simple_url = url.split('+', 1)[1] return (s2f[scheme[0]], simple_url) # scheme is explicitly dedicated to one method. if scheme in ['git', 'svn']: return (s2f[scheme], url) # Not guaranteed, but a good guess if urlp.path.endswith('.git'): return (git, url) # give these to git. if scheme in ['ssh', 'rsync']: return (git, url) # if there is a non-".git" file extension, probably just a file if os.path.splitext(urlp.path)[1]: return (web, url) # what is left might be svn or plain web. Let's check url_no_query = urlparse.ParseResult(urlp.scheme, urlp.netloc, urlp.path, '', '', '').geturl() def noop(stuff): pass # keep proc quiet rc = proc.run('svn info %s' % url_no_query, logger=noop) if rc == 0: return (svn, url) return (web, url)
def ungit(src, dst, creates, treeish='HEAD'): '''Produce an archive from a local git repository <src> and unpack it to <dst>. Unlike the other un*() functions in this module the <creates> argument must be given and it is interpreted as the sub-directory under <dst> that should be created. It will usually be chosen be the <package_name> or <package_name>-<package_version>. An optional git "treeish" can be specified, else HEAD is used. ''' if not prepare(src, dst, creates): return cmd = 'git --git-dir={src} archive --format=tar --prefix={creates}/ {treeish} | tar -xf- -C {dst}'.format( **locals()) print cmd return proc.run(cmd)
def guess_method(url): '''Guess the function to use to download a given URL. Return a tuple containing the function (one of "web", "git" or "svn") and the URL modified to be what is expected by that method ''' urlp = urlparse.urlparse(url) scheme = urlp.scheme.split('+') # we could instead do this with an eval() s2f = {'git':git, 'svn':svn, 'web':web} # caller follows the rules and provides <method>+<actual_scheme> if len(scheme) == 2: if scheme == ('svn', 'ssh'): # svn directly accepts svn+ssh:// return (svn, url) # but not others like svn+http:// simple_url = url.split('+',1)[1] return (s2f[scheme[0]], simple_url) # scheme is explicitly dedicated to one method. if scheme in ['git','svn']: return (s2f[scheme], url) # Not guaranteed, but a good guess if urlp.path.endswith('.git'): return (git, url) # give these to git. if scheme in ['ssh','rsync']: return (git, url) # if there is a non-".git" file extension, probably just a file if os.path.splitext(urlp.path)[1]: return (web, url) # what is left might be svn or plain web. Let's check url_no_query = urlparse.ParseResult(urlp.scheme, urlp.netloc, urlp.path, '', '', '').geturl() def noop(stuff): pass # keep proc quiet rc = proc.run('svn info %s' % url_no_query, logger=noop) if rc == 0: return (svn, url) return (web, url)
def package_shim_directories(pathlist, named, env = None): ''' Return a list of directories from the given <pathlist> which contain directory in <named> (a string or list of strings). If a "version" shim script is found in a directory, run it in the given environment <env> and only keep the directory if a zero error code is returned. ''' ret = [] fullenv = dict(os.environ) if env: fullenv.update(env) for name in named: name = name.strip() # be nice for path in pathlist: pdir = os.path.join(path, name) #logging.debug('Checking shim directory %s' % pdir) if not os.path.exists(pdir): #logging.debug('Ignoring nonexistent shim directory %s' % pdir) continue vshim = os.path.join(pdir, 'version') # no version shim script, assume applicable if not os.path.exists(vshim): #logging.debug('Using versionless shim directory %s' % pdir) ret.append(pdir) continue cmdstr = '%s %s' % (shell, vshim) print 'cmdstr="%s", env=:' % cmdstr for k,v in fullenv.iteritems(): #print '(%s)%s = (%s)%s' % (type(k),k, type(v), v) assert isinstance(k,basestring) assert isinstance(v,basestring) rc = proc.run(cmdstr, env=fullenv) if rc == 0: ret.append(pdir) continue return ret
def package_shim_environment(filename, pathlist, env): ''' Produce the package environment set up in the given <filename> by calling the "environment" shim script. ''' logging.debug('generating env file %s' % filename) env_shim = package_shim_script('environment', pathlist) if not env_shim: logging.debug('No environment shim script for %s' % env['ORCH_PACKAGE_NAME']) return if not os.path.exists(env_shim): logging.debug('Environment shim script does not exist: %s' % env_shim) return cmdstr = ' '.join([shell, env_shim, filename]) rc = proc.run(cmdstr, env=env) if rc != 0: raise RuntimeError, 'Command returned non-zero error %d: %s' % (rc, cmdstr) return
def build_package(self): command = self._build_command(self.ios_dev_cert_password) command_to_print = self._build_command("<hidden password>") print(command_to_print) paths.setup_environment_for_ios() print() if not os.path.exists(self.output_path): print("Creating output folder ( " + self.output_path + " )...") os.makedirs(self.output_path) print() sys.stdout.write("Wait for it... ") sys.stdout.flush() stdout, stderr, returncode = proc.run(command) if len(stderr) == 0: print("Done!") else: print("stdout: " + stdout.decode(encoding="UTF-8")) print("stderr: " + stderr.decode(encoding="UTF-8")) print() print("Go get your package in: " + self.output_path) winsound.PlaySound('C:\\Windows\\Media\\tada.wav', winsound.SND_FILENAME)
def transform(bitcodefile, configfile): # setting some variables corvettepath = os.getenv("CORVETTE_PATH") if platform.system() == 'Darwin': sharedlib = corvettepath + "/precision-tuning/Passes.dylib" else: sharedlib = corvettepath + "/precision-tuning/Passes.so" # modified bitcode file mbitcodefile = "m_" + sys.argv[1] mbitcode = open(mbitcodefile, 'w') # modified + optimized bitcode file obitcodefile = "o_" + sys.argv[1] obitcode = open(obitcodefile, 'w') # temp bitcode file tempfile = "temp_" + sys.argv[1] temp = open(tempfile, 'w') output = open("output.txt", 'w') # ###################################### # running the transformation LLVM passes # passes are run in the order the appear # ###################################### #command = ['opt', '-load', sharedlib, "-json-config=" + configfile, "-adjust-operators", "--die", "--time-passes", bitcodefile] command = ['opt', '-load', sharedlib, "-json-config=" + configfile, "-adjust-operators", "--die", bitcodefile] retval = call(command, stdin=None, stdout=mbitcode, stderr=None) # return -1 if running LLVM passes fails if retval <> 0: return -1 # ########################### # removing redundant castings # ########################### command = ['opt', '-load', sharedlib, "-json-config=" + configfile, "-remove-dead-casting", mbitcodefile] retval = call(command, stdin=None, stdout=obitcode, stderr=None) # return -3 if removing dead castings fails if retval <> 0: return -3 # We aren't using stats, so we don't need to run pass #command = ['opt', '-load', sharedlib, "-measure-metric", obitcodefile] #retval = call(command, stdin=None, stdout=temp, stderr=None) # ###################################### # running the modified+optimized program # ###################################### command = ['lli', obitcodefile] #command = ['lli', mbitcodefile] # retval = call(command, stdin=None, stdout=output, stderr=None) try: retval = proc.run(command, timeout=30) except proc.Timeout: return -4 output.close() # return -2 if running the modified+optimized bitcode fails if retval <> 0: return -2 # reading output output = open("sat.cov", 'r') firstline = output.readline() firstline = firstline.strip() if (firstline == "true"): # ########################## # get the dynamic score # ######################### # mbitcodefile_name = mbitcodefile[:-3] # remove .bc # command = [corvettepath + "/benchmarks/gsl/instrument.sh", mbitcodefile_name, "."] # call(command, stdin=None, stdout=None, stderr=None) # command = ['lli', obitcodefile] # call(command, stdin=None, stdout=None, stderr=None) command = ['llc', obitcodefile, '-o', sys.argv[1] + '.s'] call(command, stdin=None, stdout=None, stderr=None) command = ['gcc', sys.argv[1] + '.s', '-lm', '-o', sys.argv[1] + '.out'] call(command, stdin=None, stdout=None, stderr=None) command = ['./' + sys.argv[1] + '.out'] call(command, stdin=None, stdout=None, stderr=None) return 1 else: return 0
def fix(current_repository): out, err, exit_code = proc.run( ['/usr/bin/svn', 'cleanup', '%s' % current_repository], timeout=10) print(err, out) return not exit_code # return invert of bool
def transform(bitcodefile, configfile, timeout): # setting some variables corvettepath = os.getenv("CORVETTE_PATH") if platform.system() == 'Darwin': sharedlib = corvettepath + "/src/Passes.dylib" else: sharedlib = corvettepath + "/src/Passes.so" # modified bitcode file mbitcodefile = "m_" + sys.argv[1] mbitcode = open(mbitcodefile, 'w') # temp bitcode file tempfile = "temp_" + sys.argv[1] temp = open(tempfile, 'w') output = open("output.txt", 'w') # ###################################### # running the transformation LLVM passes # passes are run in the order the appear # ###################################### command = ['opt', '-load', sharedlib, "-json-config=" + configfile, "-adjust-operators", bitcodefile] retval = call(command, stdin=None, stdout=mbitcode, stderr=None) # return -1 if running LLVM passes fails if retval <> 0: return -1 # running modified bitcode command = ['opt', '-O2', mbitcodefile, '-o', sys.argv[1] + '_opt.bc'] call(command, stdin=None, stdout=None, stderr=None) command = ['llc', sys.argv[1] + '_opt.bc', '-o', sys.argv[1] + '.s'] call(command, stdin=None, stdout=None, stderr=None) #command = ['clang', sys.argv[1] + '.s', '-Ofast -lm -llog -L$CORVETTE_PATH/logging ', '-o', sys.argv[1] + '.out'] #call(command, stdin=None, stdout=None, stderr=None) os.system('gcc ' + sys.argv[1] + '.s ' + '-O2 -lm -llog -L$HIFP_PRECI/logging -o ' + sys.argv[1] +'.out') #retval = os.system('$CORVETTE_PATH/scripts/run_benchmark.sh ' + sys.argv[1] +'.out') command = ['./' + sys.argv[1] + '.out'] timeout=0 if timeout==0: retval = call(command, stdin=None, stdout=None, stderr=None) else: try: retval = proc.run(command, timeout/1e08) except proc.Timeout: return -4 # return -3 if crashed when run if retval <> 0: return -3 output = open("sat.cov", 'r') firstline = output.readline() firstline = firstline.strip() if (firstline == "true"): return 1 else: return 0
def transform(bitcodefile, configfile): # setting some variables corvettepath = os.getenv("CORVETTE_PATH") if platform.system() == 'Darwin': sharedlib = corvettepath + "/precision-tuning/Passes.dylib" else: sharedlib = corvettepath + "/precision-tuning/Passes.so" # modified bitcode file mbitcodefile = "m_" + sys.argv[1] mbitcode = open(mbitcodefile, 'w') # modified + optimized bitcode file obitcodefile = "o_" + sys.argv[1] obitcode = open(obitcodefile, 'w') # temp bitcode file tempfile = "temp_" + sys.argv[1] temp = open(tempfile, 'w') output = open("output.txt", 'w') # ###################################### # running the transformation LLVM passes # passes are run in the order the appear # ###################################### #command = ['opt', '-load', sharedlib, "-json-config=" + configfile, "-adjust-operators", "--die", "--time-passes", bitcodefile] command = [ 'opt', '-load', sharedlib, "-json-config=" + configfile, "-adjust-operators", "--die", bitcodefile ] retval = call(command, stdin=None, stdout=mbitcode, stderr=None) # return -1 if running LLVM passes fails if retval <> 0: return -1 # ########################### # removing redundant castings # ########################### command = [ 'opt', '-load', sharedlib, "-json-config=" + configfile, "-remove-dead-casting", mbitcodefile ] retval = call(command, stdin=None, stdout=obitcode, stderr=None) # return -3 if removing dead castings fails if retval <> 0: return -3 # We aren't using stats, so we don't need to run pass #command = ['opt', '-load', sharedlib, "-measure-metric", obitcodefile] #retval = call(command, stdin=None, stdout=temp, stderr=None) # ###################################### # running the modified+optimized program # ###################################### command = ['lli', obitcodefile] #command = ['lli', mbitcodefile] # retval = call(command, stdin=None, stdout=output, stderr=None) try: retval = proc.run(command, timeout=30) except proc.Timeout: return -4 output.close() # return -2 if running the modified+optimized bitcode fails if retval <> 0: return -2 # reading output output = open("sat.cov", 'r') firstline = output.readline() firstline = firstline.strip() if (firstline == "true"): # ########################## # get the dynamic score # ######################### # mbitcodefile_name = mbitcodefile[:-3] # remove .bc # command = [corvettepath + "/benchmarks/gsl/instrument.sh", mbitcodefile_name, "."] # call(command, stdin=None, stdout=None, stderr=None) # command = ['lli', obitcodefile] # call(command, stdin=None, stdout=None, stderr=None) command = ['llc', obitcodefile, '-o', sys.argv[1] + '.s'] call(command, stdin=None, stdout=None, stderr=None) command = [ 'gcc', sys.argv[1] + '.s', '-lm', '-o', sys.argv[1] + '.out' ] call(command, stdin=None, stdout=None, stderr=None) command = ['./' + sys.argv[1] + '.out'] call(command, stdin=None, stdout=None, stderr=None) return 1 else: return 0
def un_tar_bz(src, dst): import corepkgs tar = corepkgs.get_tar_bootstrap() #tar = proc.get_program("tar", "extract .tar.bz2 files") log.mkdirs_if_needed(dst) proc.run([tar, "-xjf", src, "-C", dst])
def runMakeOp(executor, op, bootstrap): args = op['args'] env = op.get('env') with ScopedEnv(executor) as env: proc.run(['make', '-j' + str(multiprocessing.cpu_count())] + args, env=make_env(executor, bootstrap, env))
def new_disk(self, name, size): "Create a new disk and add it to the current set" run("qemu-img create -f qcow2 {0} {1}".format(self.disk_get_path(name), size))