def run(self): self.testReady() # submits the input file to Gaussian process = Popen([self.executablePath, self.inputFilePath, self.outputFilePath]) process.communicate() # necessary to wait for executable termination! return self.verifyOutputFile()
def ly2png(lily, outfile, kvs): p = Popen([ "lilypond", "-dno-point-and-click", "-dbackend=eps", "-djob-count=2", "-ddelete-intermediate-files", "-o", outfile, "-" ], stdin=PIPE, stdout=stderr) p.stdin.write(("\\paper{\n" "indent=0\\mm\n" "oddFooterMarkup=##f\n" "oddHeaderMarkup=##f\n" "bookTitleMarkup = ##f\n" "scoreTitleMarkup = ##f\n" "line-width = %s\n" "}\n" "#(set-global-staff-size %s)\n" % ( kvs['width'][:-2] + '\\' + kvs['width'][-2:], kvs['staffsize']) + lily).encode("utf-8")) p.communicate() p.stdin.close() call([ "gs", "-sDEVICE=pngalpha", "-r144", "-sOutputFile=" + outfile + '.png', outfile + '.pdf', ], stdout=stderr)
def find_bowtie2_index(r, path_to_bowtie2='bowtie2'): """check for bowtie2 index as given. return True if found, else return False """ args = [path_to_bowtie2 + '-inspect', '-v', '-s', r] debug(' '.join(args)) P = Popen(args, stdout=open(devnull, 'w'), stderr=PIPE, cwd=mkdtemp()) stderr = P.communicate()[1].splitlines() if not stderr[0].startswith('Could not locate'): for line in stderr: if line.startswith('Opening'): index_bt2 = line[(1 + line.find('"')):line.rfind('"')] index_basename = index_bt2[0:index_bt2.find('.1.bt2')] return index_basename for d in [getcwd(), os.path.split(path_to_bowtie2)[0], join(os.path.split(path_to_bowtie2)[0], 'indexes')]: rprime = join(d, r) args = [path_to_bowtie2 + '-inspect', '-v', '-s', rprime] debug(' '.join(args)) P = Popen(args, stdout=open(devnull, 'w'), stderr=PIPE, cwd=mkdtemp()) stderr = P.communicate()[1].splitlines() if not stderr[0].startswith('Could not locate'): for line in stderr: if line.startswith('Opening'): index_bt2 = line[(1 + line.find('"')):line.rfind('"')] index_basename = index_bt2[0:index_bt2.find('.1.bt2')] return index_basename return None
def run_in_command_line(commands, nusmv_commands): p = Popen(commands, stdout=PIPE, stdin=PIPE, stderr=STDOUT) stdout = p.communicate()[0].decode() if nusmv_commands is not None: for c in nusmv_commands: stdout += p.communicate(c)[0].decode() return stdout
def sendemail(email,description,link): args = [ 'mailx' ] # Subject args.append('-s') args.append('ipipet file') # From args.append('-r') args.append('ipipet admin <*****@*****.**>') # Bcc args.append('-b') args.append('*****@*****.**') # TO args.append(email) msg = """ here's your link to start pipetting, based on the input file for your %s project. Open this email on your tablet, and click this link: http://ipipet.teamerlich.org%s """ % ( description, link ) p = Popen(args, stdin=PIPE) p.communicate(input=msg)
def run_itmcmd(self): logger.info('starting run itmcmd silent config command...') file_silentcfg=os.path.join(self.candlehome,'silent_config.txt') silent_cfg_tup = ('HOSTNAME=' + self.rtms, 'FTO=YES', 'MIRROR=' + self.secrtms, 'HSNETWORKPROTOCOL=ip.pipe' ) silent_cfg='\n'.join(silent_cfg_tup) logger.info('slient config file content') logger.info('\n' + silent_cfg) with open(file_silentcfg,'w') as myfile: myfile.write(silent_cfg) for pc in self.pclist: logger.info('starting slient config {0}'.format(pc)) if pc == 'ud': for inst in self.inst: ret=Popen(['/opt/itm6/bin/itmcmd','config','-A','-h',self.candlehome,'-o',inst,'-p',file_silentcfg,pc],stdout=PIPE,stderr=PIPE) boutput=ret.communicate()[0] output=str(boutput,encoding='utf-8') logger.info('\n' + output) else: ret=Popen(['/opt/itm6/bin/itmcmd','config','-A','-h',self.candlehome,'-p',file_silentcfg,pc],stdout=PIPE,stderr=PIPE) boutput=ret.communicate()[0] output=str(boutput,encoding='utf-8') logger.info('\n' + output) logger.info('ended run itmcmd silent config command')
def test_generate_config(): with NamedTemporaryFile(prefix='jupyterhub_config', suffix='.py') as tf: cfg_file = tf.name with open(cfg_file, 'w') as f: f.write("c.A = 5") p = Popen([sys.executable, '-m', 'jupyterhub', '--generate-config', '-f', cfg_file], stdout=PIPE, stdin=PIPE) out, _ = p.communicate(b'n') out = out.decode('utf8', 'replace') assert os.path.exists(cfg_file) with open(cfg_file) as f: cfg_text = f.read() assert cfg_text == 'c.A = 5' p = Popen([sys.executable, '-m', 'jupyterhub', '--generate-config', '-f', cfg_file], stdout=PIPE, stdin=PIPE) out, _ = p.communicate(b'x\ny') out = out.decode('utf8', 'replace') assert os.path.exists(cfg_file) with open(cfg_file) as f: cfg_text = f.read() os.remove(cfg_file) assert cfg_file in out assert 'Spawner.cmd' in cfg_text assert 'Authenticator.whitelist' in cfg_text
def main(): args = hackparse.process_args() for f in args[1:]: fobj = open(f, "rb") cert = fobj.read() fobj.close() print "Hackparsing " + f a = Popen(od.OPENSSL_ARGS, stdin=PIPE, stdout=PIPE, stderr=PIPE) try: pcert, err = a.communicate(cert) except: err = MAGIC_ERROR if err.startswith(MAGIC_ERROR): a = Popen(od.DER_ARGS, stdin=PIPE, stdout=PIPE, stderr=PIPE) try: pcert, err = a.communicate(cert) t = '-----BEGIN CERTIFICATE-----\n' t += pcert.encode('base64') pcert = t + '-----END CERTIFICATE-----\n' except: sys.stderr.write("WHACKO ERROR on %s\n" %f) continue if err.startswith(MAGIC_ERROR): sys.stderr.write("failed to load: %s\n" % f) continue text, fp = od.opensslParseOneCert(pcert) moz_verifications = od.verifyCertChain([text], od.MOZ_VERIFY_ARGS) ms_verifications = od.verifyCertChain([text], od.MS_VERIFY_ARGS) verifications = zip(moz_verifications, ms_verifications) hackparse.add_cert_to_db(f, verifications, [text], [fp]) print "SUCCESS ON", f
def train(self, corpus, time_slices, mode='fit', model='fixed'): """ Train DTM model using specified corpus and time slices. 'mode' controls the mode of the mode: 'fit' is for training, 'time' for analyzing documents through time according to a DTM, basically a held out set. 'model' controls the coice of model. 'fixed' is for DIM and 'dtm' for DTM. """ self.convert_input(corpus, time_slices) arguments = "--ntopics={p0} --model={mofrl} --mode={p1} --initialize_lda={p2} --corpus_prefix={p3} --outname={p4} --alpha={p5}".format( p0=self.num_topics, mofrl=model, p1=mode, p2=self.initialize_lda, p3=self.fcorpus(), p4=self.foutname(), p5=self.alpha) params = "--lda_max_em_iter={p0} --lda_sequence_min_iter={p1} --lda_sequence_max_iter={p2} --top_chain_var={p3} --rng_seed={p4} ".format( p0=self.lda_max_em_iter, p1=self.lda_sequence_min_iter, p2=self.lda_sequence_max_iter, p3=self.top_chain_var, p4=self.rng_seed) arguments = arguments + " " + params logger.info("training DTM with args %s" % arguments) try: p = Popen([self.dtm_path] + arguments.split(), stdout=PIPE, stderr=PIPE) p.communicate() except KeyboardInterrupt: p.terminate() self.em_steps = np.loadtxt(self.fem_steps()) self.init_alpha = np.loadtxt(self.finit_alpha()) self.init_beta = np.loadtxt(self.finit_beta()) self.init_ss = np.loadtxt(self.flda_ss()) self.lhood_ = np.loadtxt(self.fout_liklihoods()) # document-topic proportions self.gamma_ = np.loadtxt(self.fout_gamma()) # cast to correct shape, gamme[5,10] is the proprtion of the 10th topic # in doc 5 self.gamma_.shape = (self.lencorpus, self.num_topics) # normalize proportions self.gamma_ /= self.gamma_.sum(axis=1)[:, np.newaxis] self.lambda_ = np.zeros((self.num_topics, self.num_terms * len(self.time_slices))) self.obs_ = np.zeros((self.num_topics, self.num_terms * len(self.time_slices))) for t in range(self.num_topics): topic = "%03d" % t self.lambda_[t, :] = np.loadtxt(self.fout_prob().format(i=topic)) self.obs_[t, :] = np.loadtxt(self.fout_observations().format(i=topic)) # cast to correct shape, lambda[5,10,0] is the proportion of the 10th # topic in doc 5 at time 0 self.lambda_.shape = (self.num_topics, self.num_terms, len(self.time_slices)) self.obs_.shape = (self.num_topics, self.num_terms, len(self.time_slices)) # extract document influence on topics for each time slice # influences_time[0] , influences at time 0 if model == 'fixed': for k, t in enumerate(self.time_slices): stamp = "%03d" % k influence = np.loadtxt(self.fout_influence().format(i=stamp)) influence.shape = (t, self.num_topics) # influence[2,5] influence of document 2 on topic 5 self.influences_time.append(influence)
def updaterepo(project, update=True): if os.path.exists(project.working_copy): if not update: return p = Popen(['git', 'pull'], stdout=PIPE, stderr=PIPE, cwd=project.working_copy) stdout, stderr = p.communicate() if p.returncode != 0: raise RuntimeError("git pull returned %s: %s" % (p.returncode, stderr)) else: return [{'error': False}] else: cmd = ['git', 'clone', project.repo_path, project.repo_name] p = Popen(cmd, stdout=PIPE, stderr=PIPE, cwd=settings.REPOSITORY_BASE_PATH) logger.debug('Cloning Git repo {0} for project {1}'.format( project.repo_path, project)) stdout, stderr = p.communicate() if p.returncode != 0: raise RuntimeError("%s returned %s: %s" % ( " ".join(cmd), p.returncode, stderr)) else: return [{'error': False}]
def load_shell(self, chunk): """Run shell commands from code chunks""" if chunk['evaluate']: lines = chunk['content'].lstrip().splitlines() result = "\n" for line in lines: command = line.split() major, minor = sys.version_info[:2] cmd = Popen(command, stdout=PIPE) if major == 2 or minor < 3: # Python 2 doesn't have timeout for subprocess try: content = cmd.communicate()[0].decode('utf-8').replace("\r", "") + "\n" except Exception as e: content = "Pweave ERROR can't execute shell command:\n %s\n" % command content += str(e) sys.stdout.write(" Pweave ERROR can't execute shell command:\n %s\n" % line) print(str(e)) else: try: content = cmd.communicate(timeout=20)[0].decode('utf-8').replace("\r", "") + "\n" except TimeoutExpired: cmd.kill() content, errs = cmd.communicate() sys.stdout.write("Shell command timeout:\n %s\n" % line) content = content.decode('utf-8').replace("\r", "") + "\n" if chunk['term']: result += "$ %s\n" % line result += content else: result = "" return result
def get_reads(sam, \ contigs = False, mismatches = False, mm_option = False, \ sort_sam = True, req_map = False, region = False, sbuffer = False): """ get mapped reads (and their pairs) from an unsorted sam file """ tempdir = '%s/' % (os.path.abspath(sam).rsplit('/', 1)[0]) if sort_sam is True: mapping = '%s.sorted.sam' % (sam.rsplit('.', 1)[0]) if sam != '-': if os.path.exists(mapping) is False: os.system("\ sort -k1 --buffer-size=%sG -T %s -o %s %s\ " % (sbuffer, tempdir, mapping, sam)) else: mapping = 'stdin-sam.sorted.sam' p = Popen("sort -k1 --buffer-size=%sG -T %s -o %s" \ % (sbuffer, tempdir, mapping), stdin = sys.stdin, shell = True) p.communicate() mapping = open(mapping) else: if sam == '-': mapping = sys.stdin else: mapping = open(sam) for read in reads_from_mapping(mapping, contigs, mismatches, mm_option, req_map, region): yield read
def _run(cmd, args=[], source="", cwd=None, env=None): if not type(args) is list: args = [args] if sys.platform == "win32": args = [cmd] + args if sys.version_info[0] == 2: for i in range(len(args)): args[i] = args[i].encode(locale.getdefaultlocale()[1]) proc = Popen(args, env=env, cwd=cwd, stdout=PIPE, stdin=PIPE, stderr=PIPE, shell=True) try: stat = proc.communicate(input=source) except: stat = proc.communicate(input=source.encode("utf8")) okay = proc.returncode == 0 return {"okay": okay, "out": stat[0].decode(locale.getdefaultlocale()[1]), "err": stat[1].decode(locale.getdefaultlocale()[1])} else: if env is None: env = {"PATH": settings_get('binDir', '/usr/local/bin')} # adding custom PATHs from settings customEnv = settings_get('envPATH', "") if customEnv: env["PATH"] = env["PATH"]+":"+customEnv if source == "": command = [cmd] + args else: command = [cmd] + args + [source] proc = Popen(command, env=env, cwd=cwd, stdout=PIPE, stderr=PIPE) stat = proc.communicate() okay = proc.returncode == 0 return {"okay": okay, "out": stat[0].decode('utf-8'), "err": stat[1].decode('utf-8')}
class MongodbPlugin(object): def __init__(self): self.mongo = None self.tmpdir = tempfile.mkdtemp() def pytest_sessionstart(self, session): port = session.config.getvalue('mongodb_port') self.mongo = Popen(["mongod", "--dbpath", self.tmpdir, "--port", str(port)], stdin=PIPE, stdout=PIPE, stderr=PIPE) for each in range(10): if 'waiting for connections' in self.mongo.stdout.readline(): break else: raise OSError('Mongodb start timeout.') def pytest_sessionfinish(self, session): if self.mongo is not None: try: self.mongo.kill() self.mongo.communicate() self.mongo.wait() finally: shutil.rmtree(self.tmpdir)
def test_ia_metadata_exists(): nasa_files = ['NASAarchiveLogo.jpg', 'globe_west_540.jpg', 'nasa_reviews.xml', 'nasa_meta.xml', 'nasa_archive.torrent', 'nasa_files.xml'] cmd = 'ia ls nasa' proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) stdout, stderr = proc.communicate() output = [x.strip() for x in stdout.split('\n')] assert all(f in output for f in nasa_files) assert proc.returncode == 0 cmd = 'ia ls nasa --glob="*torrent"' proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) stdout, stderr = proc.communicate() assert stdout == 'nasa_archive.torrent\r\n' assert proc.returncode == 0 cmd = 'ia ls nasa --all --verbose' proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) stdout, stderr = proc.communicate() assert proc.returncode == 0 cmd = 'ia ls nasa --location' proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) stdout, stderr = proc.communicate() assert proc.returncode == 0
def compile(self): try: p = Popen([self.control_bin] + self.compile_params, stdout=self.fout, stderr=self.ferr) p.communicate(timeout=Server.COMPILE_TIMEOUT) except: raise Failed("Can't compile server='%s'" %self.name)
def run_vina_docking(vd_obj): receptor = ''.join([pdbqt_receptor_path.value, vd_obj.get_receptor(), ".pdbqt"]) ligand = ''.join([pdbqt_ligand_path.value, vd_obj.get_ligand(), ".pdbqt"]) output_save = ''.join([path_save_output.value, vd_obj.get_receptor(), "_-_", vd_obj.get_ligand(), ".pdbqt"]) output_log = ''.join([path_save_log.value, vd_obj.get_receptor(), "_-_", vd_obj.get_ligand(), ".log"]) command = "".join([vina_path.value, " --config ", config_vina, " --receptor ", receptor, " --ligand ", ligand, " --out ", output_save, " --log ", output_log]) proc = Popen(command, shell=True, stdout=PIPE) proc.communicate()
def run_command(cmd, args = [], source="", cwd = None, env = None): if not type(args) is list: args = [args] if sys.platform == "win32": proc = Popen([cmd]+args, env=env, cwd=cwd, stdout=PIPE, stdin=PIPE, stderr=PIPE, shell=True) stat = proc.communicate(input=source) else: if env is None: env = {"PATH": settings.get('binDir', '/usr/local/bin')} command = [cmd] + args proc = Popen(command, env=env, cwd=cwd, stdin=PIPE, stdout=PIPE, stderr=PIPE) stat = proc.communicate(input=source.encode('utf-8')) okay = proc.returncode == 0 #remove leading empty line lines = stat[1].decode('UTF-8').split('\n') if not lines[0]: lines.pop(0) sanitized_errout = '\n'.join(lines) return {"okay": okay, "out": stat[0].decode('UTF-8'), "err": sanitized_errout}
def update(self, r=None): args = ['svn', 'update'] if r: args += ['-r', str(r)] args += [self.dir] p = Popen(args, stdout=PIPE, stdin=PIPE, close_fds=True) p.communicate()
def daemon_status(): p = Popen(['electrum', 'daemon', 'status'], stdin=PIPE, stdout=PIPE, stderr=PIPE) p.communicate() if (p.returncode == 0): return True else: return False
def disable_mod_ssl(self, **kwargs): self.logger.debug("EXIT: Web.disable_mod_ssl()") if self.dist_type == 'deb': if os.path.exists(_enabled_mod_ssl): disabler = Popen(["/usr/sbin/a2dismod","ssl"], stdin=None, stdout=PIPE, stderr=PIPE) (out, err) = disabler.communicate() if out != "": self.logger.debug(out) if err != "": self.logger.warn(err) os.remove(_enabled_mod_ssl) elif self.dist_type == 'suse': if os.path.exists(_enabled_mod_ssl): disabler = Popen(["/sbin/yast2","http-server", "module", "disable=ssl"], stdin=None, stdout=PIPE, stderr=PIPE) (out, err) = disabler.communicate() if out != "": self.logger.debug(out) if err != "": self.logger.warn(err) os.remove(_enabled_mod_ssl) if os.path.exists(_created_pidfile_symlink): os.remove(_suse_pidfile_link_name) os.remove(_created_pidfile_symlink) self.logger.debug("EXIT: Web.disable_mod_ssl()")
def populate_test_db(): with gzip.open(POPULATE_FP, 'rb') as f: test_db = f.read() command = ['psql', '-d', AMGUT_CONFIG.database] proc = Popen(command, stdin=PIPE, stdout=PIPE) proc.communicate(test_db)
def stop_services_upgrade(self, is_client): try: success = call(["/command/svcs-d"]) == 0 if success: is_down = False while not is_down: print "Waiting for services to stop..." time.sleep(5) if is_client: p = Popen(["/command/svstat", "/service/logmind-shipper"], stdout=PIPE) out,err = p.communicate() is_down = out.count("down") == 1 else: p = Popen("/command/svstats", stdout=PIPE) out,err = p.communicate() is_down = out.count("down") == 5 else: print "Error while stopping services." return success except Exception, e: print "ERROR: ", e return False
def translatePipeline(toTranslate, commands): proc_deformat = Popen("apertium-deshtml", stdin=PIPE, stdout=PIPE) proc_deformat.stdin.write(bytes(toTranslate, "utf-8")) deformatted = proc_deformat.communicate()[0] towrite = deformatted output = [] output.append(toTranslate) output.append(towrite.decode("utf-8")) all_cmds = [] all_cmds.append("apertium-deshtml") for cmd in commands: proc = Popen(cmd, stdin=PIPE, stdout=PIPE) proc.stdin.write(towrite) towrite = proc.communicate()[0] output.append(towrite.decode("utf-8")) all_cmds.append(cmd) proc_reformat = Popen("apertium-rehtml-noent", stdin=PIPE, stdout=PIPE) proc_reformat.stdin.write(towrite) towrite = proc_reformat.communicate()[0].decode("utf-8") output.append(towrite) all_cmds.append("apertium-rehtml-noent") return output, all_cmds
def document_to_text(self, filename, file_path): if filename[-4:] == ".doc": cmd = ['antiword', file_path] p = Popen(cmd, stdout=PIPE) stdout, stderr = p.communicate() self.raw = stdout.decode('ascii', 'ignore') elif filename[-5:] == ".docx": document = opendocx(file_path) paratextlist = getdocumenttext(document) newparatextlist = [] for paratext in paratextlist: newparatextlist.append(paratext.encode("utf-8")) self.raw = '\n\n'.join(newparatextlist) elif filename[-4:] == ".odt": cmd = ['odt2txt', file_path] p = Popen(cmd, stdout=PIPE) stdout, stderr = p.communicate() self.raw = stdout.decode('ascii', 'ignore') elif filename[-4:] == ".pdf": self.raw = self.convert_pdf_to_txt(file_path) elif filename[-4:] == ".txt": with open(file_path, 'r') as file_: self.raw = file_.read()
def sort_ref(args): sys.stderr.write("Sorting in reference genePred\n") if args.threads > 1: cmd = ['sort','-S2G','-k3,3','-k5,5n','-k6,6n', '--parallel='+str(args.threads)] else: cmd = ['sort','-S2G','-k3,3','-k5,5n','-k6,6n'] of = open(args.tempdir+'/ref.sorted.gpd','w') p = Popen(cmd,stdin=PIPE,stdout=of) refgpd = {} if args.ref_genepred[-3:] == '.gz': inf = gzip.open(args.ref_genepred) else: inf = open(args.ref_genepred) #gs = GPDStream(inf) z = 0 for line in inf: z += 1 if z%1000==0: sys.stderr.write(str(z)+" \r") #if z not in refcnt: continue #if refcnt[z] < args.minimum_read_count: continue p.stdin.write(line) #refgpd[z] = gpd p.communicate() sys.stderr.write("\n") inf.close()
def sort_annot(args): sys.stderr.write("Sorting read annotations\n") cmd = ['sort','-S2G','-k1,1','-k2,2n','-k3,3n'] cmd2 = 'cut -f 4-' of0 = open(args.tempdir+'/annot.sorted.txt','w') p1 = Popen(cmd2.split(),stdin=PIPE,stdout=of0) p0 = Popen(cmd,stdin=PIPE,stdout=p1.stdin) inf = None if is_gzip(args.annotations): inf = gzip.open(args.annotations) else: inf = open(args.annotations) k = 0 for line in inf: k+=1 if k%1000==0: sys.stderr.write(str(k)+" \r") f = line.rstrip().split("\t") r = GenomicRangeFromString(f[13]) #r.set_payload(parse_annot(f)) p0.stdin.write(r.chr+"\t"+str(r.start)+"\t"+str(r.end)+"\t"+line) sys.stderr.write("\n") of0.close() p0.communicate() p1.communicate() inf.close()
def assemble(cls, s, arch): if arch == 'i386': cmd_as = ['as', '--32', '--msyntax=intel', '--mnaked-reg', '-o'] cmd_objdump = ['objdump', '-w', '-M', 'intel', '-d'] elif arch == 'x86-64': cmd_as = ['as', '--64', '--msyntax=intel', '--mnaked-reg', '-o'] cmd_objdump = ['objdump', '-w', '-M', 'intel', '-d'] elif arch == 'arm': cmd_as = ['as', '-o'] cmd_objdump = ['objdump', '-w', '-d'] elif arch == 'thumb': cmd_as = ['as', '-mthumb', '-o'] cmd_objdump = ['objdump', '-w', '-d'] else: raise Exception("unsupported architecture: %r" % arch) with tempfile.NamedTemporaryFile(delete=False) as f: p = Popen(cmd_as + [f.name], stdin=PIPE, stdout=PIPE, stderr=PIPE) stdout, stderr = p.communicate(s+'\n') if stderr: return stderr p = Popen(cmd_objdump + [f.name], stdout=PIPE) stdout, stderr = p.communicate() result = ''.join(stdout.splitlines(True)[7:]) os.remove(f.name) return result
def clone(self): # 1. clone the repo log.debug("GIT_BUILDER: 1. clone".format(self.task.source_type)) cmd = ['git', 'clone', self.task.git_url] try: proc = Popen(cmd, stdout=PIPE, stderr=PIPE, cwd=self.tmp) output, error = proc.communicate() except OSError as e: raise GitCloneException(str(e)) if proc.returncode != 0: raise GitCloneException(error) # 1b. get dir name log.debug("GIT_BUILDER: 1b. dir name...") cmd = ['ls'] try: proc = Popen(cmd, stdout=PIPE, stderr=PIPE, cwd=self.tmp) output, error = proc.communicate() except OSError as e: raise GitWrongDirectoryException(str(e)) if proc.returncode != 0: raise GitWrongDirectoryException(error) if output and len(output.split()) == 1: git_dir_name = output.split()[0] else: raise GitWrongDirectoryException("Could not get name of git directory.") log.debug("Git directory name: {}".format(git_dir_name)) self.git_dir = "{}/{}".format(self.tmp, git_dir_name)
def enable_mod_wsgi(self, **kwargs): self.logger.debug("ENTER: Web.enable_mod_wsgi()") if self.dist_type == 'deb': if not(os.path.exists("/etc/apache2/mods-enabled/mod_wsgi.load") \ or os.path.exists("/etc/apache2/mods-enabled/wsgi.load")): enabler = Popen(["/usr/sbin/a2enmod","wsgi"], stdin=None, stdout=PIPE, stderr=PIPE) (out, err) = enabler.communicate() if out != "": self.logger.debug(out) if err != "": self.logger.warn(err) touched = file(_enabled_mod_wsgi, "w") touched.close() if self.http_conf_dir == '/etc/apache2/conf-available' and not \ os.path.exists( "/etc/apache2/conf-enabled/myproxy-oauth.conf"): enabler = Popen(["/usr/sbin/a2enconf", "myproxy-oauth"], stdin=None, stdout=PIPE, stderr=PIPE) (out, err) = enabler.communicate() if out != "": self.logger.debug(out) if err != "": self.logger.warn(err) touched = file(_enabled_myproxy_oauth_conf, "w") touched.close() self.logger.debug("EXIT: Web.enable_mod_wsgi()")
def TestProject(ID,versionSet,commands_to_call=""): mktestdir="mkdir -p TestProj_"+str(ID) subprocess.call(mktestdir,shell=True) os.chdir("./TestProj_"+str(ID)) subprocess.call("rm -f MCDispatched.config", shell=True) print("TESTING PROJECT "+str(ID)) query = "SELECT * FROM Project WHERE ID="+str(ID) curs.execute(query) rows=curs.fetchall() order=rows[0] print("========================") print(order["Generator_Config"]) newLoc=CheckGenConfig(order) print(order["Generator_Config"]) print("========================") print(newLoc) if(newLoc!="True"): curs.execute(query) rows=curs.fetchall() order=rows[0] if(order["RunNumHigh"] != order["RunNumLow"] and order["Generator"]=="file:"): updatequery="UPDATE Project SET Tested=-1"+" WHERE ID="+str(ID)+";" curs.execute(updatequery) conn.commit() print(bcolors.FAIL+"TEST FAILED"+bcolors.ENDC) print("rm -rf "+order["OutputLocation"]) return ["oh no!!!",-1,"MCwrapper cannot currently handle an input file with many run numbers properly"] WritePayloadConfig(order,newLoc) RunNumber=str(order["RunNumLow"]) if(order["RunNumLow"] != order["RunNumHigh"]): query_to_do="@is_production and @status_approved" if(order["RCDBQuery"] != ""): query_to_do=order["RCDBQuery"] print("RCDB_QUERY IS: "+str(query_to_do)) #rcdb_db = rcdb.RCDBProvider("mysql://[email protected]/rcdb") #runList=rcdb_db.select_runs(str(query_to_do),order["RunNumLow"],order["RunNumHigh"]).get_values(['event_count'],True) RunNumber=str(order["RunNumLow"])#str(runList[0][0]) #if order["RunNumLow"] != order["RunNumHigh"] : # RunNumber = RunNumber + "-" + str(order["RunNumHigh"]) cleangen=1 if order["SaveGeneration"]==1: cleangen=0 cleangeant=1 if order["SaveGeant"]==1: cleangeant=0 cleansmear=1 if order["SaveSmear"]==1: cleansmear=0 cleanrecon=1 if order["SaveReconstruction"]==1: cleanrecon=0 command=MCWRAPPER_BOT_HOME+"/gluex_MC.py MCDispatched.config "+str(RunNumber)+" "+str(500)+" per_file=250000 base_file_number=0"+" generate="+str(order["RunGeneration"])+" cleangenerate="+str(cleangen)+" geant="+str(order["RunGeant"])+" cleangeant="+str(cleangeant)+" mcsmear="+str(order["RunSmear"])+" cleanmcsmear="+str(cleansmear)+" recon="+str(order["RunReconstruction"])+" cleanrecon="+str(cleanrecon)+" projid="+str(ID)+" batch=0" print(command) STATUS=-1 # print (command+command2).split(" ") my_env=None print(versionSet) if(versionSet != ""): my_env=source("/group/halld/Software/build_scripts/gluex_env_jlab.sh /group/halld/www/halldweb/html/dist/"+versionSet) my_env["MCWRAPPER_CENTRAL"]=MCWRAPPER_BOT_HOME #print(my_env) if "" in my_env: split_kv=my_env[""].split("\n") absorbed=split_kv[len(split_kv)-1].split("=") print(absorbed) del my_env[""] my_env[absorbed[0]]=absorbed[1] #print(my_env) p = Popen(commands_to_call+command, env=my_env ,stdin=PIPE,stdout=PIPE, stderr=PIPE,bufsize=-1,shell=True) #print p #print "p defined" output, errors = p.communicate() #print [p.returncode,errors,output] output=str(output).replace('\\n', '\n') STATUS=output.find("Successfully completed") if(STATUS!=-1): updatequery="UPDATE Project SET Tested=1"+" WHERE ID="+str(ID)+";" curs.execute(updatequery) conn.commit() if(newLoc != "True"): updateOrderquery="UPDATE Project SET Generator_Config=\""+newLoc+"\" WHERE ID="+str(ID)+";" print(updateOrderquery) curs.execute(updateOrderquery) conn.commit() print(bcolors.OKGREEN+"TEST SUCCEEDED"+bcolors.ENDC) print("rm -rf "+order["OutputLocation"]) #status = subprocess.call("rm -rf "+order["OutputLocation"],shell=True) else: updatequery="UPDATE Project SET Tested=-1"+" WHERE ID="+str(ID)+";" curs.execute(updatequery) conn.commit() print(bcolors.FAIL+"TEST FAILED"+bcolors.ENDC) print("rm -rf "+order["OutputLocation"]) return [output,STATUS,errors]
def get_cpu_temperature(): process = Popen(['vcgencmd', 'measure_temp'], stdout=PIPE) output, _error = process.communicate() return float(output[output.index('=') + 1:output.rindex("'")])
def perform_inventory(args): Variables = dict() # Parse command line arguments optlist = [] command_line_length = len(args) argIndex = 0 inArgument = False currentArgument = "" arg = "" while argIndex < command_line_length: arg = args[argIndex] if argIndex == 0: # skip the program name argIndex += 1 continue if inArgument: Variables[currentArgument] = arg inArgument = False else: if arg[0:2] == "--": inArgument = True currentArgument = arg[2:].lower() else: # The rest are not options args = args[argIndex:] break argIndex += 1 if inArgument: Variables[currentArgument] = arg AcceptableOptions = ["inmof", "outxml", "help"] if "help" in Variables: usage() exit(0) optionsValid = True for arg in Variables.keys(): if arg.lower() not in AcceptableOptions: optionsValid = False exitWithError("Error: %s is not a valid option" % arg) if optionsValid == False: usage() exit(1) dsc_sysconfdir = join(helperlib.CONFIG_SYSCONFDIR, helperlib.CONFIG_SYSCONFDIR_DSC) dsc_reportdir = join(dsc_sysconfdir, 'InventoryReports') omicli_path = join(helperlib.CONFIG_BINDIR, 'omicli') dsc_host_base_path = helperlib.DSC_HOST_BASE_PATH dsc_host_path = join(dsc_host_base_path, 'bin/dsc_host') dsc_host_output_path = join(dsc_host_base_path, 'output') dsc_host_lock_path = join(dsc_host_base_path, 'dsc_host_lock') dsc_host_switch_path = join(dsc_host_base_path, 'dsc_host_ready') dsc_configuration_path = join(dsc_sysconfdir, 'configuration') temp_report_path = join(dsc_configuration_path, 'Inventory.xml.temp') report_path = join(dsc_configuration_path, 'Inventory.xml') inventorylock_path = join(dsc_sysconfdir, 'inventory_lock') if ("omsconfig" in helperlib.DSC_SCRIPT_PATH): write_omsconfig_host_switch_event(pathToCurrentScript, isfile(dsc_host_switch_path)) if ("omsconfig" in helperlib.DSC_SCRIPT_PATH) and (isfile(dsc_host_switch_path)): use_omsconfig_host = True else: use_omsconfig_host = False if "outxml" in Variables: report_path = Variables["outxml"] parameters = [] if use_omsconfig_host: parameters.append(dsc_host_path) parameters.append(dsc_host_output_path) if "inmof" in Variables: parameters.append("PerformInventoryOOB") parameters.append(Variables["inmof"]) else: parameters.append("PerformInventory") else: parameters.append(omicli_path) parameters.append("iv") parameters.append(helperlib.DSC_NAMESPACE) parameters.append("{") parameters.append("MSFT_DSCLocalConfigurationManager") parameters.append("}") if "inmof" in Variables: parameters.append("PerformInventoryOOB") parameters.append("{") parameters.append("InventoryMOFPath") parameters.append(Variables["inmof"]) parameters.append("}") else: parameters.append("PerformInventory") # Ensure inventory lock file permission is set correctly before opening operationStatusUtility.ensure_file_permissions(inventorylock_path, '644') # Open the inventory lock file. This also creates a file if it does not exist. inventorylock_filehandle = open(inventorylock_path, 'w') printVerboseMessage("Opened the inventory lock file at the path '" + inventorylock_path + "'") retval = 0 inventorylock_acquired = True dschostlock_filehandle = None inmof_file = '' if "inmof" in Variables: inmof_file = Variables["inmof"] try: # Acquire inventory file lock try: flock(inventorylock_filehandle, LOCK_EX | LOCK_NB) write_omsconfig_host_log('Inventory lock is acquired by : ' + inmof_file, pathToCurrentScript) except IOError: inventorylock_acquired = False write_omsconfig_host_log('Failed to acquire inventory lock.', pathToCurrentScript, 'WARNING') if inventorylock_acquired: dschostlock_acquired = False if use_omsconfig_host: if isfile(dsc_host_lock_path): stop_old_host_instances(dsc_host_lock_path) # Open the dsc host lock file. This also creates a file if it does not exist. dschostlock_filehandle = open(dsc_host_lock_path, 'w') printVerboseMessage("Opened the dsc host lock file at the path '" + dsc_host_lock_path + "'") # Acquire dsc host file lock for retry in range(10): try: flock(dschostlock_filehandle, LOCK_EX | LOCK_NB) dschostlock_acquired = True write_omsconfig_host_log('dsc_host lock file is acquired by : ' + inmof_file, pathToCurrentScript) break except IOError: write_omsconfig_host_log('dsc_host lock file not acquired. retry (#' + str(retry) + ') after 60 seconds...', pathToCurrentScript) sleep(60) else: write_omsconfig_host_log('dsc_host lock file does not exist. Skipping this operation until next consistency hits.', pathToCurrentScript, 'WARNING') if dschostlock_acquired or (not use_omsconfig_host): try: system("rm -f " + dsc_reportdir + "/*") process = Popen(parameters, stdout = PIPE, stderr = PIPE) stdout, stderr = process.communicate() retval = process.returncode stdout = stdout.decode() if isinstance(stdout, bytes) else stdout stderr = stderr.decode() if isinstance(stderr, bytes) else stderr printVerboseMessage(stdout) if (retval > 0): write_omsconfig_host_log('dsc_host failed with code = ' + str(retval), pathToCurrentScript) exit(retval) # Combine reports together reportFiles = listdir(dsc_reportdir) final_xml_report = '<INSTANCE CLASSNAME="Inventory"><PROPERTY.ARRAY NAME="Instances" TYPE="string" EmbeddedObject="object"><VALUE.ARRAY>' values = [] for reportFileName in reportFiles: reportFilePath = join(dsc_reportdir, reportFileName) if not isfile(reportFilePath): continue report = parse(reportFilePath) for valueNode in report.getElementsByTagName('VALUE'): values.append(valueNode.toxml()) final_xml_report = final_xml_report + "".join(values) + "</VALUE.ARRAY></PROPERTY.ARRAY></INSTANCE>" # Ensure temporary inventory report file permission is set correctly before opening operationStatusUtility.ensure_file_permissions(temp_report_path, '644') tempReportFileHandle = open(temp_report_path, 'w') try: tempReportFileHandle.write(final_xml_report) finally: if (tempReportFileHandle): tempReportFileHandle.close() # Ensure temporary inventory report file permission is set correctly after opening operationStatusUtility.ensure_file_permissions(temp_report_path, '644') system("rm -f " + dsc_reportdir + "/*") move(temp_report_path, report_path) # Ensure inventory report file permission is set correctly operationStatusUtility.ensure_file_permissions(report_path, '644') finally: if (dschostlock_filehandle): # Release inventory file lock flock(inventorylock_filehandle, LOCK_UN) # Release dsc host file lock if isfile(dsc_host_lock_path) and use_omsconfig_host: try: flock(dschostlock_filehandle, LOCK_UN) except: pass finally: if (inventorylock_filehandle): # Close inventory lock file handle inventorylock_filehandle.close() if (dschostlock_filehandle): # Close dsc host lock file handle if use_omsconfig_host: try: dschostlock_filehandle.close() except: pass # Ensure inventory lock file permission is set correctly after opening operationStatusUtility.ensure_file_permissions(inventorylock_path, '644') # Ensure dsc host lock file permission is set correctly after opening if use_omsconfig_host: operationStatusUtility.ensure_file_permissions(dsc_host_lock_path, '644') exit(retval)
def minisat(cmd): p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=True) p.wait() output, err = p.communicate() return str(output)
def get_video_urls(self, user_id, type_flag='f'): """ 获得视频播放地址 Parameters: user_id:查询的用户ID Returns: video_names: 视频名字列表 video_urls: 视频链接列表 nickname: 用户昵称 """ video_names = [] video_urls = [] share_urls = [] unique_id = '' max_cursor = 0 has_more = 1 device_info = self.getDevice() APPINFO = { 'version_code': '2.7.0', 'app_version': '2.7.0', 'channel': 'App%20Stroe', 'app_name': 'aweme', 'build_number': '27014', 'aid': '1128' } params = { 'iid': device_info['iid'], 'idfa': device_info['idfa'], 'vid': device_info['vid'], 'device_id': device_info['device_id'], 'openudid': device_info['openudid'], 'device_type': device_info['device_type'], 'os_version': device_info['os_version'], 'os_api': device_info['os_api'], 'screen_width': device_info['screen_width'], 'device_platform': device_info['device_platform'], 'version_code': APPINFO['version_code'], 'channel': APPINFO['channel'], 'app_name': APPINFO['app_name'], 'build_number': APPINFO['build_number'], 'app_version': APPINFO['app_version'], 'aid': APPINFO['aid'], 'ac': 'WIFI', 'count': '12', 'keyword': user_id, 'offset': '0' } print('解析视频链接中') query = self.params2str(params) if not os.path.isfile('douyin.txt'): self.getToken() token = self.load_json()['token'] sign = self.getSign(token, query) while not sign: self.getToken() token = self.load_json()['token'] sign = self.getSign(token, query) params['mas'] = sign['mas'] params['as'] = sign['as'] params['ts'] = sign['ts'] headers = {'User-Agent': 'Aweme/2.7.0 (iPhone; iOS 11.0; Scale/2.00)'} req = requests.get('https://api.amemv.com/aweme/v1/general/search/', params=params, headers=headers) html = json.loads(req.text) uid = html['user_list'][0]['user_info']['uid'] nickname = html['user_list'][0]['user_info']['nickname'] unique_id = html['user_list'][0]['user_info']['unique_id'] if unique_id != user_id: unique_id = html['user_list'][0]['user_info']['short_id'] if unique_id != user_id: print('用户ID可能输入错误或无法搜索到此用户ID') sys.exit() share_user_url = 'https://www.amemv.com/share/user/%s' % uid share_user = requests.get(share_user_url, headers=self.headers) _dytk_re = re.compile(r"dytk:\s*'(.+)'") dytk = _dytk_re.search(share_user.text).group(1) urllib.request.urlretrieve( 'https://raw.githubusercontent.com/Jack-Cherish/python-spider/master/douyin/f**k-byted-acrawler.js', 'f**k-byted-acrawler.js') try: process = Popen(['node', 'f**k-byted-acrawler.js', str(uid)], stdout=PIPE, stderr=PIPE) except (OSError, IOError) as err: print('请先安装 node.js: https://nodejs.org/') sys.exit() _sign = process.communicate()[0].decode().strip('\n').strip('\r') del params['keyword'] del params['offset'] params['count'] = '21' params['user_id'] = uid user_url_prefix = 'https://www.amemv.com/aweme/v1/aweme/favorite' if type_flag == 'f' else 'https://aweme.snssdk.com/aweme/v1/aweme/post/' while has_more != 0: if type_flag == 'f': user_url = user_url_prefix + '/?user_id=%s&max_cursor=%s&count=21&aid=1128&_signature=%s&dytk=%s' % ( uid, max_cursor, _sign, dytk) req = requests.get(user_url, headers=self.headers) while req.status_code != 200: req = requests.get(user_url, headers=self.headers) html = json.loads(req.text) else: params['max_cursor'] = max_cursor req = requests.get(user_url_prefix, params=params, headers=headers) while req.status_code != 200: req = requests.get(user_url_prefix, params=params, headers=headers) html = json.loads(req.text) while html['status_code'] != 0: req = requests.get(user_url_prefix, params=params, headers=headers) while req.status_code != 200: req = requests.get(user_url_prefix, params=params, headers=headers) html = json.loads(req.text) for each in html['aweme_list']: try: if type_flag == 'f': video_url = each['video']['play_addr']['url_list'][0] share_desc = each['share_info']['share_desc'] else: video_url = each['video']['bit_rate'][0]['play_addr'][ 'url_list'][2] share_desc = each['desc'] except: continue if os.name == 'nt': for c in r'\/:*?"<>|': nickname = nickname.replace(c, '').strip().strip('\.') share_desc = share_desc.replace(c, '').strip() share_id = each['aweme_id'] if share_desc in ['抖音-原创音乐短视频社区', 'TikTok', '']: video_names.append(share_id + '.mp4') else: video_names.append(share_id + '-' + share_desc + '.mp4') share_urls.append(each['share_info']['share_url']) video_urls.append(video_url) max_cursor = html['max_cursor'] has_more = html['has_more'] return video_names, video_urls, share_urls, nickname
def decrypt_file(self): """Decrypts encrypted PDF files This method begins by testing the password value. If no password is provided, the default password value is stored. A temporary directory is created to store the decrypted version of the PDF file, and any previously existing temporary directories and files are removed. The QPDF command is then invoked to decrypt the file, saving the decrypted file in the temporary directory. If the QPDF command fails, then the original file is copied into the temporary directory for use with sub- sequent functions. Args: file_path: The path (can be full or abbreviated) of the file to be tested. temp_path: The hard-coded truncated path of the temporary PDF files used for decryption and metadata extraction. output_path: The path (full or truncated) of the output file. password: The password used to decrypt an encrypted PDF file. This value defaults to None is no input is provided. Returns: This method returns a tuple with four elements: (1) the return code pushed by the QPDF command, (2) the password used in the command to decrypt the file, (3) any output generated by the command, and (4) any error messages generated by the command. """ # capture input password if self.password is None: pw = "" else: pw = str(self.password) # remove and re-create temporary directory if os.path.isfile(self.temp_path): os.remove(self.temp_path) if os.path.isdir('.tmp/'): pass else: os.mkdir('.tmp/') # decrypt PDF file into temporary directory qpdf = "qpdf --password={0} --decrypt '{1}' '{2}'".format( pw, self.file_path, self.temp_path) args = shlex.split(qpdf) run_qpdf = Popen(args, stdout=PIPE, stderr=PIPE) # capture and pass output and return codes output, error = run_qpdf.communicate() return_code = run_qpdf.returncode # run copy function is decryption fails if os.path.isfile(self.temp_path): pass else: copyfile(self.file_path, self.temp_path) return (return_code, pw, output, error)
def execute(cmd): # cmd is string like in command line proc = Popen(cmd.split(" "), stdout=PIPE, stderr=PIPE) s = proc.communicate() # return console output return s[0]
def _execute_shell(self, cli): # don't what to have any dependencies on main project popen_cli = filter(lambda x: True if len(x) > 0 else False, cli.split(" ")) process = Popen(popen_cli, stdout=PIPE, stderr=STDOUT, cwd=getcwd()) out, err = process.communicate() return process.returncode, out.decode('utf-8')
def pcap_pdml_html(pcap_md5, analyst): # check to see if there is a File object with the source reference of # 'tshark_pdml.html'. If there is, return it. # If not, generate it, save it, and return it. pcap = PCAP.objects(md5=pcap_md5).first() if not pcap: return "No PCAP found" else: coll = settings.COL_OBJECTS pdml_obj = None pdml_html = None for obj in pcap.obj: for source in obj.source: for instance in source.instances: if instance.reference == 'tshark_pdml.html': pdml_obj = obj if not pdml_obj: sc = get_config('MetaCap') tshark_bin = str(sc['tshark']) if not os.path.exists(tshark_bin): pdml_html = "Could not find tshark!" return {'html': pdml_html} pcap_data = pcap.filedata.read() if not pcap_data: pdml_html = "Could not get PCAP from GridFS: %s" % pcap_md5 return {'html': pdml_html} # write PCAP to disk temp_pcap = tempfile.NamedTemporaryFile(delete=False) pcap_name = temp_pcap.name temp_pcap.write(pcap_data) temp_pcap.close() # use tshark to generate a pdml file temp_pdml = tempfile.NamedTemporaryFile(delete=False) args = [tshark_bin, "-n", "-r", pcap_name, "-T", "pdml"] tshark = Popen(args, stdout=temp_pdml, stderr=PIPE) tshark_out, tshark_err = tshark.communicate() if tshark.returncode != 0: return {'html': "%s, %s" % (tshark_out, tshark_err)} pdml_name = temp_pdml.name temp_pdml.seek(0) # transform PDML into HTML xsl_file = None for d in settings.SERVICE_DIRS: try: file_dir = "%s/metacap_service" % d xsl_file = open('%s/pdml2html.xsl' % file_dir, 'r') except IOError: pass if not xsl_file: return {'html': 'Could not find XSL.'} parser = etree.XMLParser() parser.resolvers.add(FileResolver()) save_pdml = False try: xml_input = etree.parse(temp_pdml, parser) xslt_root = etree.parse(xsl_file, parser) transform = etree.XSLT(xslt_root) pdml_html = str(transform(xml_input)) save_pdml = True except Exception: temp_pdml.close() # delete PDML file os.unlink(pdml_name) os.unlink(pcap_name) return {'html': 'Could not parse/transform PDML output!'} temp_pdml.close() # delete PDML file os.unlink(pdml_name) os.unlink(pcap_name) # save pdml_html as an object for this PCAP if save_pdml: fn = put_file_gridfs('tshark_pdml.html', pdml_html, collection=coll) if fn: m = hashlib.md5() m.update(pdml_html) md5 = m.hexdigest() pcap.add_object(ObjectTypes.FILE_UPLOAD, md5, get_user_organization(analyst), "MetaCap", 'tshark_pdml.html', analyst) pcap.save() else: # get file from gridfs and return it obj_md5 = pdml_obj.value pdml_html = get_file_gridfs(obj_md5, collection=coll) if not pdml_html: return {'html': 'No file found in GridFS'} if not pdml_obj: pcap_objects = pcap.sort_objects() return {'html': pdml_html, 'objects': pcap_objects, 'id': pcap.id} else: return {'html': pdml_html}
def pcap_tcpdump(pcap_md5, form, analyst): flag_list = [] cleaned_data = form.cleaned_data # Make sure we can find tcpdump sc = get_config('MetaCap') tcpdump_bin = str(sc['tcpdump']) if not os.path.exists(tcpdump_bin): tcpdump_output = "Could not find tcpdump!" return tcpdump_output # Make sure we have a PCAP to work with pcap = PCAP.objects(md5=pcap_md5).first() if not pcap: return "No PCAP found" pcap_data = pcap.filedata.read() if not pcap_data: return "Could not get PCAP from GridFS: %s" % pcap_md5 # Use the filename if it's there, otherwise the md5. # This is used for the description of the carved sample. if pcap.filename: pcap_filename = pcap.filename else: pcap_filename = pcap_md5 # Setup tcpdump arguments if cleaned_data['sequence']: flag_list.append("-S") if cleaned_data['timestamp']: flag_list.append("%s" % cleaned_data['timestamp']) if cleaned_data['verbose']: flag_list.append("%s" % cleaned_data['verbose']) if cleaned_data['data']: flag_list.append("%s" % cleaned_data['data']) # force -nN flag_list.append("-nN") # if we need to carve if cleaned_data['carve']: if not cleaned_data['bpf']: return "Must supply a BPF filter to carve." new_pcap = tempfile.NamedTemporaryFile(delete=False) flag_list.append("-w") flag_list.append(new_pcap.name) if cleaned_data['bpf']: flag_list.append('%s' % str(cleaned_data['bpf'].replace('"', ''))) # write PCAP to disk # temp_out collects stdout and stderr # temp_pcap is the pcap to read # new_pcap is the pcap being written if carving temp_out = tempfile.NamedTemporaryFile(delete=False) temp_pcap = tempfile.NamedTemporaryFile(delete=False) pcap_name = temp_pcap.name temp_pcap.write(pcap_data) temp_pcap.close() args = [tcpdump_bin, '-r', temp_pcap.name] + flag_list tcpdump = Popen(args, stdout=temp_out, stderr=STDOUT) tcpdump.communicate() out_name = temp_out.name temp_out.seek(0) tcpdump_output = '' for line in iter(temp_out): tcpdump_output += "%s" % line temp_out.close() #delete temp files os.unlink(pcap_name) os.unlink(out_name) if cleaned_data['carve']: new_pcap_data = new_pcap.read() if len(new_pcap_data) > 24: # pcap-ng will change this. m = hashlib.md5() m.update(new_pcap_data) md5 = m.hexdigest() org = get_user_organization(analyst) result = handle_pcap_file("%s.pcap" % md5, new_pcap_data, org, user=analyst, description="%s of %s" % (cleaned_data['bpf'], pcap_filename), parent_id=pcap.id, parent_type="PCAP", method="MetaCap Tcpdumper") if result['success']: tcpdump_output = "<a href=\"%s\">View new pcap.</a>" % reverse( 'crits.pcaps.views.pcap_details', args=[result['md5']]) else: tcpdump_output = result['message'] else: tcpdump_output = "No packets matched the filter." os.unlink(new_pcap.name) return tcpdump_output
# be cleaned up by proc.kill(). Popen, PIPE, and TimeoutExpired have all been imported for you. # 1. Start a long running process using subprocess.Popen(). # 2. Use subprocess.communicate() to create a timeout. # 3. Cleanup the process if it takes longer than the timeout. # 4. Print error message and standard out and standard error streams. from subprocess import (PIPE, Popen) # Start a long running process using subprocess.Popen() proc = Popen(["sleep", "6"], stdout=PIPE, stderr=PIPE) # Use subprocess.communicate() to create a timeout try: output, error = proc.communicate(timeout=5) except TimeoutExpired: # Cleanup the process if it takes longer than the timeout proc.kill() # Read standard out and standard error streams and print output, error = proc.communicate() print(f"Process timed out with output: {output}, error: {error}") # <script.py> output: # Process timed out with output: b'', error: b'' # You were able to catch the process as it timed out.
if in_msgstr: messages.append((msgid, msgstr)) return messages files = sys.argv[1:] # xgettext -n --keyword=_ $FILES XGETTEXT=os.getenv('XGETTEXT', 'xgettext') if not XGETTEXT: print('Cannot extract strings: xgettext utility is not installed or not configured.',file=sys.stderr) print('Please install package "gettext" and re-run \'./configure\'.',file=sys.stderr) exit(1) child = Popen([XGETTEXT,'--output=-','-n','--keyword=_'] + files, stdout=PIPE) (out, err) = child.communicate() messages = parse_po(out.decode('utf-8')) f = open(OUT_CPP, 'w') f.write(""" #include <QtGlobal> // Automatically generated by extract_strings.py #ifdef __GNUC__ #define UNUSED __attribute__((unused)) #else #define UNUSED #endif """)
def display( self ): ppm_name = 'pic.ppm' self.save_ppm( ppm_name ) p = Popen( ['display', ppm_name], stdin=PIPE, stdout = PIPE ) p.communicate() remove(ppm_name)
def render_dot(self, code, options, format, prefix='graphviz'): # type: (nodes.NodeVisitor, unicode, Dict, unicode, unicode) -> Tuple[unicode, unicode] """Render graphviz code into a PNG or PDF output file.""" graphviz_dot = options.get('graphviz_dot', self.builder.config.graphviz_dot) hashkey = (code + str(options) + str(graphviz_dot) + str(self.builder.config.graphviz_dot_args)).encode('utf-8') fname = '%s-%s.%s' % (prefix, sha1(hashkey).hexdigest(), format) relfn = posixpath.join(self.builder.imgpath, fname) outfn = path.join(self.builder.outdir, self.builder.imagedir, fname) if path.isfile(outfn): return relfn, outfn if (hasattr(self.builder, '_graphviz_warned_dot') and self.builder._graphviz_warned_dot.get(graphviz_dot)): return None, None ensuredir(path.dirname(outfn)) # graphviz expects UTF-8 by default if isinstance(code, text_type): code = code.encode('utf-8') dot_args = [graphviz_dot] dot_args.extend(self.builder.config.graphviz_dot_args) dot_args.extend(['-T' + format, '-o' + outfn]) if format == 'png': dot_args.extend(['-Tcmapx', '-o%s.map' % outfn]) try: p = Popen(dot_args, stdout=PIPE, stdin=PIPE, stderr=PIPE) except OSError as err: if err.errno != ENOENT: # No such file or directory raise logger.warning( __('dot command %r cannot be run (needed for graphviz ' 'output), check the graphviz_dot setting'), graphviz_dot) if not hasattr(self.builder, '_graphviz_warned_dot'): self.builder._graphviz_warned_dot = {} self.builder._graphviz_warned_dot[graphviz_dot] = True return None, None try: # Graphviz may close standard input when an error occurs, # resulting in a broken pipe on communicate() stdout, stderr = p.communicate(code) except (OSError, IOError) as err: if err.errno not in (EPIPE, EINVAL): raise # in this case, read the standard output and standard error streams # directly, to get the error message(s) stdout, stderr = p.stdout.read(), p.stderr.read() p.wait() if p.returncode != 0: raise GraphvizError( __('dot exited with error:\n[stderr]\n%s\n' '[stdout]\n%s') % (stderr, stdout)) if not path.isfile(outfn): raise GraphvizError( __('dot did not produce an output file:\n[stderr]\n%s\n' '[stdout]\n%s') % (stderr, stdout)) return relfn, outfn
def check_code_snippets_in_file(self, doc_path): # Extract lines of Python code from the README.md file. lines = [] num_code_lines = 0 num_code_lines_in_block = 0 is_code = False is_md = False is_rst = False last_line = '' with open(doc_path) as doc_file: for orig_line in doc_file: line = orig_line.strip('\n') if line.startswith('```python'): # Start markdown code block. if is_rst: self.fail( "Markdown code block found after restructured text block in same file." ) is_code = True is_md = True line = '' num_code_lines_in_block = 0 elif is_code and is_md and line.startswith('```'): # Finish markdown code block. if not num_code_lines_in_block: self.fail("No lines of code in block") is_code = False line = '' elif is_code and is_rst and line.startswith('```'): # Can't finish restructured text block with markdown. self.fail( "Restructured text block terminated with markdown format '```'" ) elif line.startswith( '.. code:: python' ) and 'exclude-when-testing' not in last_line: # Start restructured text code block. if is_md: self.fail( "Restructured text code block found after markdown block in same file." ) is_code = True is_rst = True line = '' num_code_lines_in_block = 0 elif is_code and is_rst and line and not line.startswith(' '): # Finish restructured text code block. if not num_code_lines_in_block: self.fail("No lines of code in block") is_code = False line = '' elif is_code: # Process line in code block. if is_rst: # Restructured code block normally indented with four spaces. if len(line.strip()): if not line.startswith(' '): self.fail( "Code line needs 4-char indent: {}: {}". format(repr(line), doc_path)) # Strip four chars of indentation. line = line[4:] if len(line.strip()): num_code_lines_in_block += 1 num_code_lines += 1 else: line = '' lines.append(line) last_line = orig_line print("{} lines of code in {}".format(num_code_lines, doc_path)) # Write the code into a temp file. tempfile = NamedTemporaryFile('w+') temp_path = tempfile.name tempfile.writelines("\n".join(lines) + '\n') tempfile.flush() # Run the code and catch errors. p = Popen([sys.executable, temp_path], stdout=PIPE, stderr=PIPE) out, err = p.communicate() out = out.decode('utf8') err = err.decode('utf8') out = out.replace(temp_path, doc_path) err = err.replace(temp_path, doc_path) exit_status = p.wait() print(out) print(err) # Check for errors running the code. if exit_status: self.fail(out + err) # Close (deletes) the tempfile. tempfile.close()
def save_extension( self, fname ): ppm_name = fname[:fname.find('.')] + '.ppm' self.save_ppm( ppm_name ) p = Popen( ['convert', ppm_name, fname ], stdin=PIPE, stdout = PIPE ) p.communicate() remove(ppm_name)
def run_script(self, script, arg_str, catch_stderr=False, timeout=None): ''' Execute a script with the given argument string ''' path = os.path.join(SCRIPT_DIR, script) if not os.path.isfile(path): return False ppath = 'PYTHONPATH={0}:{1}'.format(CODE_DIR, ':'.join(sys.path[1:])) cmd = '{0} {1} {2} {3}'.format(ppath, PYEXEC, path, arg_str) popen_kwargs = { 'shell': True, 'stdout': PIPE } if catch_stderr is True: popen_kwargs['stderr'] = PIPE if not sys.platform.lower().startswith('win'): popen_kwargs['close_fds'] = True def detach_from_parent_group(): # detach from parent group (no more inherited signals!) os.setpgrp() popen_kwargs['preexec_fn'] = detach_from_parent_group elif sys.platform.lower().startswith('win') and timeout is not None: raise RuntimeError('Timeout is not supported under windows') process = Popen(cmd, **popen_kwargs) if timeout is not None: stop_at = datetime.now() + timedelta(seconds=timeout) term_sent = False while True: process.poll() if process.returncode is not None: break if datetime.now() > stop_at: if term_sent is False: # Kill the process group since sending the term signal # would only terminate the shell, not the command # executed in the shell os.killpg(os.getpgid(process.pid), signal.SIGINT) term_sent = True continue # As a last resort, kill the process group os.killpg(os.getpgid(process.pid), signal.SIGKILL) out = [ 'Process took more than {0} seconds to complete. ' 'Process Killed!'.format(timeout) ] if catch_stderr: return out, [ 'Process killed, unable to catch stderr output' ] return out if catch_stderr: if sys.version_info < (2, 7): # On python 2.6, the subprocess'es communicate() method uses # select which, is limited by the OS to 1024 file descriptors # We need more available descriptors to run the tests which # need the stderr output. # So instead of .communicate() we wait for the process to # finish, but, as the python docs state "This will deadlock # when using stdout=PIPE and/or stderr=PIPE and the child # process generates enough output to a pipe such that it # blocks waiting for the OS pipe buffer to accept more data. # Use communicate() to avoid that." <- a catch, catch situation # # Use this work around were it's needed only, python 2.6 process.wait() out = process.stdout.read() err = process.stderr.read() else: out, err = process.communicate() # Force closing stderr/stdout to release file descriptors process.stdout.close() process.stderr.close() try: return out.splitlines(), err.splitlines() finally: try: process.terminate() except OSError, err: # process already terminated pass
def cmd(self, command, **kwargs): # prepare the environ, based on the system + our own env env = copy(environ) env.update(self.environ) # prepare the process kwargs.setdefault('env', env) kwargs.setdefault('stdout', PIPE) kwargs.setdefault('stderr', PIPE) kwargs.setdefault('close_fds', True) kwargs.setdefault('shell', True) kwargs.setdefault('show_output', self.log_level > 1) show_output = kwargs.pop('show_output') get_stdout = kwargs.pop('get_stdout', False) get_stderr = kwargs.pop('get_stderr', False) break_on_error = kwargs.pop('break_on_error', True) sensible = kwargs.pop('sensible', False) if not sensible: self.debug('Run {0!r}'.format(command)) else: if type(command) in (list, tuple): self.debug('Run {0!r} ...'.format(command[0])) else: self.debug('Run {0!r} ...'.format(command.split()[0])) self.debug('Cwd {}'.format(kwargs.get('cwd'))) # open the process if sys.platform == 'win32': kwargs.pop('close_fds', None) process = Popen(command, **kwargs) # prepare fds fd_stdout = process.stdout.fileno() fd_stderr = process.stderr.fileno() if fcntl: fcntl.fcntl(fd_stdout, fcntl.F_SETFL, fcntl.fcntl(fd_stdout, fcntl.F_GETFL) | os.O_NONBLOCK) fcntl.fcntl(fd_stderr, fcntl.F_SETFL, fcntl.fcntl(fd_stderr, fcntl.F_GETFL) | os.O_NONBLOCK) ret_stdout = [] if get_stdout else None ret_stderr = [] if get_stderr else None while True: try: readx = select.select([fd_stdout, fd_stderr], [], [])[0] except select.error: break if fd_stdout in readx: chunk = process.stdout.read() if not chunk: break if get_stdout: ret_stdout.append(chunk) if show_output: stdout.write(chunk.decode('utf-8', 'replace')) if fd_stderr in readx: chunk = process.stderr.read() if not chunk: break if get_stderr: ret_stderr.append(chunk) if show_output: stderr.write(chunk.decode('utf-8', 'replace')) stdout.flush() stderr.flush() process.communicate() if process.returncode != 0 and break_on_error: self.error('Command failed: {0}'.format(command)) self.log_env(self.ERROR, kwargs['env']) self.error('') self.error('Buildozer failed to execute the last command') if self.log_level <= self.INFO: self.error( 'If the error is not obvious, please raise the log_level to 2' ) self.error('and retry the latest command.') else: self.error( 'The error might be hidden in the log above this error') self.error( 'Please read the full log, and search for it before') self.error('raising an issue with buildozer itself.') self.error( 'In case of a bug report, please add a full log with log_level = 2' ) raise BuildozerCommandException() if ret_stdout: ret_stdout = b''.join(ret_stdout) if ret_stderr: ret_stderr = b''.join(ret_stderr) return (ret_stdout.decode('utf-8', 'ignore') if ret_stdout else None, ret_stderr.decode('utf-8') if ret_stderr else None, process.returncode)
if options.checkIfOnT2: print "### WARNING: make sure you have loaded CRAB3 and a valid CMS VO proxy ###" print "Check if the samples are available on T2" for sample in samples: PrimaryDataset, ProcessedDataset, DataTier = filter(None, sample.split("/")) print "Checking availability on T2/T3 of dataset ", PrimaryDataset call(["das_client.py --query \"site dataset=" + sample + "\" | grep T[2-3]"], shell=True) if options.checkNFiles: print "### WARNING: make sure you have loaded CRAB3 and a valid CMS VO proxy ###" print "Check how many files to be processed there are per sample" for sample in samples: PrimaryDataset, ProcessedDataset, DataTier = filter(None, sample.split("/")) p = Popen(["das_client.py --query \"file dataset=" + sample + "\" --limit=0 | grep store | wc -l"], shell=True, stdout=PIPE) out, err = p.communicate() print "Number of files for ", PrimaryDataset, out if options.createCrabConfig: print ("\nCreating CRAB configurations in %s" % options.campaign) print ("--------------------------------------------------------") if not options.task_dir: options.task_dir = options.campaign if not os.path.isdir(options.task_dir): os.mkdir(options.task_dir) os.chdir(options.task_dir) print ("Parameter set: %s\nflashggVersion: %s\ncrab template: %s\n" % (options.parameterSet,flashggVersion,options.crabTemplate)) print ("Copying over parameter set") Popen(['cp', '-p', options.parameterSet, './']) rel = os.environ.get('CMSSW_BASE')
def execute(self, workingDir, reuse=True): """Generate sun matrix. Args: workingDir: Folder to execute and write the output. reuse: Reuse the matrix if already existed in the folder. Returns: Full path to analemma, sunlist and sunmatrix. """ fp = os.path.join(workingDir, self.analemmafile) lfp = os.path.join(workingDir, self.sunlistfile) mfp = os.path.join(workingDir, self.sunmtxfile) hrf = os.path.join(workingDir, self.name + '.hrs') if reuse: if self.hoursMatch(hrf): for f in (fp, lfp, mfp): if not os.path.isfile(f): break else: print('Found the sun matrix!') return fp, lfp, mfp with open(hrf, 'wb') as outf: outf.write(','.join(str(h) for h in self.hoys) + '\n') # written based on scripts/analemma provided by @sariths wea = self.wea monthDateTime = (DateTime.fromHoy(idx) for idx in self.hoys) latitude, longitude = wea.location.latitude, -wea.location.longitude meridian = -(15 * wea.location.timezone) gdlit = Gendaylit(rotation=self.north) gdlit.gendaylitParameters.meridian = meridian gdlit.gendaylitParameters.longitude = longitude gdlit.gendaylitParameters.latitude = latitude sunValues = [] sunUpHours = [] # collect hours that sun is up # use gendaylit to calculate radiation values for each hour. print('Calculating sun positions and radiation values.') with open(os.devnull, 'w') as warningDump: for timeStamp in monthDateTime: month, day, hour = timeStamp.month, timeStamp.day, timeStamp.hour + 0.5 dnr, dhr = wea.directNormalRadiation[timeStamp.intHOY], \ wea.diffuseHorizontalRadiation[timeStamp.intHOY] if dnr + dhr == 0: # no need to run gendaylit as there is no radiation / sun continue # update gendaylit params gdlit.gendaylitParameters.dirNormDifHorzIrrad = (dnr, dhr) gdlit.monthDayHour = (month, day, hour) cmdgd, ro = tuple( c.strip() for c in gdlit.toRadString().split('>')[0].split('|')) # run cmd, get results in the form of a list of lines. cmdRun = Popen(cmdgd, stdout=PIPE, stderr=warningDump) gd = Popen(ro, stdin=cmdRun.stdout, stdout=PIPE) data = gd.communicate()[0].split('\n') # clean the output by throwing out comments and brightness functions. sunCurrentValue = [] for lines in data[:-6]: if lines[0] == "#": continue ls = lines.strip() if ls: sunCurrentValue.extend(ls.split()) # If a sun definition was captured in the last for-loop, store info. if sunCurrentValue and max(map(float, sunCurrentValue[6:9])): sunCurrentValue[2] = 'solar%s' % (len(sunValues) + 1) sunCurrentValue[9] = 'solar%s' % (len(sunValues) + 1) sunValues.append(sunCurrentValue) sunUpHours.append(timeStamp.intHOY) numOfSuns = len(sunUpHours) print('Writing sun positions and radiation values to {}'.format(fp)) # create solar discs. with open(fp, 'w') as annfile: annfile.write("\n".join((" ".join(sun) for sun in sunValues))) annfile.write('\n') print('Writing list of suns to {}'.format(lfp)) # create list of suns. with open(lfp, 'w') as sunlist: sunlist.write("\n".join( ("solar%s" % (idx + 1) for idx in xrange(numOfSuns)))) sunlist.write('\n') # Start creating header for the sun matrix. fileHeader = ['#?RADIANCE'] fileHeader += ['Sun matrix created by Honeybee'] fileHeader += ['LATLONG= %s %s' % (latitude, -longitude)] fileHeader += ['NROWS=%s' % numOfSuns] fileHeader += ['NCOLS=%s' % len(self.hoys)] fileHeader += ['NCOMP=3'] fileHeader += ['FORMAT=ascii'] print('Writing sun matrix to {}'.format(mfp)) # Write the matrix to file. with open(mfp, 'w') as sunMtx: sunMtx.write('\n'.join(fileHeader) + '\n' + '\n') for idx, sunValue in enumerate(sunValues): sunRadList = ['0 0 0'] * len(self.hoys) sunRadList[sunUpHours[idx]] = ' '.join(sunValue[6:9]) sunMtx.write('\n'.join(sunRadList) + '\n\n') # This last one is for the ground. sunRadList = ['0 0 0'] * len(self.hoys) sunMtx.write('\n'.join(sunRadList)) sunMtx.write('\n') return fp, lfp, mfp
def output(partIdx,ch_aux): print '== If you use queryDocTrainData or queryDocTrainRel, make sure it\'s in the current working directory' print '== Running your code ...' print '== Your code should output results (and nothing else) to stdout' # tempoutfile = tempfile.mkdtemp() # getFiles(tempoutfile) outputString = '' tempoutfile = 'tmp' + str(random.randint(0,1<<20)) localfile = open(tempoutfile,'w') localfile.write(ch_aux) localfile.close() linesOutput = 1065 if not os.path.exists("people.txt"): print "There is no people.txt file in this directory. Please make people.txt file in this directory with your and your partner's SUNet ID in separate lines (do NOT include @stanford.edu)" sys.exit(1) people = open("people.txt").read().splitlines(); if len(people) == 0: print "people.txt is empty! Write the SUNet ids of you and your partner, if any, in separate lines.." sys.exit(1); for x in people: if len(x) < 3 or len(x) > 8 or ' ' in x.strip() == True: print "The SUNet IDs don't seem to be correct. Make sure to remove empty lines. They are supposed to be between 3 and 8 characters. Also, make sure to not include @stanford.edu." sys.exit(1); peopleStr = "_".join(str(x.strip()) for x in people if x) stats = {} stats['USERIDS']= peopleStr stats['TIMESUBMITTED'] = str(datetime.datetime.now()); elapsed= 0.0 if partIdx == 0: print 'Submitting the report' elif partIdx == 1: print 'Calling ./rank.sh for Task 1 (this might take a while)' start = time() child = Popen(['./rank.sh', tempoutfile,'cosine'], stdout=PIPE, stderr=PIPE, shell=False); (res, err) = child.communicate("") elapsed = time() - start guesses = res.splitlines() print err if (len(guesses) != linesOutput): print 'Warning. The number of url-document pairs ' + str(len(guesses)) + ' is not correct. Please ensure that the output is formatted properly.' stats['result'] = res elif partIdx == 2: print 'Calling ./rank.sh for Task 2 (this might take a while)' start = time() child = Popen(['./rank.sh', tempoutfile,'bm25'], stdout=PIPE, stderr=PIPE, shell=False) (res, err) = child.communicate("") elapsed = time() - start guesses = res.splitlines() print err if (len(guesses) != linesOutput): print 'Warning. The number of url-document pairs is not correct. Please ensure that the output is formatted properly.' stats['result'] = res elif partIdx == 3: print 'Calling ./rank.sh for Task 3 (this might take a while)' start = time() child = Popen(['./rank.sh',tempoutfile,'window'], stdout=PIPE, stderr=PIPE, shell=False) (res, err) = child.communicate("") elapsed = time() - start guesses = res.splitlines() print err if (len(guesses) != linesOutput): print 'Warning. The number of url-document pairs is not correct. Please ensure that the output is formatted properly.' stats['result'] = res elif partIdx == 4: print 'Calling ./rank.sh for Task 4 (this might take a while)' start = time() child = Popen(['./rank.sh', tempoutfile,'extra'], stdout=PIPE, stderr=PIPE, shell=False) (res, err) = child.communicate("") elapsed = time() - start guesses = res.splitlines() print err if (len(guesses) != linesOutput): print 'Warning. The number of url-document pairs is not correct. Please ensure that the output is formatted properly.' stats['result'] = res else: print '[WARNING]\t[output]\tunknown partId: %s' % partIdx sys.exit(1) print '== Finished running your code' os.remove(tempoutfile) return json.dumps(stats).strip()
class TransferTest(unittest.TestCase): """Subclass and override inputs/expectedOutputs (and possibly other stuff) to create new postchunk tests.""" bindata = "data/nno-nob.t1x.bin" t1xdata = "data/apertium-nno-nob.nob-nno.t1x" flags = ["-b", "-z"] inputs = [""] expectedOutputs = [""] expectedRetCodeFail = False def alarmHandler(self, signum, frame): raise Alarm def withTimeout(self, seconds, cmd, *args, **kwds): signal.signal(signal.SIGALRM, self.alarmHandler) signal.alarm(seconds) ret = cmd(*args, **kwds) signal.alarm(0) # reset the alarm return ret def communicateFlush(self, string): self.proc.stdin.write(string.encode('utf-8')) self.proc.stdin.write(b'\0') self.proc.stdin.flush() output = [] char = None try: char = self.withTimeout(2, self.proc.stdout.read, 1) except Alarm: pass while char and char != b'\0': output.append(char) try: char = self.withTimeout(2, self.proc.stdout.read, 1) except Alarm: break # send what we got up till now return b"".join(output).decode('utf-8') def compile(self): compileCmd = ["../apertium/apertium-preprocess-transfer", self.t1xdata, self.bindata] self.assertEqual(call(compileCmd), 0) def runTest(self): self.compile() try: cmd = ["../apertium/apertium-transfer"] \ + self.flags \ + [self.t1xdata, self.bindata] self.proc = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) for inp, exp in zip(self.inputs, self.expectedOutputs): self.assertEqual(self.communicateFlush(inp+"[][\n]"), exp+"[][\n]") self.proc.communicate() # let it terminate self.proc.stdin.close() self.proc.stdout.close() self.proc.stderr.close() retCode = self.proc.poll() if self.expectedRetCodeFail: self.assertNotEqual(retCode, 0) else: self.assertEqual(retCode, 0) finally: pass
class TritonServerLocal(TritonServer): """ Concrete Implementation of TritonServer interface that runs tritonserver locally as as subprocess. """ def __init__(self, path, config): """ Parameters ---------- path : str The absolute path to the tritonserver executable config : TritonServerConfig the config object containing arguments for this server instance """ self._tritonserver_process = None self._server_config = config self._server_path = path self._log = None assert self._server_config['model-repository'], \ "Triton Server requires --model-repository argument to be set." def start(self): """ Starts the tritonserver container locally """ # Create command list and run subprocess cmd = [self._server_path] cmd += self._server_config.to_cli_string().replace('=', ' ').split() self._tritonserver_process = Popen(cmd, start_new_session=True, stdout=PIPE, stderr=STDOUT, universal_newlines=True) logger.info('Triton Server started.') def stop(self): """ Stops the running tritonserver """ # Terminate process, capture output if self._tritonserver_process is not None: self._tritonserver_process.terminate() try: self._log, _ = self._tritonserver_process.communicate( timeout=SERVER_OUTPUT_TIMEOUT_SECS) except TimeoutExpired: self._tritonserver_process.kill() self._log, _ = self._tritonserver_process.communicate() self._tritonserver_process = None logger.info('Triton Server stopped.') def logs(self): """ Retrieves the Triton server's stdout as a str """ return self._log def cpu_stats(self): """ Returns the CPU memory usage and CPU available memory in MB """ if self._tritonserver_process: server_process = psutil.Process(self._tritonserver_process.pid) process_memory_info = server_process.memory_full_info() system_memory_info = psutil.virtual_memory() # Divide by 1.0e6 to convert from bytes to MB return (process_memory_info.uss // 1.0e6), (system_memory_info.available // 1.0e6) else: return 0.0, 0.0
def _do_createproject(self, req): """ Update project properties from the Admin form """ data = {} data['project_created'] = True group = req.args['group'] templates_dir = self.env.config.get('inherit', 'templates_dir') inherit_file = self.env.config.get('inherit', 'file') #Try to get custom settings for this project group templates_dir = self.env.config.get('projectmanager', 'groups.%s.templates_dir' % group, templates_dir) inherit_file = self.env.config.get('projectmanager', 'groups.%s.inherit_file' % group, inherit_file) group_dir = self.env.config.get('projectmanager', 'groups.%s.dirname' % group, group) sql_defaults = self.env.config.get('projectmanager', 'groups.%s.sql_defaults' % group, self.sql_defaults) assert self.repos_dir, "repos_dir not defined in .ini" assert self.environments_dir, "environments_dir not defined in .ini" if req.args.has_key('makesvn'): cmd = "%s create %s" % (self.svnadmin_command,os.path.join(self.repos_dir, group_dir, req.args['short_name'])) p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE) (stdout, stderr) = p.communicate() if p.returncode != 0: data['svn_error'] = True data['svn_created'] = True data['svn_output'] = stdout data['svn_errors'] = stderr if req.args.has_key('maketrac'): repos_dir = os.path.join(self.repos_dir, group_dir, req.args['short_name']) env_dir = os.path.join(self.environments_dir, group_dir, req.args['short_name']) cmd = '%s "%s" initenv "%s" sqlite:%s svn "%s" --inherit="%s"' % ( self.tracadmin_command, env_dir, req.args['full_name'], os.path.join('db','trac.db'), repos_dir, inherit_file ) #stdin, stdout, stderr = os.popen3(command) p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE) (stdout, stderr) = p.communicate() data['trac_error'] = p.returncode != 0 try: template = os.path.join(env_dir, 'templates', 'site.html') stdout = stdout + 'Removing template: %s' % template os.remove(template) except: pass data['trac_created'] = True data['trac_output'] = stdout data['trac_errors'] = stderr try: env = open_environment(env_dir, use_cache=False) db = env.get_db_cnx() cursor = db.cursor() output = "" output = output + "\n\nSQL Defaults, executing statements\n" output = output + "==================================\n" for statement in sql_defaults.splitlines(): output = output + "Executing: %s\n" % statement cursor.execute(statement) db.commit() data['trac_output'] = data['trac_output'] + output env.shutdown() except: data['trac_output'] = data['trac_output'] + "\n\nError running SQL Defaults statements, skipping\n" return data
dataXsample = np.asarray([[-1.0 for i in range(n)] for j in range(n)], dtype=np.float) dataXdata = np.asarray([[-1.0 for i in range(n)] for j in range(n)], dtype=np.float) errors = [False, False, False] for i in range(n): for j in range(n): try: p = Popen([ './bottleneck_dist', sample_strings[i], sample_strings[j] ], stdin=PIPE, stdout=PIPE, stderr=PIPE) output, err = p.communicate( b"input data that is passed to subprocess' stdin") output_string = output.decode() sampleXsample[i][j] = float(output_string) except ValueError as valerr: errors[0] = True print(str(err) + " " + str(valerr)) print(sample_strings[i] + " " + sample_strings[j]) print(output_string) try: p = Popen( ['./bottleneck_dist', data_strings[i], sample_strings[j]], stdin=PIPE, stdout=PIPE, stderr=PIPE) output, err = p.communicate(
def send_email_old(address, subject, content): p = Popen(["/usr/sbin/sendmail", "-i", "--", address], stdin=PIPE) msg = MIMEText(content) msg['Subject'] = subject msg['To'] = address p.communicate(msg.as_bytes())
def cmdline(cmd, sh=True): """ Execute command @cmd and return output using Popen(). """ process = Popen(args=cmd, stdout=PIPE, shell=sh) return process.communicate()[0].decode("ISO-8859-1")
def get_mysql_config_info(mysql_config): """Get MySQL information using mysql_config tool Returns a dict. """ options = ['cflags', 'include', 'libs', 'libs_r', 'plugindir', 'version'] cmd = [mysql_config] + ["--{0}".format(opt) for opt in options] try: proc = Popen(cmd, stdout=PIPE, universal_newlines=True) stdout, _ = proc.communicate() except OSError as exc: raise DistutilsExecError("Failed executing mysql_config: {0}".format( str(exc))) info = parse_mysql_config_info(options, stdout) # Try to figure out the architecture info['arch'] = None if os.name == 'posix': if platform.uname()[0] == 'SunOS': print("info['lib_dir']: {0}".format(info['lib_dir'])) print("info['libs'][0]: {0}".format(info['libs'][0])) pathname = os.path.abspath( os.path.join(info['lib_dir'], 'lib', info['libs'][0])) + '/*' else: pathname = os.path.join(info['lib_dir'], 'lib' + info['libs'][0]) + '*' print("# Looking mysqlclient_lib at path: {0}".format(pathname)) log.debug("# searching mysqlclient_lib at: %s", pathname) libs = glob(pathname) mysqlclient_libs = [] for filepath in libs: _, filename = os.path.split(filepath) log.debug("# filename {0}".format(filename)) if filename.startswith('libmysqlclient') and \ not os.path.islink(filepath) and \ '_r' not in filename and \ '.a' not in filename: mysqlclient_libs.append(filepath) mysqlclient_libs.sort() stdout = None try: log.debug("# mysqlclient_lib: {0}".format(mysqlclient_libs[-1])) for mysqlclient_lib in mysqlclient_libs: log.debug("#+ {0}".format(mysqlclient_lib)) log.debug("# tested mysqlclient_lib[-1]: " "{0}".format(mysqlclient_libs[-1])) if platform.uname()[0] == 'SunOS': print("mysqlclient_lib: {0}".format(mysqlclient_libs[-1])) cmd_list = ['file', mysqlclient_libs[-1]] else: cmd_list = ['file', '-L', mysqlclient_libs[-1]] proc = Popen(cmd_list, stdout=PIPE, universal_newlines=True) stdout, _ = proc.communicate() stdout = stdout.split(':')[1] except OSError as exc: raise DistutilsExecError( "Although the system seems POSIX, the file-command could not " "be executed: {0}".format(str(exc))) if stdout: if '64' in stdout: info['arch'] = "x86_64" else: info['arch'] = "i386" else: raise DistutilsExecError( "Failed getting out put from the file-command") else: raise DistutilsExecError( "Cannot determine architecture on {0} systems".format(os.name)) return info
def main(): UPDATED_DIGEST = 'SHA256' DIGESTS_TO_UPDATE = ['WHIRLPOOL', 'RIPEMD'] keys = defaultdict(lambda: 0) keys_to_update = [] dynamodb_resource = resource('dynamodb') table = dynamodb_resource.Table('credential-store') response = table.scan() items = response['Items'] # appending all dynamodb entries to items dict while True: if response.get('LastEvaluatedKey'): response = table.scan( ExclusiveStartKey=response['LastEvaluatedKey']) items += response['Items'] else: break # storing latest version of keys with their digests for i in range(len(items)): try: digest = items[i]['digest'] version = int(items[i]['version']) key = items[i]['name'] except: continue if key in keys: if version > keys[key][0]: keys[key][0] = version keys[key][1] = digest else: keys[key] = [version, digest] # store keys to be updated for k, v in keys.items(): if v[1] in DIGESTS_TO_UPDATE: keys_to_update.append(k) # confirms update of digests if len(keys_to_update): print('\nThe following keys will be updated to {0}:\n'.format( UPDATED_DIGEST)) for key in keys_to_update: print('{0}\n'.format(key)) confirmed = None while not confirmed: val = input('Continue? y/n ') if val.lower() == 'y' or val.lower() == 'yes': confirmed = True elif val.lower() == 'n' or val.lower() == 'no': print('\nexiting...\n') sys.exit() else: print('\nInvalid input\n') else: print('\nNo digests to update!\n') sys.exit() # updating deprecated digests for key in keys_to_update: p = Popen(['credstash', 'get', key], stdout=PIPE, stderr=PIPE) secret, err = p.communicate() secret = secret[:-1] # removes credstash-added newline for stdout if not err: p = Popen( ['credstash', 'put', key, secret, '-a', '-d', UPDATED_DIGEST], stdout=PIPE) update, err = p.communicate() print('{0} has been updated!\n'.format(key)) else: print('Error found, skipping update of {0}. Error: {1}'.format( key, err))
def parse_recentApps(self, dataSource, progressBar): # we don't know how much work there is yet progressBar.switchToIndeterminate() # Set the database to be read to the once created by the prefetch parser program skCase = Case.getCurrentCase().getSleuthkitCase(); fileManager = Case.getCurrentCase().getServices().getFileManager() files = fileManager.findFiles(dataSource, "%", "/Windows/System32/wbem/Repository/") numFiles = len(files) progressBar.switchToDeterminate(numFiles) fileCount = 0; # Create Event Log directory in temp directory, if it exists then continue on processing Temp_Dir = Case.getCurrentCase().getTempDirectory() self.log(Level.INFO, "create Directory " + Temp_Dir) temp_dir = os.path.join(Temp_Dir, "Recently_Used") try: os.mkdir(temp_dir) except: self.log(Level.INFO, "Recently Used Directory already exists " + temp_dir) # Write out each Event Log file to the temp directory for file in files: # Check if the user pressed cancel while we were busy if self.context.isJobCancelled(): return IngestModule.ProcessResult.OK #self.log(Level.INFO, "Processing file: " + file.getName()) fileCount += 1 if (file.getName() == '.' or file.getName() == '..'): self.log(Level.INFO, "Parent or Root Directory File not writing") else: # Save the DB locally in the temp folder. use file id as name to reduce collisions lclDbPath = os.path.join(temp_dir, file.getName()) ContentUtils.writeToFile(file, File(lclDbPath)) self.log(Level.INFO, "Running prog ==> " + self.path_to_recentApps_exe + " win7 " + temp_dir + " " + \ temp_dir + "\recentlyUsedApps.db3") pipe = Popen([self.path_to_recentApps_exe, "win7", temp_dir, os.path.join(temp_dir, "recentlyUsedApps.db3")], stdout=PIPE, stderr=PIPE) out_text = pipe.communicate()[0] self.log(Level.INFO, "Output from run is ==> " + out_text) lclDbPath = os.path.join(temp_dir, "recentlyUsedApps.db3") if ("Exiting" in out_text): message = IngestMessage.createMessage(IngestMessage.MessageType.DATA, "CCM Recently Used Apps", " Error in CCM Recently Used Apps module " ) IngestServices.getInstance().postMessage(message) else: # Add custom Artifact to blackboard try: self.log(Level.INFO, "Begin Create New Artifacts ==> TSK_CCM_RECENTLY_USED_APPS") artID_art = skCase.addArtifactType("TSK_CCM_RECENTLY_USED_APPS", "WMI Recently Used Apps") except: self.log(Level.INFO, "Artifacts Creation Error, artifact TSK_CCM_RECENTLY_USED_APPS exists. ==> ") # Add Custom attributes to blackboard try: attID_efn = skCase.addArtifactAttributeType("TSK_EXPLORER_FILE_NAME", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "Explorer File Name") except: self.log(Level.INFO, "Attributes Creation Error, Explorer File Name ==> ") try: attID_efn = skCase.addArtifactAttributeType("TSK_FILE_SIZE", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "File Size") except: self.log(Level.INFO, "Attributes Creation Error, File Size ==> ") try: attID_efn = skCase.addArtifactAttributeType("TSK_LAST_USED_TIME", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.DATETIME, "Last Used Time") except: self.log(Level.INFO, "Attributes Creation Error, Last Used Time ==> ") try: attID_efn = skCase.addArtifactAttributeType("TSK_TIME_ZONE_OFFSET", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "Time Zone Offset") except: self.log(Level.INFO, "Attributes Creation Error, Time Zone Offset ==> ") try: attID_efn = skCase.addArtifactAttributeType("TSK_LAUNCH_COUNT", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "Launch Count") except: self.log(Level.INFO, "Attributes Creation Error, Launch Count ==> ") try: attID_efn = skCase.addArtifactAttributeType("TSK_ORIG_FILE_NAME", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "Original File Name") except: self.log(Level.INFO, "Attributes Creation Error, Original File Name ==> ") try: attID_efn = skCase.addArtifactAttributeType("TSK_FILE_DESC", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "File Description") except: self.log(Level.INFO, "Attributes Creation Error, File Description ==> ") try: attID_efn = skCase.addArtifactAttributeType("TSK_PROD_NAME", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "Product Name") except: self.log(Level.INFO, "Attributes Creation Error, Product Name ==> ") try: attID_efn = skCase.addArtifactAttributeType("TSK_PROD_VERSION", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "Product Version") except: self.log(Level.INFO, "Attributes Creation Error, Product Version ==> ") try: attID_efn = skCase.addArtifactAttributeType("TSK_FILE_VERSION", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "File Version") except: self.log(Level.INFO, "Attributes Creation Error, File Version ==> ") try: attID_efn = skCase.addArtifactAttributeType("TSK_ADDITIONAL_PROD_CODES", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "Additional Product Codes") except: self.log(Level.INFO, "Attributes Creation Error, Additional Product Codes ==> ") try: attID_efn = skCase.addArtifactAttributeType("TSK_MSI_VERSION", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "MSI Version") except: self.log(Level.INFO, "Attributes Creation Error, MSI Version ==> ") try: attID_efn = skCase.addArtifactAttributeType("TSK_MSI_DISPLAY_NAME", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "MSI Display Name") except: self.log(Level.INFO, "Attributes Creation Error, MSI Display Name ==> ") try: attID_efn = skCase.addArtifactAttributeType("TSK_PRODUCT_CODE", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "Product Code") except: self.log(Level.INFO, "Attributes Creation Error, Product Code ==> ") try: attID_efn = skCase.addArtifactAttributeType("TSK_SOFTWARE_PROP_HASH", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "Software Property Hash") except: self.log(Level.INFO, "Attributes Creation Error, Software Property Hash ==> ") try: attID_efn = skCase.addArtifactAttributeType("TSK_PROD_LANG", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "Product Language") except: self.log(Level.INFO, "Attributes Creation Error, Product Language ==> ") try: attID_efn = skCase.addArtifactAttributeType("TSK_FILE_PROP_HASH", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "File Property Hash") except: self.log(Level.INFO, "Attributes Creation Error, File Property Hash ==> ") try: attID_efn = skCase.addArtifactAttributeType("TSK_MSI_PUBLISHER", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "MSI Publisher") except: self.log(Level.INFO, "Attributes Creation Error, MSI Publisher ==> ") for file in files: if (file.getName() == "OBJECTS.DATA"): # Open the DB using JDBC lclDbPath = os.path.join(temp_dir, "recentlyUsedApps.db3") self.log(Level.INFO, "Path the recentlyUsedApps.db3 database file created ==> " + lclDbPath) try: Class.forName("org.sqlite.JDBC").newInstance() dbConn = DriverManager.getConnection("jdbc:sqlite:%s" % lclDbPath) except SQLException as e: self.log(Level.INFO, "Could not open database file (not SQLite) recentlyUsedApps.db3 (" + e.getMessage() + ")") return IngestModule.ProcessResult.OK # Query the history_visits table in the database and get all columns. try: stmt = dbConn.createStatement() recently_used_sql = "select FolderPath 'TSK_PATH', ExplorerFileName 'TSK_EXPLORER_FILE_NAME', " + \ "FileSize 'TSK_FILE_SIZE', LastUserName 'TSK_USER_ID', strftime('%s',LastUsedTime) " + \ "'TSK_LAST_USED_TIME', TimeZoneOffset 'TSK_TIME_ZONE_OFFSET', LaunchCount " + \ "'TSK_LAUNCH_COUNT', OriginalFileName 'TSK_ORIG_FILE_NAME', FileDescription " + \ "'TSK_FILE_DESC', CompanyName 'TSK_ORGANIZATION', ProductName 'TSK_PROD_NAME', " + \ "ProductVersion 'TSK_PROD_VERSION', FileVersion 'TSK_FILE_VERSION', " + \ "AdditionalProductCodes 'TSK_ADDITIONAL_PROD_CODES', msiVersion " + \ "'TSK_MSI_VERSION', msiDisplayName 'TSK_MSI_DISPLAY_NAME', " + \ "ProductCode 'TSK_PRODUCT_CODE', SoftwarePropertiesHash " + \ "'TSK_SOFTWARE_PROP_HASH', ProductLanguage 'TSK_PROD_LANG', " + \ "FilePropertiesHash 'TSK_FILE_PROP_HASH', msiPublisher 'TSK_MSI_PUBLISHER' " + \ "from recently_used;" self.log(Level.INFO, recently_used_sql) resultSet = stmt.executeQuery(recently_used_sql) self.log(Level.INFO, "query recently_used table") except SQLException as e: self.log(Level.INFO, "Error querying database for recently_used table (" + e.getMessage() + ")") return IngestModule.ProcessResult.OK artID_hst = skCase.getArtifactTypeID("TSK_CCM_RECENTLY_USED_APPS") artID_hst_evt = skCase.getArtifactType("TSK_CCM_RECENTLY_USED_APPS") meta = resultSet.getMetaData() columncount = meta.getColumnCount() column_names = [] self.log(Level.INFO, "Number of Columns in the table ==> " + str(columncount)) for x in range (1, columncount + 1): self.log(Level.INFO, "Column Name ==> " + meta.getColumnLabel(x)) column_names.append(meta.getColumnLabel(x)) self.log(Level.INFO, "All Columns ==> " + str(column_names)) # Cycle through each row and create artifacts while resultSet.next(): try: #self.log(Level.INFO, SQL_String_1) self.log(Level.INFO, "Artifact Is ==> " + str(artID_hst)) art = file.newArtifact(artID_hst) self.log(Level.INFO, "Inserting attribute URL") for col_name in column_names: attID_ex1 = skCase.getAttributeType(col_name) self.log(Level.INFO, "Inserting attribute ==> " + str(attID_ex1)) self.log(Level.INFO, "Attribute Type ==> " + str(attID_ex1.getValueType())) if attID_ex1.getValueType() == BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING: try: art.addAttribute(BlackboardAttribute(attID_ex1, ParseRecentlyUsedAppsIngestModuleFactory.moduleName, resultSet.getString(col_name))) except: self.log(Level.INFO, "Attributes String Creation Error, " + col_name + " ==> ") elif attID_ex1.getValueType() == BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.INTEGER: try: art.addAttribute(BlackboardAttribute(attID_ex1, ParseRecentlyUsedAppsIngestModuleFactory.moduleName, resultSet.getInt(col_name))) except: self.log(Level.INFO, "Attributes Integer Creation Error, " + col_name + " ==> ") elif attID_ex1.getValueType() == BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.LONG: try: art.addAttribute(BlackboardAttribute(attID_ex1, ParseRecentlyUsedAppsIngestModuleFactory.moduleName, resultSet.getInt(col_name))) except: self.log(Level.INFO, "Attributes Long Creation Error, " + col_name + " ==> ") elif attID_ex1.getValueType() == BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.DOUBLE: try: art.addAttribute(BlackboardAttribute(attID_ex1, ParseRecentlyUsedAppsIngestModuleFactory.moduleName, resultSet.getInt(col_name))) except: self.log(Level.INFO, "Attributes Double Creation Error, " + col_name + " ==> ") elif attID_ex1.getValueType() == BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.BYTE: try: art.addAttribute(BlackboardAttribute(attID_ex1, ParseRecentlyUsedAppsIngestModuleFactory.moduleName, resultSet.getString(col_name))) except: self.log(Level.INFO, "Attributes Byte Creation Error, " + col_name + " ==> ") else: try: art.addAttribute(BlackboardAttribute(attID_ex1, ParseRecentlyUsedAppsIngestModuleFactory.moduleName, int(resultSet.getString(col_name)))) except: self.log(Level.INFO, "Attributes Datatime Creation Error, " + col_name + " ==> ") except SQLException as e: self.log(Level.INFO, "Error getting values from web_history table (" + e.getMessage() + ")") IngestServices.getInstance().fireModuleDataEvent( ModuleDataEvent(ParseRecentlyUsedAppsIngestModuleFactory.moduleName, artID_hst_evt, None)) stmt.close() dbConn.close() # Clean up try: os.remove(lclDbPath) except: self.log(Level.INFO, "removal of Recently Used database failed ") #Clean up EventLog directory and files for file in files: try: os.remove(os.path.join(temp_dir, file.getName())) except: self.log(Level.INFO, "removal of Recently Used files failed " + temp_dir + "\\" + file.getName()) try: os.rmdir(temp_dir) except: self.log(Level.INFO, "removal of recently used directory failed " + temp_dir)