def run(self):
self.testReady()
# submits the input file to Gaussian
process = Popen([self.executablePath, self.inputFilePath, self.outputFilePath])
process.communicate() # necessary to wait for executable termination!
return self.verifyOutputFile()
def ly2png(lily, outfile, kvs):
p = Popen([
"lilypond",
"-dno-point-and-click",
"-dbackend=eps",
"-djob-count=2",
"-ddelete-intermediate-files",
"-o", outfile,
"-"
], stdin=PIPE, stdout=stderr)
p.stdin.write(("\\paper{\n"
"indent=0\\mm\n"
"oddFooterMarkup=##f\n"
"oddHeaderMarkup=##f\n"
"bookTitleMarkup = ##f\n"
"scoreTitleMarkup = ##f\n"
"line-width = %s\n"
"}\n"
"#(set-global-staff-size %s)\n" % (
kvs['width'][:-2] + '\\' + kvs['width'][-2:],
kvs['staffsize']) +
lily).encode("utf-8"))
p.communicate()
p.stdin.close()
call([
"gs",
"-sDEVICE=pngalpha",
"-r144",
"-sOutputFile=" + outfile + '.png',
outfile + '.pdf',
], stdout=stderr)
def find_bowtie2_index(r, path_to_bowtie2='bowtie2'):
"""check for bowtie2 index as given.
return True if found, else return False
"""
args = [path_to_bowtie2 + '-inspect', '-v', '-s', r]
debug(' '.join(args))
P = Popen(args, stdout=open(devnull, 'w'), stderr=PIPE, cwd=mkdtemp())
stderr = P.communicate()[1].splitlines()
if not stderr[0].startswith('Could not locate'):
for line in stderr:
if line.startswith('Opening'):
index_bt2 = line[(1 + line.find('"')):line.rfind('"')]
index_basename = index_bt2[0:index_bt2.find('.1.bt2')]
return index_basename
for d in [getcwd(), os.path.split(path_to_bowtie2)[0],
join(os.path.split(path_to_bowtie2)[0], 'indexes')]:
rprime = join(d, r)
args = [path_to_bowtie2 + '-inspect', '-v', '-s', rprime]
debug(' '.join(args))
P = Popen(args, stdout=open(devnull, 'w'), stderr=PIPE, cwd=mkdtemp())
stderr = P.communicate()[1].splitlines()
if not stderr[0].startswith('Could not locate'):
for line in stderr:
if line.startswith('Opening'):
index_bt2 = line[(1 + line.find('"')):line.rfind('"')]
index_basename = index_bt2[0:index_bt2.find('.1.bt2')]
return index_basename
return None
def run_in_command_line(commands, nusmv_commands):
p = Popen(commands, stdout=PIPE, stdin=PIPE, stderr=STDOUT)
stdout = p.communicate()[0].decode()
if nusmv_commands is not None:
for c in nusmv_commands:
stdout += p.communicate(c)[0].decode()
return stdout
def sendemail(email,description,link):
args = [ 'mailx' ]
# Subject
args.append('-s')
args.append('ipipet file')
# From
args.append('-r')
args.append('ipipet admin <*****@*****.**>')
# Bcc
args.append('-b')
args.append('*****@*****.**')
# TO
args.append(email)
msg = """
here's your link to start pipetting, based on the input file for your %s project.
Open this email on your tablet, and click this link:
http://ipipet.teamerlich.org%s
""" % ( description, link )
p = Popen(args, stdin=PIPE)
p.communicate(input=msg)
def run_itmcmd(self):
logger.info('starting run itmcmd silent config command...')
file_silentcfg=os.path.join(self.candlehome,'silent_config.txt')
silent_cfg_tup = ('HOSTNAME=' + self.rtms,
'FTO=YES',
'MIRROR=' + self.secrtms,
'HSNETWORKPROTOCOL=ip.pipe'
)
silent_cfg='\n'.join(silent_cfg_tup)
logger.info('slient config file content')
logger.info('\n' + silent_cfg)
with open(file_silentcfg,'w') as myfile:
myfile.write(silent_cfg)
for pc in self.pclist:
logger.info('starting slient config {0}'.format(pc))
if pc == 'ud':
for inst in self.inst:
ret=Popen(['/opt/itm6/bin/itmcmd','config','-A','-h',self.candlehome,'-o',inst,'-p',file_silentcfg,pc],stdout=PIPE,stderr=PIPE)
boutput=ret.communicate()[0]
output=str(boutput,encoding='utf-8')
logger.info('\n' + output)
else:
ret=Popen(['/opt/itm6/bin/itmcmd','config','-A','-h',self.candlehome,'-p',file_silentcfg,pc],stdout=PIPE,stderr=PIPE)
boutput=ret.communicate()[0]
output=str(boutput,encoding='utf-8')
logger.info('\n' + output)
logger.info('ended run itmcmd silent config command')
def test_generate_config():
with NamedTemporaryFile(prefix='jupyterhub_config', suffix='.py') as tf:
cfg_file = tf.name
with open(cfg_file, 'w') as f:
f.write("c.A = 5")
p = Popen([sys.executable, '-m', 'jupyterhub',
'--generate-config', '-f', cfg_file],
stdout=PIPE, stdin=PIPE)
out, _ = p.communicate(b'n')
out = out.decode('utf8', 'replace')
assert os.path.exists(cfg_file)
with open(cfg_file) as f:
cfg_text = f.read()
assert cfg_text == 'c.A = 5'
p = Popen([sys.executable, '-m', 'jupyterhub',
'--generate-config', '-f', cfg_file],
stdout=PIPE, stdin=PIPE)
out, _ = p.communicate(b'x\ny')
out = out.decode('utf8', 'replace')
assert os.path.exists(cfg_file)
with open(cfg_file) as f:
cfg_text = f.read()
os.remove(cfg_file)
assert cfg_file in out
assert 'Spawner.cmd' in cfg_text
assert 'Authenticator.whitelist' in cfg_text
def main():
args = hackparse.process_args()
for f in args[1:]:
fobj = open(f, "rb")
cert = fobj.read()
fobj.close()
print "Hackparsing " + f
a = Popen(od.OPENSSL_ARGS, stdin=PIPE, stdout=PIPE, stderr=PIPE)
try: pcert, err = a.communicate(cert)
except: err = MAGIC_ERROR
if err.startswith(MAGIC_ERROR):
a = Popen(od.DER_ARGS, stdin=PIPE, stdout=PIPE, stderr=PIPE)
try:
pcert, err = a.communicate(cert)
t = '-----BEGIN CERTIFICATE-----\n'
t += pcert.encode('base64')
pcert = t + '-----END CERTIFICATE-----\n'
except:
sys.stderr.write("WHACKO ERROR on %s\n" %f)
continue
if err.startswith(MAGIC_ERROR):
sys.stderr.write("failed to load: %s\n" % f)
continue
text, fp = od.opensslParseOneCert(pcert)
moz_verifications = od.verifyCertChain([text], od.MOZ_VERIFY_ARGS)
ms_verifications = od.verifyCertChain([text], od.MS_VERIFY_ARGS)
verifications = zip(moz_verifications, ms_verifications)
hackparse.add_cert_to_db(f, verifications, [text], [fp])
print "SUCCESS ON", f
def train(self, corpus, time_slices, mode='fit', model='fixed'):
"""
Train DTM model using specified corpus and time slices.
'mode' controls the mode of the mode: 'fit' is for training, 'time' for
analyzing documents through time according to a DTM, basically a held out set.
'model' controls the coice of model. 'fixed' is for DIM and 'dtm' for DTM.
"""
self.convert_input(corpus, time_slices)
arguments = "--ntopics={p0} --model={mofrl} --mode={p1} --initialize_lda={p2} --corpus_prefix={p3} --outname={p4} --alpha={p5}".format(
p0=self.num_topics, mofrl=model, p1=mode, p2=self.initialize_lda, p3=self.fcorpus(), p4=self.foutname(), p5=self.alpha)
params = "--lda_max_em_iter={p0} --lda_sequence_min_iter={p1} --lda_sequence_max_iter={p2} --top_chain_var={p3} --rng_seed={p4} ".format(
p0=self.lda_max_em_iter, p1=self.lda_sequence_min_iter, p2=self.lda_sequence_max_iter, p3=self.top_chain_var, p4=self.rng_seed)
arguments = arguments + " " + params
logger.info("training DTM with args %s" % arguments)
try:
p = Popen([self.dtm_path] + arguments.split(), stdout=PIPE, stderr=PIPE)
p.communicate()
except KeyboardInterrupt:
p.terminate()
self.em_steps = np.loadtxt(self.fem_steps())
self.init_alpha = np.loadtxt(self.finit_alpha())
self.init_beta = np.loadtxt(self.finit_beta())
self.init_ss = np.loadtxt(self.flda_ss())
self.lhood_ = np.loadtxt(self.fout_liklihoods())
# document-topic proportions
self.gamma_ = np.loadtxt(self.fout_gamma())
# cast to correct shape, gamme[5,10] is the proprtion of the 10th topic
# in doc 5
self.gamma_.shape = (self.lencorpus, self.num_topics)
# normalize proportions
self.gamma_ /= self.gamma_.sum(axis=1)[:, np.newaxis]
self.lambda_ = np.zeros((self.num_topics, self.num_terms * len(self.time_slices)))
self.obs_ = np.zeros((self.num_topics, self.num_terms * len(self.time_slices)))
for t in range(self.num_topics):
topic = "%03d" % t
self.lambda_[t, :] = np.loadtxt(self.fout_prob().format(i=topic))
self.obs_[t, :] = np.loadtxt(self.fout_observations().format(i=topic))
# cast to correct shape, lambda[5,10,0] is the proportion of the 10th
# topic in doc 5 at time 0
self.lambda_.shape = (self.num_topics, self.num_terms, len(self.time_slices))
self.obs_.shape = (self.num_topics, self.num_terms, len(self.time_slices))
# extract document influence on topics for each time slice
# influences_time[0] , influences at time 0
if model == 'fixed':
for k, t in enumerate(self.time_slices):
stamp = "%03d" % k
influence = np.loadtxt(self.fout_influence().format(i=stamp))
influence.shape = (t, self.num_topics)
# influence[2,5] influence of document 2 on topic 5
self.influences_time.append(influence)
def updaterepo(project, update=True):
if os.path.exists(project.working_copy):
if not update:
return
p = Popen(['git', 'pull'], stdout=PIPE, stderr=PIPE,
cwd=project.working_copy)
stdout, stderr = p.communicate()
if p.returncode != 0:
raise RuntimeError("git pull returned %s: %s" % (p.returncode,
stderr))
else:
return [{'error': False}]
else:
cmd = ['git', 'clone', project.repo_path, project.repo_name]
p = Popen(cmd, stdout=PIPE, stderr=PIPE,
cwd=settings.REPOSITORY_BASE_PATH)
logger.debug('Cloning Git repo {0} for project {1}'.format(
project.repo_path, project))
stdout, stderr = p.communicate()
if p.returncode != 0:
raise RuntimeError("%s returned %s: %s" % (
" ".join(cmd), p.returncode, stderr))
else:
return [{'error': False}]
def load_shell(self, chunk):
"""Run shell commands from code chunks"""
if chunk['evaluate']:
lines = chunk['content'].lstrip().splitlines()
result = "\n"
for line in lines:
command = line.split()
major, minor = sys.version_info[:2]
cmd = Popen(command, stdout=PIPE)
if major == 2 or minor < 3: # Python 2 doesn't have timeout for subprocess
try:
content = cmd.communicate()[0].decode('utf-8').replace("\r", "") + "\n"
except Exception as e:
content = "Pweave ERROR can't execute shell command:\n %s\n" % command
content += str(e)
sys.stdout.write(" Pweave ERROR can't execute shell command:\n %s\n" % line)
print(str(e))
else:
try:
content = cmd.communicate(timeout=20)[0].decode('utf-8').replace("\r", "") + "\n"
except TimeoutExpired:
cmd.kill()
content, errs = cmd.communicate()
sys.stdout.write("Shell command timeout:\n %s\n" % line)
content = content.decode('utf-8').replace("\r", "") + "\n"
if chunk['term']:
result += "$ %s\n" % line
result += content
else:
result = ""
return result
def get_reads(sam, \
contigs = False, mismatches = False, mm_option = False, \
sort_sam = True, req_map = False, region = False, sbuffer = False):
"""
get mapped reads (and their pairs) from an unsorted sam file
"""
tempdir = '%s/' % (os.path.abspath(sam).rsplit('/', 1)[0])
if sort_sam is True:
mapping = '%s.sorted.sam' % (sam.rsplit('.', 1)[0])
if sam != '-':
if os.path.exists(mapping) is False:
os.system("\
sort -k1 --buffer-size=%sG -T %s -o %s %s\
" % (sbuffer, tempdir, mapping, sam))
else:
mapping = 'stdin-sam.sorted.sam'
p = Popen("sort -k1 --buffer-size=%sG -T %s -o %s" \
% (sbuffer, tempdir, mapping), stdin = sys.stdin, shell = True)
p.communicate()
mapping = open(mapping)
else:
if sam == '-':
mapping = sys.stdin
else:
mapping = open(sam)
for read in reads_from_mapping(mapping, contigs, mismatches, mm_option, req_map, region):
yield read
def _run(cmd, args=[], source="", cwd=None, env=None):
if not type(args) is list:
args = [args]
if sys.platform == "win32":
args = [cmd] + args
if sys.version_info[0] == 2:
for i in range(len(args)):
args[i] = args[i].encode(locale.getdefaultlocale()[1])
proc = Popen(args, env=env, cwd=cwd, stdout=PIPE, stdin=PIPE, stderr=PIPE, shell=True)
try:
stat = proc.communicate(input=source)
except:
stat = proc.communicate(input=source.encode("utf8"))
okay = proc.returncode == 0
return {"okay": okay, "out": stat[0].decode(locale.getdefaultlocale()[1]), "err": stat[1].decode(locale.getdefaultlocale()[1])}
else:
if env is None:
env = {"PATH": settings_get('binDir', '/usr/local/bin')}
# adding custom PATHs from settings
customEnv = settings_get('envPATH', "")
if customEnv:
env["PATH"] = env["PATH"]+":"+customEnv
if source == "":
command = [cmd] + args
else:
command = [cmd] + args + [source]
proc = Popen(command, env=env, cwd=cwd, stdout=PIPE, stderr=PIPE)
stat = proc.communicate()
okay = proc.returncode == 0
return {"okay": okay, "out": stat[0].decode('utf-8'), "err": stat[1].decode('utf-8')}
class MongodbPlugin(object):
def __init__(self):
self.mongo = None
self.tmpdir = tempfile.mkdtemp()
def pytest_sessionstart(self, session):
port = session.config.getvalue('mongodb_port')
self.mongo = Popen(["mongod", "--dbpath", self.tmpdir,
"--port", str(port)],
stdin=PIPE, stdout=PIPE, stderr=PIPE)
for each in range(10):
if 'waiting for connections' in self.mongo.stdout.readline():
break
else:
raise OSError('Mongodb start timeout.')
def pytest_sessionfinish(self, session):
if self.mongo is not None:
try:
self.mongo.kill()
self.mongo.communicate()
self.mongo.wait()
finally:
shutil.rmtree(self.tmpdir)
def test_ia_metadata_exists():
nasa_files = ['NASAarchiveLogo.jpg', 'globe_west_540.jpg', 'nasa_reviews.xml',
'nasa_meta.xml', 'nasa_archive.torrent', 'nasa_files.xml']
cmd = 'ia ls nasa'
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
output = [x.strip() for x in stdout.split('\n')]
assert all(f in output for f in nasa_files)
assert proc.returncode == 0
cmd = 'ia ls nasa --glob="*torrent"'
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
assert stdout == 'nasa_archive.torrent\r\n'
assert proc.returncode == 0
cmd = 'ia ls nasa --all --verbose'
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
assert proc.returncode == 0
cmd = 'ia ls nasa --location'
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
assert proc.returncode == 0
def compile(self):
try:
p = Popen([self.control_bin] + self.compile_params,
stdout=self.fout, stderr=self.ferr)
p.communicate(timeout=Server.COMPILE_TIMEOUT)
except:
raise Failed("Can't compile server='%s'" %self.name)
def run_vina_docking(vd_obj):
receptor = ''.join([pdbqt_receptor_path.value,
vd_obj.get_receptor(),
".pdbqt"])
ligand = ''.join([pdbqt_ligand_path.value,
vd_obj.get_ligand(),
".pdbqt"])
output_save = ''.join([path_save_output.value,
vd_obj.get_receptor(),
"_-_",
vd_obj.get_ligand(),
".pdbqt"])
output_log = ''.join([path_save_log.value,
vd_obj.get_receptor(),
"_-_",
vd_obj.get_ligand(),
".log"])
command = "".join([vina_path.value,
" --config ",
config_vina,
" --receptor ",
receptor,
" --ligand ",
ligand,
" --out ",
output_save,
" --log ",
output_log])
proc = Popen(command, shell=True, stdout=PIPE)
proc.communicate()
def run_command(cmd, args = [], source="", cwd = None, env = None):
if not type(args) is list:
args = [args]
if sys.platform == "win32":
proc = Popen([cmd]+args, env=env, cwd=cwd, stdout=PIPE, stdin=PIPE, stderr=PIPE, shell=True)
stat = proc.communicate(input=source)
else:
if env is None:
env = {"PATH": settings.get('binDir', '/usr/local/bin')}
command = [cmd] + args
proc = Popen(command, env=env, cwd=cwd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
stat = proc.communicate(input=source.encode('utf-8'))
okay = proc.returncode == 0
#remove leading empty line
lines = stat[1].decode('UTF-8').split('\n')
if not lines[0]:
lines.pop(0)
sanitized_errout = '\n'.join(lines)
return {"okay": okay, "out": stat[0].decode('UTF-8'), "err": sanitized_errout}
def update(self, r=None):
args = ['svn', 'update']
if r: args += ['-r', str(r)]
args += [self.dir]
p = Popen(args, stdout=PIPE, stdin=PIPE, close_fds=True)
p.communicate()
def daemon_status():
p = Popen(['electrum', 'daemon', 'status'], stdin=PIPE, stdout=PIPE, stderr=PIPE)
p.communicate()
if (p.returncode == 0):
return True
else:
return False
def disable_mod_ssl(self, **kwargs):
self.logger.debug("EXIT: Web.disable_mod_ssl()")
if self.dist_type == 'deb':
if os.path.exists(_enabled_mod_ssl):
disabler = Popen(["/usr/sbin/a2dismod","ssl"],
stdin=None, stdout=PIPE, stderr=PIPE)
(out, err) = disabler.communicate()
if out != "":
self.logger.debug(out)
if err != "":
self.logger.warn(err)
os.remove(_enabled_mod_ssl)
elif self.dist_type == 'suse':
if os.path.exists(_enabled_mod_ssl):
disabler = Popen(["/sbin/yast2","http-server", "module",
"disable=ssl"], stdin=None, stdout=PIPE, stderr=PIPE)
(out, err) = disabler.communicate()
if out != "":
self.logger.debug(out)
if err != "":
self.logger.warn(err)
os.remove(_enabled_mod_ssl)
if os.path.exists(_created_pidfile_symlink):
os.remove(_suse_pidfile_link_name)
os.remove(_created_pidfile_symlink)
self.logger.debug("EXIT: Web.disable_mod_ssl()")
def populate_test_db():
with gzip.open(POPULATE_FP, 'rb') as f:
test_db = f.read()
command = ['psql', '-d', AMGUT_CONFIG.database]
proc = Popen(command, stdin=PIPE, stdout=PIPE)
proc.communicate(test_db)
def stop_services_upgrade(self, is_client):
try:
success = call(["/command/svcs-d"]) == 0
if success:
is_down = False
while not is_down:
print "Waiting for services to stop..."
time.sleep(5)
if is_client:
p = Popen(["/command/svstat", "/service/logmind-shipper"], stdout=PIPE)
out,err = p.communicate()
is_down = out.count("down") == 1
else:
p = Popen("/command/svstats", stdout=PIPE)
out,err = p.communicate()
is_down = out.count("down") == 5
else:
print "Error while stopping services."
return success
except Exception, e:
print "ERROR: ", e
return False
def translatePipeline(toTranslate, commands):
proc_deformat = Popen("apertium-deshtml", stdin=PIPE, stdout=PIPE)
proc_deformat.stdin.write(bytes(toTranslate, "utf-8"))
deformatted = proc_deformat.communicate()[0]
towrite = deformatted
output = []
output.append(toTranslate)
output.append(towrite.decode("utf-8"))
all_cmds = []
all_cmds.append("apertium-deshtml")
for cmd in commands:
proc = Popen(cmd, stdin=PIPE, stdout=PIPE)
proc.stdin.write(towrite)
towrite = proc.communicate()[0]
output.append(towrite.decode("utf-8"))
all_cmds.append(cmd)
proc_reformat = Popen("apertium-rehtml-noent", stdin=PIPE, stdout=PIPE)
proc_reformat.stdin.write(towrite)
towrite = proc_reformat.communicate()[0].decode("utf-8")
output.append(towrite)
all_cmds.append("apertium-rehtml-noent")
return output, all_cmds
def document_to_text(self, filename, file_path):
if filename[-4:] == ".doc":
cmd = ['antiword', file_path]
p = Popen(cmd, stdout=PIPE)
stdout, stderr = p.communicate()
self.raw = stdout.decode('ascii', 'ignore')
elif filename[-5:] == ".docx":
document = opendocx(file_path)
paratextlist = getdocumenttext(document)
newparatextlist = []
for paratext in paratextlist:
newparatextlist.append(paratext.encode("utf-8"))
self.raw = '\n\n'.join(newparatextlist)
elif filename[-4:] == ".odt":
cmd = ['odt2txt', file_path]
p = Popen(cmd, stdout=PIPE)
stdout, stderr = p.communicate()
self.raw = stdout.decode('ascii', 'ignore')
elif filename[-4:] == ".pdf":
self.raw = self.convert_pdf_to_txt(file_path)
elif filename[-4:] == ".txt":
with open(file_path, 'r') as file_:
self.raw = file_.read()
def sort_ref(args):
sys.stderr.write("Sorting in reference genePred\n")
if args.threads > 1:
cmd = ['sort','-S2G','-k3,3','-k5,5n','-k6,6n',
'--parallel='+str(args.threads)]
else:
cmd = ['sort','-S2G','-k3,3','-k5,5n','-k6,6n']
of = open(args.tempdir+'/ref.sorted.gpd','w')
p = Popen(cmd,stdin=PIPE,stdout=of)
refgpd = {}
if args.ref_genepred[-3:] == '.gz':
inf = gzip.open(args.ref_genepred)
else:
inf = open(args.ref_genepred)
#gs = GPDStream(inf)
z = 0
for line in inf:
z += 1
if z%1000==0: sys.stderr.write(str(z)+" \r")
#if z not in refcnt: continue
#if refcnt[z] < args.minimum_read_count: continue
p.stdin.write(line)
#refgpd[z] = gpd
p.communicate()
sys.stderr.write("\n")
inf.close()
def sort_annot(args):
sys.stderr.write("Sorting read annotations\n")
cmd = ['sort','-S2G','-k1,1','-k2,2n','-k3,3n']
cmd2 = 'cut -f 4-'
of0 = open(args.tempdir+'/annot.sorted.txt','w')
p1 = Popen(cmd2.split(),stdin=PIPE,stdout=of0)
p0 = Popen(cmd,stdin=PIPE,stdout=p1.stdin)
inf = None
if is_gzip(args.annotations):
inf = gzip.open(args.annotations)
else:
inf = open(args.annotations)
k = 0
for line in inf:
k+=1
if k%1000==0: sys.stderr.write(str(k)+" \r")
f = line.rstrip().split("\t")
r = GenomicRangeFromString(f[13])
#r.set_payload(parse_annot(f))
p0.stdin.write(r.chr+"\t"+str(r.start)+"\t"+str(r.end)+"\t"+line)
sys.stderr.write("\n")
of0.close()
p0.communicate()
p1.communicate()
inf.close()
def assemble(cls, s, arch):
if arch == 'i386':
cmd_as = ['as', '--32', '--msyntax=intel', '--mnaked-reg', '-o']
cmd_objdump = ['objdump', '-w', '-M', 'intel', '-d']
elif arch == 'x86-64':
cmd_as = ['as', '--64', '--msyntax=intel', '--mnaked-reg', '-o']
cmd_objdump = ['objdump', '-w', '-M', 'intel', '-d']
elif arch == 'arm':
cmd_as = ['as', '-o']
cmd_objdump = ['objdump', '-w', '-d']
elif arch == 'thumb':
cmd_as = ['as', '-mthumb', '-o']
cmd_objdump = ['objdump', '-w', '-d']
else:
raise Exception("unsupported architecture: %r" % arch)
with tempfile.NamedTemporaryFile(delete=False) as f:
p = Popen(cmd_as + [f.name], stdin=PIPE, stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate(s+'\n')
if stderr:
return stderr
p = Popen(cmd_objdump + [f.name], stdout=PIPE)
stdout, stderr = p.communicate()
result = ''.join(stdout.splitlines(True)[7:])
os.remove(f.name)
return result
def clone(self):
# 1. clone the repo
log.debug("GIT_BUILDER: 1. clone".format(self.task.source_type))
cmd = ['git', 'clone', self.task.git_url]
try:
proc = Popen(cmd, stdout=PIPE, stderr=PIPE, cwd=self.tmp)
output, error = proc.communicate()
except OSError as e:
raise GitCloneException(str(e))
if proc.returncode != 0:
raise GitCloneException(error)
# 1b. get dir name
log.debug("GIT_BUILDER: 1b. dir name...")
cmd = ['ls']
try:
proc = Popen(cmd, stdout=PIPE, stderr=PIPE, cwd=self.tmp)
output, error = proc.communicate()
except OSError as e:
raise GitWrongDirectoryException(str(e))
if proc.returncode != 0:
raise GitWrongDirectoryException(error)
if output and len(output.split()) == 1:
git_dir_name = output.split()[0]
else:
raise GitWrongDirectoryException("Could not get name of git directory.")
log.debug("Git directory name: {}".format(git_dir_name))
self.git_dir = "{}/{}".format(self.tmp, git_dir_name)
def enable_mod_wsgi(self, **kwargs):
self.logger.debug("ENTER: Web.enable_mod_wsgi()")
if self.dist_type == 'deb':
if not(os.path.exists("/etc/apache2/mods-enabled/mod_wsgi.load") \
or os.path.exists("/etc/apache2/mods-enabled/wsgi.load")):
enabler = Popen(["/usr/sbin/a2enmod","wsgi"],
stdin=None, stdout=PIPE, stderr=PIPE)
(out, err) = enabler.communicate()
if out != "":
self.logger.debug(out)
if err != "":
self.logger.warn(err)
touched = file(_enabled_mod_wsgi, "w")
touched.close()
if self.http_conf_dir == '/etc/apache2/conf-available' and not \
os.path.exists(
"/etc/apache2/conf-enabled/myproxy-oauth.conf"):
enabler = Popen(["/usr/sbin/a2enconf", "myproxy-oauth"],
stdin=None, stdout=PIPE, stderr=PIPE)
(out, err) = enabler.communicate()
if out != "":
self.logger.debug(out)
if err != "":
self.logger.warn(err)
touched = file(_enabled_myproxy_oauth_conf, "w")
touched.close()
self.logger.debug("EXIT: Web.enable_mod_wsgi()")
def TestProject(ID,versionSet,commands_to_call=""):
mktestdir="mkdir -p TestProj_"+str(ID)
subprocess.call(mktestdir,shell=True)
os.chdir("./TestProj_"+str(ID))
subprocess.call("rm -f MCDispatched.config", shell=True)
print("TESTING PROJECT "+str(ID))
query = "SELECT * FROM Project WHERE ID="+str(ID)
curs.execute(query)
rows=curs.fetchall()
order=rows[0]
print("========================")
print(order["Generator_Config"])
newLoc=CheckGenConfig(order)
print(order["Generator_Config"])
print("========================")
print(newLoc)
if(newLoc!="True"):
curs.execute(query)
rows=curs.fetchall()
order=rows[0]
if(order["RunNumHigh"] != order["RunNumLow"] and order["Generator"]=="file:"):
updatequery="UPDATE Project SET Tested=-1"+" WHERE ID="+str(ID)+";"
curs.execute(updatequery)
conn.commit()
print(bcolors.FAIL+"TEST FAILED"+bcolors.ENDC)
print("rm -rf "+order["OutputLocation"])
return ["oh no!!!",-1,"MCwrapper cannot currently handle an input file with many run numbers properly"]
WritePayloadConfig(order,newLoc)
RunNumber=str(order["RunNumLow"])
if(order["RunNumLow"] != order["RunNumHigh"]):
query_to_do="@is_production and @status_approved"
if(order["RCDBQuery"] != ""):
query_to_do=order["RCDBQuery"]
print("RCDB_QUERY IS: "+str(query_to_do))
#rcdb_db = rcdb.RCDBProvider("mysql://[email protected]/rcdb")
#runList=rcdb_db.select_runs(str(query_to_do),order["RunNumLow"],order["RunNumHigh"]).get_values(['event_count'],True)
RunNumber=str(order["RunNumLow"])#str(runList[0][0])
#if order["RunNumLow"] != order["RunNumHigh"] :
# RunNumber = RunNumber + "-" + str(order["RunNumHigh"])
cleangen=1
if order["SaveGeneration"]==1:
cleangen=0
cleangeant=1
if order["SaveGeant"]==1:
cleangeant=0
cleansmear=1
if order["SaveSmear"]==1:
cleansmear=0
cleanrecon=1
if order["SaveReconstruction"]==1:
cleanrecon=0
command=MCWRAPPER_BOT_HOME+"/gluex_MC.py MCDispatched.config "+str(RunNumber)+" "+str(500)+" per_file=250000 base_file_number=0"+" generate="+str(order["RunGeneration"])+" cleangenerate="+str(cleangen)+" geant="+str(order["RunGeant"])+" cleangeant="+str(cleangeant)+" mcsmear="+str(order["RunSmear"])+" cleanmcsmear="+str(cleansmear)+" recon="+str(order["RunReconstruction"])+" cleanrecon="+str(cleanrecon)+" projid="+str(ID)+" batch=0"
print(command)
STATUS=-1
# print (command+command2).split(" ")
my_env=None
print(versionSet)
if(versionSet != ""):
my_env=source("/group/halld/Software/build_scripts/gluex_env_jlab.sh /group/halld/www/halldweb/html/dist/"+versionSet)
my_env["MCWRAPPER_CENTRAL"]=MCWRAPPER_BOT_HOME
#print(my_env)
if "" in my_env:
split_kv=my_env[""].split("\n")
absorbed=split_kv[len(split_kv)-1].split("=")
print(absorbed)
del my_env[""]
my_env[absorbed[0]]=absorbed[1]
#print(my_env)
p = Popen(commands_to_call+command, env=my_env ,stdin=PIPE,stdout=PIPE, stderr=PIPE,bufsize=-1,shell=True)
#print p
#print "p defined"
output, errors = p.communicate()
#print [p.returncode,errors,output]
output=str(output).replace('\\n', '\n')
STATUS=output.find("Successfully completed")
if(STATUS!=-1):
updatequery="UPDATE Project SET Tested=1"+" WHERE ID="+str(ID)+";"
curs.execute(updatequery)
conn.commit()
if(newLoc != "True"):
updateOrderquery="UPDATE Project SET Generator_Config=\""+newLoc+"\" WHERE ID="+str(ID)+";"
print(updateOrderquery)
curs.execute(updateOrderquery)
conn.commit()
print(bcolors.OKGREEN+"TEST SUCCEEDED"+bcolors.ENDC)
print("rm -rf "+order["OutputLocation"])
#status = subprocess.call("rm -rf "+order["OutputLocation"],shell=True)
else:
updatequery="UPDATE Project SET Tested=-1"+" WHERE ID="+str(ID)+";"
curs.execute(updatequery)
conn.commit()
print(bcolors.FAIL+"TEST FAILED"+bcolors.ENDC)
print("rm -rf "+order["OutputLocation"])
return [output,STATUS,errors]
def get_cpu_temperature():
process = Popen(['vcgencmd', 'measure_temp'], stdout=PIPE)
output, _error = process.communicate()
return float(output[output.index('=') + 1:output.rindex("'")])
def perform_inventory(args):
Variables = dict()
# Parse command line arguments
optlist = []
command_line_length = len(args)
argIndex = 0
inArgument = False
currentArgument = ""
arg = ""
while argIndex < command_line_length:
arg = args[argIndex]
if argIndex == 0:
# skip the program name
argIndex += 1
continue
if inArgument:
Variables[currentArgument] = arg
inArgument = False
else:
if arg[0:2] == "--":
inArgument = True
currentArgument = arg[2:].lower()
else:
# The rest are not options
args = args[argIndex:]
break
argIndex += 1
if inArgument:
Variables[currentArgument] = arg
AcceptableOptions = ["inmof", "outxml", "help"]
if "help" in Variables:
usage()
exit(0)
optionsValid = True
for arg in Variables.keys():
if arg.lower() not in AcceptableOptions:
optionsValid = False
exitWithError("Error: %s is not a valid option" % arg)
if optionsValid == False:
usage()
exit(1)
dsc_sysconfdir = join(helperlib.CONFIG_SYSCONFDIR, helperlib.CONFIG_SYSCONFDIR_DSC)
dsc_reportdir = join(dsc_sysconfdir, 'InventoryReports')
omicli_path = join(helperlib.CONFIG_BINDIR, 'omicli')
dsc_host_base_path = helperlib.DSC_HOST_BASE_PATH
dsc_host_path = join(dsc_host_base_path, 'bin/dsc_host')
dsc_host_output_path = join(dsc_host_base_path, 'output')
dsc_host_lock_path = join(dsc_host_base_path, 'dsc_host_lock')
dsc_host_switch_path = join(dsc_host_base_path, 'dsc_host_ready')
dsc_configuration_path = join(dsc_sysconfdir, 'configuration')
temp_report_path = join(dsc_configuration_path, 'Inventory.xml.temp')
report_path = join(dsc_configuration_path, 'Inventory.xml')
inventorylock_path = join(dsc_sysconfdir, 'inventory_lock')
if ("omsconfig" in helperlib.DSC_SCRIPT_PATH):
write_omsconfig_host_switch_event(pathToCurrentScript, isfile(dsc_host_switch_path))
if ("omsconfig" in helperlib.DSC_SCRIPT_PATH) and (isfile(dsc_host_switch_path)):
use_omsconfig_host = True
else:
use_omsconfig_host = False
if "outxml" in Variables:
report_path = Variables["outxml"]
parameters = []
if use_omsconfig_host:
parameters.append(dsc_host_path)
parameters.append(dsc_host_output_path)
if "inmof" in Variables:
parameters.append("PerformInventoryOOB")
parameters.append(Variables["inmof"])
else:
parameters.append("PerformInventory")
else:
parameters.append(omicli_path)
parameters.append("iv")
parameters.append(helperlib.DSC_NAMESPACE)
parameters.append("{")
parameters.append("MSFT_DSCLocalConfigurationManager")
parameters.append("}")
if "inmof" in Variables:
parameters.append("PerformInventoryOOB")
parameters.append("{")
parameters.append("InventoryMOFPath")
parameters.append(Variables["inmof"])
parameters.append("}")
else:
parameters.append("PerformInventory")
# Ensure inventory lock file permission is set correctly before opening
operationStatusUtility.ensure_file_permissions(inventorylock_path, '644')
# Open the inventory lock file. This also creates a file if it does not exist.
inventorylock_filehandle = open(inventorylock_path, 'w')
printVerboseMessage("Opened the inventory lock file at the path '" + inventorylock_path + "'")
retval = 0
inventorylock_acquired = True
dschostlock_filehandle = None
inmof_file = ''
if "inmof" in Variables:
inmof_file = Variables["inmof"]
try:
# Acquire inventory file lock
try:
flock(inventorylock_filehandle, LOCK_EX | LOCK_NB)
write_omsconfig_host_log('Inventory lock is acquired by : ' + inmof_file, pathToCurrentScript)
except IOError:
inventorylock_acquired = False
write_omsconfig_host_log('Failed to acquire inventory lock.', pathToCurrentScript, 'WARNING')
if inventorylock_acquired:
dschostlock_acquired = False
if use_omsconfig_host:
if isfile(dsc_host_lock_path):
stop_old_host_instances(dsc_host_lock_path)
# Open the dsc host lock file. This also creates a file if it does not exist.
dschostlock_filehandle = open(dsc_host_lock_path, 'w')
printVerboseMessage("Opened the dsc host lock file at the path '" + dsc_host_lock_path + "'")
# Acquire dsc host file lock
for retry in range(10):
try:
flock(dschostlock_filehandle, LOCK_EX | LOCK_NB)
dschostlock_acquired = True
write_omsconfig_host_log('dsc_host lock file is acquired by : ' + inmof_file, pathToCurrentScript)
break
except IOError:
write_omsconfig_host_log('dsc_host lock file not acquired. retry (#' + str(retry) + ') after 60 seconds...', pathToCurrentScript)
sleep(60)
else:
write_omsconfig_host_log('dsc_host lock file does not exist. Skipping this operation until next consistency hits.', pathToCurrentScript, 'WARNING')
if dschostlock_acquired or (not use_omsconfig_host):
try:
system("rm -f " + dsc_reportdir + "/*")
process = Popen(parameters, stdout = PIPE, stderr = PIPE)
stdout, stderr = process.communicate()
retval = process.returncode
stdout = stdout.decode() if isinstance(stdout, bytes) else stdout
stderr = stderr.decode() if isinstance(stderr, bytes) else stderr
printVerboseMessage(stdout)
if (retval > 0):
write_omsconfig_host_log('dsc_host failed with code = ' + str(retval), pathToCurrentScript)
exit(retval)
# Combine reports together
reportFiles = listdir(dsc_reportdir)
final_xml_report = '<INSTANCE CLASSNAME="Inventory"><PROPERTY.ARRAY NAME="Instances" TYPE="string" EmbeddedObject="object"><VALUE.ARRAY>'
values = []
for reportFileName in reportFiles:
reportFilePath = join(dsc_reportdir, reportFileName)
if not isfile(reportFilePath):
continue
report = parse(reportFilePath)
for valueNode in report.getElementsByTagName('VALUE'):
values.append(valueNode.toxml())
final_xml_report = final_xml_report + "".join(values) + "</VALUE.ARRAY></PROPERTY.ARRAY></INSTANCE>"
# Ensure temporary inventory report file permission is set correctly before opening
operationStatusUtility.ensure_file_permissions(temp_report_path, '644')
tempReportFileHandle = open(temp_report_path, 'w')
try:
tempReportFileHandle.write(final_xml_report)
finally:
if (tempReportFileHandle):
tempReportFileHandle.close()
# Ensure temporary inventory report file permission is set correctly after opening
operationStatusUtility.ensure_file_permissions(temp_report_path, '644')
system("rm -f " + dsc_reportdir + "/*")
move(temp_report_path, report_path)
# Ensure inventory report file permission is set correctly
operationStatusUtility.ensure_file_permissions(report_path, '644')
finally:
if (dschostlock_filehandle):
# Release inventory file lock
flock(inventorylock_filehandle, LOCK_UN)
# Release dsc host file lock
if isfile(dsc_host_lock_path) and use_omsconfig_host:
try:
flock(dschostlock_filehandle, LOCK_UN)
except:
pass
finally:
if (inventorylock_filehandle):
# Close inventory lock file handle
inventorylock_filehandle.close()
if (dschostlock_filehandle):
# Close dsc host lock file handle
if use_omsconfig_host:
try:
dschostlock_filehandle.close()
except:
pass
# Ensure inventory lock file permission is set correctly after opening
operationStatusUtility.ensure_file_permissions(inventorylock_path, '644')
# Ensure dsc host lock file permission is set correctly after opening
if use_omsconfig_host:
operationStatusUtility.ensure_file_permissions(dsc_host_lock_path, '644')
exit(retval)
def minisat(cmd):
p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=True)
p.wait()
output, err = p.communicate()
return str(output)
def get_video_urls(self, user_id, type_flag='f'):
"""
获得视频播放地址
Parameters:
user_id：查询的用户ID
Returns:
video_names: 视频名字列表
video_urls: 视频链接列表
nickname: 用户昵称
"""
video_names = []
video_urls = []
share_urls = []
unique_id = ''
max_cursor = 0
has_more = 1
device_info = self.getDevice()
APPINFO = {
'version_code': '2.7.0',
'app_version': '2.7.0',
'channel': 'App%20Stroe',
'app_name': 'aweme',
'build_number': '27014',
'aid': '1128'
}
params = {
'iid': device_info['iid'],
'idfa': device_info['idfa'],
'vid': device_info['vid'],
'device_id': device_info['device_id'],
'openudid': device_info['openudid'],
'device_type': device_info['device_type'],
'os_version': device_info['os_version'],
'os_api': device_info['os_api'],
'screen_width': device_info['screen_width'],
'device_platform': device_info['device_platform'],
'version_code': APPINFO['version_code'],
'channel': APPINFO['channel'],
'app_name': APPINFO['app_name'],
'build_number': APPINFO['build_number'],
'app_version': APPINFO['app_version'],
'aid': APPINFO['aid'],
'ac': 'WIFI',
'count': '12',
'keyword': user_id,
'offset': '0'
}
print('解析视频链接中')
query = self.params2str(params)
if not os.path.isfile('douyin.txt'):
self.getToken()
token = self.load_json()['token']
sign = self.getSign(token, query)
while not sign:
self.getToken()
token = self.load_json()['token']
sign = self.getSign(token, query)
params['mas'] = sign['mas']
params['as'] = sign['as']
params['ts'] = sign['ts']
headers = {'User-Agent': 'Aweme/2.7.0 (iPhone; iOS 11.0; Scale/2.00)'}
req = requests.get('https://api.amemv.com/aweme/v1/general/search/',
params=params,
headers=headers)
html = json.loads(req.text)
uid = html['user_list'][0]['user_info']['uid']
nickname = html['user_list'][0]['user_info']['nickname']
unique_id = html['user_list'][0]['user_info']['unique_id']
if unique_id != user_id:
unique_id = html['user_list'][0]['user_info']['short_id']
if unique_id != user_id:
print('用户ID可能输入错误或无法搜索到此用户ID')
sys.exit()
share_user_url = 'https://www.amemv.com/share/user/%s' % uid
share_user = requests.get(share_user_url, headers=self.headers)
_dytk_re = re.compile(r"dytk:\s*'(.+)'")
dytk = _dytk_re.search(share_user.text).group(1)
urllib.request.urlretrieve(
'https://raw.githubusercontent.com/Jack-Cherish/python-spider/master/douyin/f**k-byted-acrawler.js',
'f**k-byted-acrawler.js')
try:
process = Popen(['node', 'f**k-byted-acrawler.js',
str(uid)],
stdout=PIPE,
stderr=PIPE)
except (OSError, IOError) as err:
print('请先安装 node.js: https://nodejs.org/')
sys.exit()
_sign = process.communicate()[0].decode().strip('\n').strip('\r')
del params['keyword']
del params['offset']
params['count'] = '21'
params['user_id'] = uid
user_url_prefix = 'https://www.amemv.com/aweme/v1/aweme/favorite' if type_flag == 'f' else 'https://aweme.snssdk.com/aweme/v1/aweme/post/'
while has_more != 0:
if type_flag == 'f':
user_url = user_url_prefix + '/?user_id=%s&max_cursor=%s&count=21&aid=1128&_signature=%s&dytk=%s' % (
uid, max_cursor, _sign, dytk)
req = requests.get(user_url, headers=self.headers)
while req.status_code != 200:
req = requests.get(user_url, headers=self.headers)
html = json.loads(req.text)
else:
params['max_cursor'] = max_cursor
req = requests.get(user_url_prefix,
params=params,
headers=headers)
while req.status_code != 200:
req = requests.get(user_url_prefix,
params=params,
headers=headers)
html = json.loads(req.text)
while html['status_code'] != 0:
req = requests.get(user_url_prefix,
params=params,
headers=headers)
while req.status_code != 200:
req = requests.get(user_url_prefix,
params=params,
headers=headers)
html = json.loads(req.text)
for each in html['aweme_list']:
try:
if type_flag == 'f':
video_url = each['video']['play_addr']['url_list'][0]
share_desc = each['share_info']['share_desc']
else:
video_url = each['video']['bit_rate'][0]['play_addr'][
'url_list'][2]
share_desc = each['desc']
except:
continue
if os.name == 'nt':
for c in r'\/:*?"<>|':
nickname = nickname.replace(c, '').strip().strip('\.')
share_desc = share_desc.replace(c, '').strip()
share_id = each['aweme_id']
if share_desc in ['抖音-原创音乐短视频社区', 'TikTok', '']:
video_names.append(share_id + '.mp4')
else:
video_names.append(share_id + '-' + share_desc + '.mp4')
share_urls.append(each['share_info']['share_url'])
video_urls.append(video_url)
max_cursor = html['max_cursor']
has_more = html['has_more']
return video_names, video_urls, share_urls, nickname
def decrypt_file(self):
"""Decrypts encrypted PDF files
This method begins by testing the password value. If
no password is provided, the default password value is
stored. A temporary directory is created to store the
decrypted version of the PDF file, and any previously
existing temporary directories and files are removed.
The QPDF command is then invoked to decrypt the file,
saving the decrypted file in the temporary directory.
If the QPDF command fails, then the original file is
copied into the temporary directory for use with sub-
sequent functions.
Args:
file_path: The path (can be full or abbreviated) of
the file to be tested.
temp_path: The hard-coded truncated path of the
temporary PDF files used for decryption and metadata
extraction.
output_path: The path (full or truncated) of the
output file.
password: The password used to decrypt an encrypted
PDF file. This value defaults to None is no input
is provided.
Returns:
This method returns a tuple with four elements: (1)
the return code pushed by the QPDF command, (2) the
password used in the command to decrypt the file, (3)
any output generated by the command, and (4) any
error messages generated by the command.
"""
# capture input password
if self.password is None:
pw = ""
else:
pw = str(self.password)
# remove and re-create temporary directory
if os.path.isfile(self.temp_path):
os.remove(self.temp_path)
if os.path.isdir('.tmp/'):
pass
else:
os.mkdir('.tmp/')
# decrypt PDF file into temporary directory
qpdf = "qpdf --password={0} --decrypt '{1}' '{2}'".format(
pw, self.file_path, self.temp_path)
args = shlex.split(qpdf)
run_qpdf = Popen(args, stdout=PIPE, stderr=PIPE)
# capture and pass output and return codes
output, error = run_qpdf.communicate()
return_code = run_qpdf.returncode
# run copy function is decryption fails
if os.path.isfile(self.temp_path):
pass
else:
copyfile(self.file_path, self.temp_path)
return (return_code, pw, output, error)
def execute(cmd): # cmd is string like in command line
proc = Popen(cmd.split(" "), stdout=PIPE, stderr=PIPE)
s = proc.communicate()
# return console output
return s[0]
def _execute_shell(self, cli):
# don't what to have any dependencies on main project
popen_cli = filter(lambda x: True if len(x) > 0 else False, cli.split(" "))
process = Popen(popen_cli, stdout=PIPE, stderr=STDOUT, cwd=getcwd())
out, err = process.communicate()
return process.returncode, out.decode('utf-8')
def pcap_pdml_html(pcap_md5, analyst):
# check to see if there is a File object with the source reference of
# 'tshark_pdml.html'. If there is, return it.
# If not, generate it, save it, and return it.
pcap = PCAP.objects(md5=pcap_md5).first()
if not pcap:
return "No PCAP found"
else:
coll = settings.COL_OBJECTS
pdml_obj = None
pdml_html = None
for obj in pcap.obj:
for source in obj.source:
for instance in source.instances:
if instance.reference == 'tshark_pdml.html':
pdml_obj = obj
if not pdml_obj:
sc = get_config('MetaCap')
tshark_bin = str(sc['tshark'])
if not os.path.exists(tshark_bin):
pdml_html = "Could not find tshark!"
return {'html': pdml_html}
pcap_data = pcap.filedata.read()
if not pcap_data:
pdml_html = "Could not get PCAP from GridFS: %s" % pcap_md5
return {'html': pdml_html}
# write PCAP to disk
temp_pcap = tempfile.NamedTemporaryFile(delete=False)
pcap_name = temp_pcap.name
temp_pcap.write(pcap_data)
temp_pcap.close()
# use tshark to generate a pdml file
temp_pdml = tempfile.NamedTemporaryFile(delete=False)
args = [tshark_bin, "-n", "-r", pcap_name, "-T", "pdml"]
tshark = Popen(args, stdout=temp_pdml, stderr=PIPE)
tshark_out, tshark_err = tshark.communicate()
if tshark.returncode != 0:
return {'html': "%s, %s" % (tshark_out, tshark_err)}
pdml_name = temp_pdml.name
temp_pdml.seek(0)
# transform PDML into HTML
xsl_file = None
for d in settings.SERVICE_DIRS:
try:
file_dir = "%s/metacap_service" % d
xsl_file = open('%s/pdml2html.xsl' % file_dir, 'r')
except IOError:
pass
if not xsl_file:
return {'html': 'Could not find XSL.'}
parser = etree.XMLParser()
parser.resolvers.add(FileResolver())
save_pdml = False
try:
xml_input = etree.parse(temp_pdml, parser)
xslt_root = etree.parse(xsl_file, parser)
transform = etree.XSLT(xslt_root)
pdml_html = str(transform(xml_input))
save_pdml = True
except Exception:
temp_pdml.close()
# delete PDML file
os.unlink(pdml_name)
os.unlink(pcap_name)
return {'html': 'Could not parse/transform PDML output!'}
temp_pdml.close()
# delete PDML file
os.unlink(pdml_name)
os.unlink(pcap_name)
# save pdml_html as an object for this PCAP
if save_pdml:
fn = put_file_gridfs('tshark_pdml.html',
pdml_html,
collection=coll)
if fn:
m = hashlib.md5()
m.update(pdml_html)
md5 = m.hexdigest()
pcap.add_object(ObjectTypes.FILE_UPLOAD, md5,
get_user_organization(analyst), "MetaCap",
'tshark_pdml.html', analyst)
pcap.save()
else:
# get file from gridfs and return it
obj_md5 = pdml_obj.value
pdml_html = get_file_gridfs(obj_md5, collection=coll)
if not pdml_html:
return {'html': 'No file found in GridFS'}
if not pdml_obj:
pcap_objects = pcap.sort_objects()
return {'html': pdml_html, 'objects': pcap_objects, 'id': pcap.id}
else:
return {'html': pdml_html}
def pcap_tcpdump(pcap_md5, form, analyst):
flag_list = []
cleaned_data = form.cleaned_data
# Make sure we can find tcpdump
sc = get_config('MetaCap')
tcpdump_bin = str(sc['tcpdump'])
if not os.path.exists(tcpdump_bin):
tcpdump_output = "Could not find tcpdump!"
return tcpdump_output
# Make sure we have a PCAP to work with
pcap = PCAP.objects(md5=pcap_md5).first()
if not pcap:
return "No PCAP found"
pcap_data = pcap.filedata.read()
if not pcap_data:
return "Could not get PCAP from GridFS: %s" % pcap_md5
# Use the filename if it's there, otherwise the md5.
# This is used for the description of the carved sample.
if pcap.filename:
pcap_filename = pcap.filename
else:
pcap_filename = pcap_md5
# Setup tcpdump arguments
if cleaned_data['sequence']:
flag_list.append("-S")
if cleaned_data['timestamp']:
flag_list.append("%s" % cleaned_data['timestamp'])
if cleaned_data['verbose']:
flag_list.append("%s" % cleaned_data['verbose'])
if cleaned_data['data']:
flag_list.append("%s" % cleaned_data['data'])
# force -nN
flag_list.append("-nN")
# if we need to carve
if cleaned_data['carve']:
if not cleaned_data['bpf']:
return "Must supply a BPF filter to carve."
new_pcap = tempfile.NamedTemporaryFile(delete=False)
flag_list.append("-w")
flag_list.append(new_pcap.name)
if cleaned_data['bpf']:
flag_list.append('%s' % str(cleaned_data['bpf'].replace('"', '')))
# write PCAP to disk
# temp_out collects stdout and stderr
# temp_pcap is the pcap to read
# new_pcap is the pcap being written if carving
temp_out = tempfile.NamedTemporaryFile(delete=False)
temp_pcap = tempfile.NamedTemporaryFile(delete=False)
pcap_name = temp_pcap.name
temp_pcap.write(pcap_data)
temp_pcap.close()
args = [tcpdump_bin, '-r', temp_pcap.name] + flag_list
tcpdump = Popen(args, stdout=temp_out, stderr=STDOUT)
tcpdump.communicate()
out_name = temp_out.name
temp_out.seek(0)
tcpdump_output = ''
for line in iter(temp_out):
tcpdump_output += "%s" % line
temp_out.close()
#delete temp files
os.unlink(pcap_name)
os.unlink(out_name)
if cleaned_data['carve']:
new_pcap_data = new_pcap.read()
if len(new_pcap_data) > 24: # pcap-ng will change this.
m = hashlib.md5()
m.update(new_pcap_data)
md5 = m.hexdigest()
org = get_user_organization(analyst)
result = handle_pcap_file("%s.pcap" % md5,
new_pcap_data,
org,
user=analyst,
description="%s of %s" %
(cleaned_data['bpf'], pcap_filename),
parent_id=pcap.id,
parent_type="PCAP",
method="MetaCap Tcpdumper")
if result['success']:
tcpdump_output = "<a href=\"%s\">View new pcap.</a>" % reverse(
'crits.pcaps.views.pcap_details', args=[result['md5']])
else:
tcpdump_output = result['message']
else:
tcpdump_output = "No packets matched the filter."
os.unlink(new_pcap.name)
return tcpdump_output
# be cleaned up by proc.kill(). Popen, PIPE, and TimeoutExpired have all been imported for you.
# 1. Start a long running process using subprocess.Popen().
# 2. Use subprocess.communicate() to create a timeout.
# 3. Cleanup the process if it takes longer than the timeout.
# 4. Print error message and standard out and standard error streams.
from subprocess import (PIPE, Popen)
# Start a long running process using subprocess.Popen()
proc = Popen(["sleep", "6"], stdout=PIPE, stderr=PIPE)
# Use subprocess.communicate() to create a timeout
try:
output, error = proc.communicate(timeout=5)
except TimeoutExpired:
# Cleanup the process if it takes longer than the timeout
proc.kill()
# Read standard out and standard error streams and print
output, error = proc.communicate()
print(f"Process timed out with output: {output}, error: {error}")
# <script.py> output:
# Process timed out with output: b'', error: b''
# You were able to catch the process as it timed out.
if in_msgstr:
messages.append((msgid, msgstr))
return messages
files = sys.argv[1:]
# xgettext -n --keyword=_ $FILES
XGETTEXT=os.getenv('XGETTEXT', 'xgettext')
if not XGETTEXT:
print('Cannot extract strings: xgettext utility is not installed or not configured.',file=sys.stderr)
print('Please install package "gettext" and re-run \'./configure\'.',file=sys.stderr)
exit(1)
child = Popen([XGETTEXT,'--output=-','-n','--keyword=_'] + files, stdout=PIPE)
(out, err) = child.communicate()
messages = parse_po(out.decode('utf-8'))
f = open(OUT_CPP, 'w')
f.write("""
#include <QtGlobal>
// Automatically generated by extract_strings.py
#ifdef __GNUC__
#define UNUSED __attribute__((unused))
#else
#define UNUSED
#endif
""")
def display( self ):
ppm_name = 'pic.ppm'
self.save_ppm( ppm_name )
p = Popen( ['display', ppm_name], stdin=PIPE, stdout = PIPE )
p.communicate()
remove(ppm_name)
def render_dot(self, code, options, format, prefix='graphviz'):
# type: (nodes.NodeVisitor, unicode, Dict, unicode, unicode) -> Tuple[unicode, unicode]
"""Render graphviz code into a PNG or PDF output file."""
graphviz_dot = options.get('graphviz_dot',
self.builder.config.graphviz_dot)
hashkey = (code + str(options) + str(graphviz_dot) +
str(self.builder.config.graphviz_dot_args)).encode('utf-8')
fname = '%s-%s.%s' % (prefix, sha1(hashkey).hexdigest(), format)
relfn = posixpath.join(self.builder.imgpath, fname)
outfn = path.join(self.builder.outdir, self.builder.imagedir, fname)
if path.isfile(outfn):
return relfn, outfn
if (hasattr(self.builder, '_graphviz_warned_dot')
and self.builder._graphviz_warned_dot.get(graphviz_dot)):
return None, None
ensuredir(path.dirname(outfn))
# graphviz expects UTF-8 by default
if isinstance(code, text_type):
code = code.encode('utf-8')
dot_args = [graphviz_dot]
dot_args.extend(self.builder.config.graphviz_dot_args)
dot_args.extend(['-T' + format, '-o' + outfn])
if format == 'png':
dot_args.extend(['-Tcmapx', '-o%s.map' % outfn])
try:
p = Popen(dot_args, stdout=PIPE, stdin=PIPE, stderr=PIPE)
except OSError as err:
if err.errno != ENOENT: # No such file or directory
raise
logger.warning(
__('dot command %r cannot be run (needed for graphviz '
'output), check the graphviz_dot setting'), graphviz_dot)
if not hasattr(self.builder, '_graphviz_warned_dot'):
self.builder._graphviz_warned_dot = {}
self.builder._graphviz_warned_dot[graphviz_dot] = True
return None, None
try:
# Graphviz may close standard input when an error occurs,
# resulting in a broken pipe on communicate()
stdout, stderr = p.communicate(code)
except (OSError, IOError) as err:
if err.errno not in (EPIPE, EINVAL):
raise
# in this case, read the standard output and standard error streams
# directly, to get the error message(s)
stdout, stderr = p.stdout.read(), p.stderr.read()
p.wait()
if p.returncode != 0:
raise GraphvizError(
__('dot exited with error:\n[stderr]\n%s\n'
'[stdout]\n%s') % (stderr, stdout))
if not path.isfile(outfn):
raise GraphvizError(
__('dot did not produce an output file:\n[stderr]\n%s\n'
'[stdout]\n%s') % (stderr, stdout))
return relfn, outfn
def check_code_snippets_in_file(self, doc_path):
# Extract lines of Python code from the README.md file.
lines = []
num_code_lines = 0
num_code_lines_in_block = 0
is_code = False
is_md = False
is_rst = False
last_line = ''
with open(doc_path) as doc_file:
for orig_line in doc_file:
line = orig_line.strip('\n')
if line.startswith('```python'):
# Start markdown code block.
if is_rst:
self.fail(
"Markdown code block found after restructured text block in same file."
)
is_code = True
is_md = True
line = ''
num_code_lines_in_block = 0
elif is_code and is_md and line.startswith('```'):
# Finish markdown code block.
if not num_code_lines_in_block:
self.fail("No lines of code in block")
is_code = False
line = ''
elif is_code and is_rst and line.startswith('```'):
# Can't finish restructured text block with markdown.
self.fail(
"Restructured text block terminated with markdown format '```'"
)
elif line.startswith(
'.. code:: python'
) and 'exclude-when-testing' not in last_line:
# Start restructured text code block.
if is_md:
self.fail(
"Restructured text code block found after markdown block in same file."
)
is_code = True
is_rst = True
line = ''
num_code_lines_in_block = 0
elif is_code and is_rst and line and not line.startswith(' '):
# Finish restructured text code block.
if not num_code_lines_in_block:
self.fail("No lines of code in block")
is_code = False
line = ''
elif is_code:
# Process line in code block.
if is_rst:
# Restructured code block normally indented with four spaces.
if len(line.strip()):
if not line.startswith(' '):
self.fail(
"Code line needs 4-char indent: {}: {}".
format(repr(line), doc_path))
# Strip four chars of indentation.
line = line[4:]
if len(line.strip()):
num_code_lines_in_block += 1
num_code_lines += 1
else:
line = ''
lines.append(line)
last_line = orig_line
print("{} lines of code in {}".format(num_code_lines, doc_path))
# Write the code into a temp file.
tempfile = NamedTemporaryFile('w+')
temp_path = tempfile.name
tempfile.writelines("\n".join(lines) + '\n')
tempfile.flush()
# Run the code and catch errors.
p = Popen([sys.executable, temp_path], stdout=PIPE, stderr=PIPE)
out, err = p.communicate()
out = out.decode('utf8')
err = err.decode('utf8')
out = out.replace(temp_path, doc_path)
err = err.replace(temp_path, doc_path)
exit_status = p.wait()
print(out)
print(err)
# Check for errors running the code.
if exit_status:
self.fail(out + err)
# Close (deletes) the tempfile.
tempfile.close()
def save_extension( self, fname ):
ppm_name = fname[:fname.find('.')] + '.ppm'
self.save_ppm( ppm_name )
p = Popen( ['convert', ppm_name, fname ], stdin=PIPE, stdout = PIPE )
p.communicate()
remove(ppm_name)
def run_script(self, script, arg_str, catch_stderr=False, timeout=None):
'''
Execute a script with the given argument string
'''
path = os.path.join(SCRIPT_DIR, script)
if not os.path.isfile(path):
return False
ppath = 'PYTHONPATH={0}:{1}'.format(CODE_DIR, ':'.join(sys.path[1:]))
cmd = '{0} {1} {2} {3}'.format(ppath, PYEXEC, path, arg_str)
popen_kwargs = {
'shell': True,
'stdout': PIPE
}
if catch_stderr is True:
popen_kwargs['stderr'] = PIPE
if not sys.platform.lower().startswith('win'):
popen_kwargs['close_fds'] = True
def detach_from_parent_group():
# detach from parent group (no more inherited signals!)
os.setpgrp()
popen_kwargs['preexec_fn'] = detach_from_parent_group
elif sys.platform.lower().startswith('win') and timeout is not None:
raise RuntimeError('Timeout is not supported under windows')
process = Popen(cmd, **popen_kwargs)
if timeout is not None:
stop_at = datetime.now() + timedelta(seconds=timeout)
term_sent = False
while True:
process.poll()
if process.returncode is not None:
break
if datetime.now() > stop_at:
if term_sent is False:
# Kill the process group since sending the term signal
# would only terminate the shell, not the command
# executed in the shell
os.killpg(os.getpgid(process.pid), signal.SIGINT)
term_sent = True
continue
# As a last resort, kill the process group
os.killpg(os.getpgid(process.pid), signal.SIGKILL)
out = [
'Process took more than {0} seconds to complete. '
'Process Killed!'.format(timeout)
]
if catch_stderr:
return out, [
'Process killed, unable to catch stderr output'
]
return out
if catch_stderr:
if sys.version_info < (2, 7):
# On python 2.6, the subprocess'es communicate() method uses
# select which, is limited by the OS to 1024 file descriptors
# We need more available descriptors to run the tests which
# need the stderr output.
# So instead of .communicate() we wait for the process to
# finish, but, as the python docs state "This will deadlock
# when using stdout=PIPE and/or stderr=PIPE and the child
# process generates enough output to a pipe such that it
# blocks waiting for the OS pipe buffer to accept more data.
# Use communicate() to avoid that." <- a catch, catch situation
#
# Use this work around were it's needed only, python 2.6
process.wait()
out = process.stdout.read()
err = process.stderr.read()
else:
out, err = process.communicate()
# Force closing stderr/stdout to release file descriptors
process.stdout.close()
process.stderr.close()
try:
return out.splitlines(), err.splitlines()
finally:
try:
process.terminate()
except OSError, err:
# process already terminated
pass
def cmd(self, command, **kwargs):
# prepare the environ, based on the system + our own env
env = copy(environ)
env.update(self.environ)
# prepare the process
kwargs.setdefault('env', env)
kwargs.setdefault('stdout', PIPE)
kwargs.setdefault('stderr', PIPE)
kwargs.setdefault('close_fds', True)
kwargs.setdefault('shell', True)
kwargs.setdefault('show_output', self.log_level > 1)
show_output = kwargs.pop('show_output')
get_stdout = kwargs.pop('get_stdout', False)
get_stderr = kwargs.pop('get_stderr', False)
break_on_error = kwargs.pop('break_on_error', True)
sensible = kwargs.pop('sensible', False)
if not sensible:
self.debug('Run {0!r}'.format(command))
else:
if type(command) in (list, tuple):
self.debug('Run {0!r} ...'.format(command[0]))
else:
self.debug('Run {0!r} ...'.format(command.split()[0]))
self.debug('Cwd {}'.format(kwargs.get('cwd')))
# open the process
if sys.platform == 'win32':
kwargs.pop('close_fds', None)
process = Popen(command, **kwargs)
# prepare fds
fd_stdout = process.stdout.fileno()
fd_stderr = process.stderr.fileno()
if fcntl:
fcntl.fcntl(fd_stdout, fcntl.F_SETFL,
fcntl.fcntl(fd_stdout, fcntl.F_GETFL) | os.O_NONBLOCK)
fcntl.fcntl(fd_stderr, fcntl.F_SETFL,
fcntl.fcntl(fd_stderr, fcntl.F_GETFL) | os.O_NONBLOCK)
ret_stdout = [] if get_stdout else None
ret_stderr = [] if get_stderr else None
while True:
try:
readx = select.select([fd_stdout, fd_stderr], [], [])[0]
except select.error:
break
if fd_stdout in readx:
chunk = process.stdout.read()
if not chunk:
break
if get_stdout:
ret_stdout.append(chunk)
if show_output:
stdout.write(chunk.decode('utf-8', 'replace'))
if fd_stderr in readx:
chunk = process.stderr.read()
if not chunk:
break
if get_stderr:
ret_stderr.append(chunk)
if show_output:
stderr.write(chunk.decode('utf-8', 'replace'))
stdout.flush()
stderr.flush()
process.communicate()
if process.returncode != 0 and break_on_error:
self.error('Command failed: {0}'.format(command))
self.log_env(self.ERROR, kwargs['env'])
self.error('')
self.error('Buildozer failed to execute the last command')
if self.log_level <= self.INFO:
self.error(
'If the error is not obvious, please raise the log_level to 2'
)
self.error('and retry the latest command.')
else:
self.error(
'The error might be hidden in the log above this error')
self.error(
'Please read the full log, and search for it before')
self.error('raising an issue with buildozer itself.')
self.error(
'In case of a bug report, please add a full log with log_level = 2'
)
raise BuildozerCommandException()
if ret_stdout:
ret_stdout = b''.join(ret_stdout)
if ret_stderr:
ret_stderr = b''.join(ret_stderr)
return (ret_stdout.decode('utf-8', 'ignore') if ret_stdout else None,
ret_stderr.decode('utf-8') if ret_stderr else None,
process.returncode)
if options.checkIfOnT2:
print "### WARNING: make sure you have loaded CRAB3 and a valid CMS VO proxy ###"
print "Check if the samples are available on T2"
for sample in samples:
PrimaryDataset, ProcessedDataset, DataTier = filter(None, sample.split("/"))
print "Checking availability on T2/T3 of dataset ", PrimaryDataset
call(["das_client.py --query \"site dataset=" + sample + "\" | grep T[2-3]"], shell=True)
if options.checkNFiles:
print "### WARNING: make sure you have loaded CRAB3 and a valid CMS VO proxy ###"
print "Check how many files to be processed there are per sample"
for sample in samples:
PrimaryDataset, ProcessedDataset, DataTier = filter(None, sample.split("/"))
p = Popen(["das_client.py --query \"file dataset=" + sample + "\" --limit=0 | grep store | wc -l"], shell=True, stdout=PIPE)
out, err = p.communicate()
print "Number of files for ", PrimaryDataset, out
if options.createCrabConfig:
print ("\nCreating CRAB configurations in %s" % options.campaign)
print ("--------------------------------------------------------")
if not options.task_dir:
options.task_dir = options.campaign
if not os.path.isdir(options.task_dir):
os.mkdir(options.task_dir)
os.chdir(options.task_dir)
print ("Parameter set: %s\nflashggVersion: %s\ncrab template: %s\n" % (options.parameterSet,flashggVersion,options.crabTemplate))
print ("Copying over parameter set")
Popen(['cp', '-p', options.parameterSet, './'])
rel = os.environ.get('CMSSW_BASE')
def execute(self, workingDir, reuse=True):
"""Generate sun matrix.
Args:
workingDir: Folder to execute and write the output.
reuse: Reuse the matrix if already existed in the folder.
Returns:
Full path to analemma, sunlist and sunmatrix.
"""
fp = os.path.join(workingDir, self.analemmafile)
lfp = os.path.join(workingDir, self.sunlistfile)
mfp = os.path.join(workingDir, self.sunmtxfile)
hrf = os.path.join(workingDir, self.name + '.hrs')
if reuse:
if self.hoursMatch(hrf):
for f in (fp, lfp, mfp):
if not os.path.isfile(f):
break
else:
print('Found the sun matrix!')
return fp, lfp, mfp
with open(hrf, 'wb') as outf:
outf.write(','.join(str(h) for h in self.hoys) + '\n')
# written based on scripts/analemma provided by @sariths
wea = self.wea
monthDateTime = (DateTime.fromHoy(idx) for idx in self.hoys)
latitude, longitude = wea.location.latitude, -wea.location.longitude
meridian = -(15 * wea.location.timezone)
gdlit = Gendaylit(rotation=self.north)
gdlit.gendaylitParameters.meridian = meridian
gdlit.gendaylitParameters.longitude = longitude
gdlit.gendaylitParameters.latitude = latitude
sunValues = []
sunUpHours = [] # collect hours that sun is up
# use gendaylit to calculate radiation values for each hour.
print('Calculating sun positions and radiation values.')
with open(os.devnull, 'w') as warningDump:
for timeStamp in monthDateTime:
month, day, hour = timeStamp.month, timeStamp.day, timeStamp.hour + 0.5
dnr, dhr = wea.directNormalRadiation[timeStamp.intHOY], \
wea.diffuseHorizontalRadiation[timeStamp.intHOY]
if dnr + dhr == 0:
# no need to run gendaylit as there is no radiation / sun
continue
# update gendaylit params
gdlit.gendaylitParameters.dirNormDifHorzIrrad = (dnr, dhr)
gdlit.monthDayHour = (month, day, hour)
cmdgd, ro = tuple(
c.strip()
for c in gdlit.toRadString().split('>')[0].split('|'))
# run cmd, get results in the form of a list of lines.
cmdRun = Popen(cmdgd, stdout=PIPE, stderr=warningDump)
gd = Popen(ro, stdin=cmdRun.stdout, stdout=PIPE)
data = gd.communicate()[0].split('\n')
# clean the output by throwing out comments and brightness functions.
sunCurrentValue = []
for lines in data[:-6]:
if lines[0] == "#":
continue
ls = lines.strip()
if ls:
sunCurrentValue.extend(ls.split())
# If a sun definition was captured in the last for-loop, store info.
if sunCurrentValue and max(map(float, sunCurrentValue[6:9])):
sunCurrentValue[2] = 'solar%s' % (len(sunValues) + 1)
sunCurrentValue[9] = 'solar%s' % (len(sunValues) + 1)
sunValues.append(sunCurrentValue)
sunUpHours.append(timeStamp.intHOY)
numOfSuns = len(sunUpHours)
print('Writing sun positions and radiation values to {}'.format(fp))
# create solar discs.
with open(fp, 'w') as annfile:
annfile.write("\n".join((" ".join(sun) for sun in sunValues)))
annfile.write('\n')
print('Writing list of suns to {}'.format(lfp))
# create list of suns.
with open(lfp, 'w') as sunlist:
sunlist.write("\n".join(
("solar%s" % (idx + 1) for idx in xrange(numOfSuns))))
sunlist.write('\n')
# Start creating header for the sun matrix.
fileHeader = ['#?RADIANCE']
fileHeader += ['Sun matrix created by Honeybee']
fileHeader += ['LATLONG= %s %s' % (latitude, -longitude)]
fileHeader += ['NROWS=%s' % numOfSuns]
fileHeader += ['NCOLS=%s' % len(self.hoys)]
fileHeader += ['NCOMP=3']
fileHeader += ['FORMAT=ascii']
print('Writing sun matrix to {}'.format(mfp))
# Write the matrix to file.
with open(mfp, 'w') as sunMtx:
sunMtx.write('\n'.join(fileHeader) + '\n' + '\n')
for idx, sunValue in enumerate(sunValues):
sunRadList = ['0 0 0'] * len(self.hoys)
sunRadList[sunUpHours[idx]] = ' '.join(sunValue[6:9])
sunMtx.write('\n'.join(sunRadList) + '\n\n')
# This last one is for the ground.
sunRadList = ['0 0 0'] * len(self.hoys)
sunMtx.write('\n'.join(sunRadList))
sunMtx.write('\n')
return fp, lfp, mfp
def output(partIdx,ch_aux):
print '== If you use queryDocTrainData or queryDocTrainRel, make sure it\'s in the current working directory'
print '== Running your code ...'
print '== Your code should output results (and nothing else) to stdout'
# tempoutfile = tempfile.mkdtemp()
# getFiles(tempoutfile)
outputString = ''
tempoutfile = 'tmp' + str(random.randint(0,1<<20))
localfile = open(tempoutfile,'w')
localfile.write(ch_aux)
localfile.close()
linesOutput = 1065
if not os.path.exists("people.txt"):
print "There is no people.txt file in this directory. Please make people.txt file in this directory with your and your partner's SUNet ID in separate lines (do NOT include @stanford.edu)"
sys.exit(1)
people = open("people.txt").read().splitlines();
if len(people) == 0:
print "people.txt is empty! Write the SUNet ids of you and your partner, if any, in separate lines.."
sys.exit(1);
for x in people:
if len(x) < 3 or len(x) > 8 or ' ' in x.strip() == True:
print "The SUNet IDs don't seem to be correct. Make sure to remove empty lines. They are supposed to be between 3 and 8 characters. Also, make sure to not include @stanford.edu."
sys.exit(1);
peopleStr = "_".join(str(x.strip()) for x in people if x)
stats = {}
stats['USERIDS']= peopleStr
stats['TIMESUBMITTED'] = str(datetime.datetime.now());
elapsed= 0.0
if partIdx == 0:
print 'Submitting the report'
elif partIdx == 1:
print 'Calling ./rank.sh for Task 1 (this might take a while)'
start = time()
child = Popen(['./rank.sh', tempoutfile,'cosine'], stdout=PIPE, stderr=PIPE, shell=False);
(res, err) = child.communicate("")
elapsed = time() - start
guesses = res.splitlines()
print err
if (len(guesses) != linesOutput):
print 'Warning. The number of url-document pairs ' + str(len(guesses)) + ' is not correct. Please ensure that the output is formatted properly.'
stats['result'] = res
elif partIdx == 2:
print 'Calling ./rank.sh for Task 2 (this might take a while)'
start = time()
child = Popen(['./rank.sh', tempoutfile,'bm25'], stdout=PIPE, stderr=PIPE, shell=False)
(res, err) = child.communicate("")
elapsed = time() - start
guesses = res.splitlines()
print err
if (len(guesses) != linesOutput):
print 'Warning. The number of url-document pairs is not correct. Please ensure that the output is formatted properly.'
stats['result'] = res
elif partIdx == 3:
print 'Calling ./rank.sh for Task 3 (this might take a while)'
start = time()
child = Popen(['./rank.sh',tempoutfile,'window'], stdout=PIPE, stderr=PIPE, shell=False)
(res, err) = child.communicate("")
elapsed = time() - start
guesses = res.splitlines()
print err
if (len(guesses) != linesOutput):
print 'Warning. The number of url-document pairs is not correct. Please ensure that the output is formatted properly.'
stats['result'] = res
elif partIdx == 4:
print 'Calling ./rank.sh for Task 4 (this might take a while)'
start = time()
child = Popen(['./rank.sh', tempoutfile,'extra'], stdout=PIPE, stderr=PIPE, shell=False)
(res, err) = child.communicate("")
elapsed = time() - start
guesses = res.splitlines()
print err
if (len(guesses) != linesOutput):
print 'Warning. The number of url-document pairs is not correct. Please ensure that the output is formatted properly.'
stats['result'] = res
else:
print '[WARNING]\t[output]\tunknown partId: %s' % partIdx
sys.exit(1)
print '== Finished running your code'
os.remove(tempoutfile)
return json.dumps(stats).strip()
class TransferTest(unittest.TestCase):
"""Subclass and override inputs/expectedOutputs (and possibly other
stuff) to create new postchunk tests."""
bindata = "data/nno-nob.t1x.bin"
t1xdata = "data/apertium-nno-nob.nob-nno.t1x"
flags = ["-b", "-z"]
inputs = [""]
expectedOutputs = [""]
expectedRetCodeFail = False
def alarmHandler(self, signum, frame):
raise Alarm
def withTimeout(self, seconds, cmd, *args, **kwds):
signal.signal(signal.SIGALRM, self.alarmHandler)
signal.alarm(seconds)
ret = cmd(*args, **kwds)
signal.alarm(0) # reset the alarm
return ret
def communicateFlush(self, string):
self.proc.stdin.write(string.encode('utf-8'))
self.proc.stdin.write(b'\0')
self.proc.stdin.flush()
output = []
char = None
try:
char = self.withTimeout(2, self.proc.stdout.read, 1)
except Alarm:
pass
while char and char != b'\0':
output.append(char)
try:
char = self.withTimeout(2, self.proc.stdout.read, 1)
except Alarm:
break # send what we got up till now
return b"".join(output).decode('utf-8')
def compile(self):
compileCmd = ["../apertium/apertium-preprocess-transfer",
self.t1xdata,
self.bindata]
self.assertEqual(call(compileCmd),
0)
def runTest(self):
self.compile()
try:
cmd = ["../apertium/apertium-transfer"] \
+ self.flags \
+ [self.t1xdata, self.bindata]
self.proc = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
for inp, exp in zip(self.inputs, self.expectedOutputs):
self.assertEqual(self.communicateFlush(inp+"[][\n]"),
exp+"[][\n]")
self.proc.communicate() # let it terminate
self.proc.stdin.close()
self.proc.stdout.close()
self.proc.stderr.close()
retCode = self.proc.poll()
if self.expectedRetCodeFail:
self.assertNotEqual(retCode, 0)
else:
self.assertEqual(retCode, 0)
finally:
pass
class TritonServerLocal(TritonServer):
"""
Concrete Implementation of TritonServer interface that runs
tritonserver locally as as subprocess.
"""
def __init__(self, path, config):
"""
Parameters
----------
path : str
The absolute path to the tritonserver executable
config : TritonServerConfig
the config object containing arguments for this server instance
"""
self._tritonserver_process = None
self._server_config = config
self._server_path = path
self._log = None
assert self._server_config['model-repository'], \
"Triton Server requires --model-repository argument to be set."
def start(self):
"""
Starts the tritonserver container locally
"""
# Create command list and run subprocess
cmd = [self._server_path]
cmd += self._server_config.to_cli_string().replace('=', ' ').split()
self._tritonserver_process = Popen(cmd,
start_new_session=True,
stdout=PIPE,
stderr=STDOUT,
universal_newlines=True)
logger.info('Triton Server started.')
def stop(self):
"""
Stops the running tritonserver
"""
# Terminate process, capture output
if self._tritonserver_process is not None:
self._tritonserver_process.terminate()
try:
self._log, _ = self._tritonserver_process.communicate(
timeout=SERVER_OUTPUT_TIMEOUT_SECS)
except TimeoutExpired:
self._tritonserver_process.kill()
self._log, _ = self._tritonserver_process.communicate()
self._tritonserver_process = None
logger.info('Triton Server stopped.')
def logs(self):
"""
Retrieves the Triton server's stdout
as a str
"""
return self._log
def cpu_stats(self):
"""
Returns the CPU memory usage and CPU available memory in MB
"""
if self._tritonserver_process:
server_process = psutil.Process(self._tritonserver_process.pid)
process_memory_info = server_process.memory_full_info()
system_memory_info = psutil.virtual_memory()
# Divide by 1.0e6 to convert from bytes to MB
return (process_memory_info.uss //
1.0e6), (system_memory_info.available // 1.0e6)
else:
return 0.0, 0.0
def _do_createproject(self, req):
"""
Update project properties from the Admin form
"""
data = {}
data['project_created'] = True
group = req.args['group']
templates_dir = self.env.config.get('inherit', 'templates_dir')
inherit_file = self.env.config.get('inherit', 'file')
#Try to get custom settings for this project group
templates_dir = self.env.config.get('projectmanager', 'groups.%s.templates_dir' % group, templates_dir)
inherit_file = self.env.config.get('projectmanager', 'groups.%s.inherit_file' % group, inherit_file)
group_dir = self.env.config.get('projectmanager', 'groups.%s.dirname' % group, group)
sql_defaults = self.env.config.get('projectmanager', 'groups.%s.sql_defaults' % group, self.sql_defaults)
assert self.repos_dir, "repos_dir not defined in .ini"
assert self.environments_dir, "environments_dir not defined in .ini"
if req.args.has_key('makesvn'):
cmd = "%s create %s" % (self.svnadmin_command,os.path.join(self.repos_dir, group_dir, req.args['short_name']))
p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE)
(stdout, stderr) = p.communicate()
if p.returncode != 0:
data['svn_error'] = True
data['svn_created'] = True
data['svn_output'] = stdout
data['svn_errors'] = stderr
if req.args.has_key('maketrac'):
repos_dir = os.path.join(self.repos_dir, group_dir, req.args['short_name'])
env_dir = os.path.join(self.environments_dir, group_dir, req.args['short_name'])
cmd = '%s "%s" initenv "%s" sqlite:%s svn "%s" --inherit="%s"' % (
self.tracadmin_command,
env_dir,
req.args['full_name'],
os.path.join('db','trac.db'),
repos_dir,
inherit_file
)
#stdin, stdout, stderr = os.popen3(command)
p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE)
(stdout, stderr) = p.communicate()
data['trac_error'] = p.returncode != 0
try:
template = os.path.join(env_dir, 'templates', 'site.html')
stdout = stdout + 'Removing template: %s' % template
os.remove(template)
except: pass
data['trac_created'] = True
data['trac_output'] = stdout
data['trac_errors'] = stderr
try:
env = open_environment(env_dir, use_cache=False)
db = env.get_db_cnx()
cursor = db.cursor()
output = ""
output = output + "\n\nSQL Defaults, executing statements\n"
output = output + "==================================\n"
for statement in sql_defaults.splitlines():
output = output + "Executing: %s\n" % statement
cursor.execute(statement)
db.commit()
data['trac_output'] = data['trac_output'] + output
env.shutdown()
except:
data['trac_output'] = data['trac_output'] + "\n\nError running SQL Defaults statements, skipping\n"
return data
dataXsample = np.asarray([[-1.0 for i in range(n)] for j in range(n)],
dtype=np.float)
dataXdata = np.asarray([[-1.0 for i in range(n)] for j in range(n)],
dtype=np.float)
errors = [False, False, False]
for i in range(n):
for j in range(n):
try:
p = Popen([
'./bottleneck_dist', sample_strings[i], sample_strings[j]
],
stdin=PIPE,
stdout=PIPE,
stderr=PIPE)
output, err = p.communicate(
b"input data that is passed to subprocess' stdin")
output_string = output.decode()
sampleXsample[i][j] = float(output_string)
except ValueError as valerr:
errors[0] = True
print(str(err) + " " + str(valerr))
print(sample_strings[i] + " " + sample_strings[j])
print(output_string)
try:
p = Popen(
['./bottleneck_dist', data_strings[i], sample_strings[j]],
stdin=PIPE,
stdout=PIPE,
stderr=PIPE)
output, err = p.communicate(
def send_email_old(address, subject, content):
p = Popen(["/usr/sbin/sendmail", "-i", "--", address], stdin=PIPE)
msg = MIMEText(content)
msg['Subject'] = subject
msg['To'] = address
p.communicate(msg.as_bytes())
def cmdline(cmd, sh=True):
""" Execute command @cmd and return output using Popen(). """
process = Popen(args=cmd, stdout=PIPE, shell=sh)
return process.communicate()[0].decode("ISO-8859-1")
def get_mysql_config_info(mysql_config):
"""Get MySQL information using mysql_config tool
Returns a dict.
"""
options = ['cflags', 'include', 'libs', 'libs_r', 'plugindir', 'version']
cmd = [mysql_config] + ["--{0}".format(opt) for opt in options]
try:
proc = Popen(cmd, stdout=PIPE, universal_newlines=True)
stdout, _ = proc.communicate()
except OSError as exc:
raise DistutilsExecError("Failed executing mysql_config: {0}".format(
str(exc)))
info = parse_mysql_config_info(options, stdout)
# Try to figure out the architecture
info['arch'] = None
if os.name == 'posix':
if platform.uname()[0] == 'SunOS':
print("info['lib_dir']: {0}".format(info['lib_dir']))
print("info['libs'][0]: {0}".format(info['libs'][0]))
pathname = os.path.abspath(
os.path.join(info['lib_dir'], 'lib', info['libs'][0])) + '/*'
else:
pathname = os.path.join(info['lib_dir'],
'lib' + info['libs'][0]) + '*'
print("# Looking mysqlclient_lib at path: {0}".format(pathname))
log.debug("# searching mysqlclient_lib at: %s", pathname)
libs = glob(pathname)
mysqlclient_libs = []
for filepath in libs:
_, filename = os.path.split(filepath)
log.debug("# filename {0}".format(filename))
if filename.startswith('libmysqlclient') and \
not os.path.islink(filepath) and \
'_r' not in filename and \
'.a' not in filename:
mysqlclient_libs.append(filepath)
mysqlclient_libs.sort()
stdout = None
try:
log.debug("# mysqlclient_lib: {0}".format(mysqlclient_libs[-1]))
for mysqlclient_lib in mysqlclient_libs:
log.debug("#+ {0}".format(mysqlclient_lib))
log.debug("# tested mysqlclient_lib[-1]: "
"{0}".format(mysqlclient_libs[-1]))
if platform.uname()[0] == 'SunOS':
print("mysqlclient_lib: {0}".format(mysqlclient_libs[-1]))
cmd_list = ['file', mysqlclient_libs[-1]]
else:
cmd_list = ['file', '-L', mysqlclient_libs[-1]]
proc = Popen(cmd_list, stdout=PIPE, universal_newlines=True)
stdout, _ = proc.communicate()
stdout = stdout.split(':')[1]
except OSError as exc:
raise DistutilsExecError(
"Although the system seems POSIX, the file-command could not "
"be executed: {0}".format(str(exc)))
if stdout:
if '64' in stdout:
info['arch'] = "x86_64"
else:
info['arch'] = "i386"
else:
raise DistutilsExecError(
"Failed getting out put from the file-command")
else:
raise DistutilsExecError(
"Cannot determine architecture on {0} systems".format(os.name))
return info
def main():
UPDATED_DIGEST = 'SHA256'
DIGESTS_TO_UPDATE = ['WHIRLPOOL', 'RIPEMD']
keys = defaultdict(lambda: 0)
keys_to_update = []
dynamodb_resource = resource('dynamodb')
table = dynamodb_resource.Table('credential-store')
response = table.scan()
items = response['Items']
# appending all dynamodb entries to items dict
while True:
if response.get('LastEvaluatedKey'):
response = table.scan(
ExclusiveStartKey=response['LastEvaluatedKey'])
items += response['Items']
else:
break
# storing latest version of keys with their digests
for i in range(len(items)):
try:
digest = items[i]['digest']
version = int(items[i]['version'])
key = items[i]['name']
except:
continue
if key in keys:
if version > keys[key][0]:
keys[key][0] = version
keys[key][1] = digest
else:
keys[key] = [version, digest]
# store keys to be updated
for k, v in keys.items():
if v[1] in DIGESTS_TO_UPDATE:
keys_to_update.append(k)
# confirms update of digests
if len(keys_to_update):
print('\nThe following keys will be updated to {0}:\n'.format(
UPDATED_DIGEST))
for key in keys_to_update:
print('{0}\n'.format(key))
confirmed = None
while not confirmed:
val = input('Continue? y/n ')
if val.lower() == 'y' or val.lower() == 'yes':
confirmed = True
elif val.lower() == 'n' or val.lower() == 'no':
print('\nexiting...\n')
sys.exit()
else:
print('\nInvalid input\n')
else:
print('\nNo digests to update!\n')
sys.exit()
# updating deprecated digests
for key in keys_to_update:
p = Popen(['credstash', 'get', key], stdout=PIPE, stderr=PIPE)
secret, err = p.communicate()
secret = secret[:-1] # removes credstash-added newline for stdout
if not err:
p = Popen(
['credstash', 'put', key, secret, '-a', '-d', UPDATED_DIGEST],
stdout=PIPE)
update, err = p.communicate()
print('{0} has been updated!\n'.format(key))
else:
print('Error found, skipping update of {0}. Error: {1}'.format(
key, err))
def parse_recentApps(self, dataSource, progressBar):
# we don't know how much work there is yet
progressBar.switchToIndeterminate()
# Set the database to be read to the once created by the prefetch parser program
skCase = Case.getCurrentCase().getSleuthkitCase();
fileManager = Case.getCurrentCase().getServices().getFileManager()
files = fileManager.findFiles(dataSource, "%", "/Windows/System32/wbem/Repository/")
numFiles = len(files)
progressBar.switchToDeterminate(numFiles)
fileCount = 0;
# Create Event Log directory in temp directory, if it exists then continue on processing
Temp_Dir = Case.getCurrentCase().getTempDirectory()
self.log(Level.INFO, "create Directory " + Temp_Dir)
temp_dir = os.path.join(Temp_Dir, "Recently_Used")
try:
os.mkdir(temp_dir)
except:
self.log(Level.INFO, "Recently Used Directory already exists " + temp_dir)
# Write out each Event Log file to the temp directory
for file in files:
# Check if the user pressed cancel while we were busy
if self.context.isJobCancelled():
return IngestModule.ProcessResult.OK
#self.log(Level.INFO, "Processing file: " + file.getName())
fileCount += 1
if (file.getName() == '.' or file.getName() == '..'):
self.log(Level.INFO, "Parent or Root Directory File not writing")
else:
# Save the DB locally in the temp folder. use file id as name to reduce collisions
lclDbPath = os.path.join(temp_dir, file.getName())
ContentUtils.writeToFile(file, File(lclDbPath))
self.log(Level.INFO, "Running prog ==> " + self.path_to_recentApps_exe + " win7 " + temp_dir + " " + \
temp_dir + "\recentlyUsedApps.db3")
pipe = Popen([self.path_to_recentApps_exe, "win7", temp_dir, os.path.join(temp_dir, "recentlyUsedApps.db3")], stdout=PIPE, stderr=PIPE)
out_text = pipe.communicate()[0]
self.log(Level.INFO, "Output from run is ==> " + out_text)
lclDbPath = os.path.join(temp_dir, "recentlyUsedApps.db3")
if ("Exiting" in out_text):
message = IngestMessage.createMessage(IngestMessage.MessageType.DATA,
"CCM Recently Used Apps", " Error in CCM Recently Used Apps module " )
IngestServices.getInstance().postMessage(message)
else:
# Add custom Artifact to blackboard
try:
self.log(Level.INFO, "Begin Create New Artifacts ==> TSK_CCM_RECENTLY_USED_APPS")
artID_art = skCase.addArtifactType("TSK_CCM_RECENTLY_USED_APPS", "WMI Recently Used Apps")
except:
self.log(Level.INFO, "Artifacts Creation Error, artifact TSK_CCM_RECENTLY_USED_APPS exists. ==> ")
# Add Custom attributes to blackboard
try:
attID_efn = skCase.addArtifactAttributeType("TSK_EXPLORER_FILE_NAME", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "Explorer File Name")
except:
self.log(Level.INFO, "Attributes Creation Error, Explorer File Name ==> ")
try:
attID_efn = skCase.addArtifactAttributeType("TSK_FILE_SIZE", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "File Size")
except:
self.log(Level.INFO, "Attributes Creation Error, File Size ==> ")
try:
attID_efn = skCase.addArtifactAttributeType("TSK_LAST_USED_TIME", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.DATETIME, "Last Used Time")
except:
self.log(Level.INFO, "Attributes Creation Error, Last Used Time ==> ")
try:
attID_efn = skCase.addArtifactAttributeType("TSK_TIME_ZONE_OFFSET", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "Time Zone Offset")
except:
self.log(Level.INFO, "Attributes Creation Error, Time Zone Offset ==> ")
try:
attID_efn = skCase.addArtifactAttributeType("TSK_LAUNCH_COUNT", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "Launch Count")
except:
self.log(Level.INFO, "Attributes Creation Error, Launch Count ==> ")
try:
attID_efn = skCase.addArtifactAttributeType("TSK_ORIG_FILE_NAME", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "Original File Name")
except:
self.log(Level.INFO, "Attributes Creation Error, Original File Name ==> ")
try:
attID_efn = skCase.addArtifactAttributeType("TSK_FILE_DESC", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "File Description")
except:
self.log(Level.INFO, "Attributes Creation Error, File Description ==> ")
try:
attID_efn = skCase.addArtifactAttributeType("TSK_PROD_NAME", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "Product Name")
except:
self.log(Level.INFO, "Attributes Creation Error, Product Name ==> ")
try:
attID_efn = skCase.addArtifactAttributeType("TSK_PROD_VERSION", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "Product Version")
except:
self.log(Level.INFO, "Attributes Creation Error, Product Version ==> ")
try:
attID_efn = skCase.addArtifactAttributeType("TSK_FILE_VERSION", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "File Version")
except:
self.log(Level.INFO, "Attributes Creation Error, File Version ==> ")
try:
attID_efn = skCase.addArtifactAttributeType("TSK_ADDITIONAL_PROD_CODES", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "Additional Product Codes")
except:
self.log(Level.INFO, "Attributes Creation Error, Additional Product Codes ==> ")
try:
attID_efn = skCase.addArtifactAttributeType("TSK_MSI_VERSION", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "MSI Version")
except:
self.log(Level.INFO, "Attributes Creation Error, MSI Version ==> ")
try:
attID_efn = skCase.addArtifactAttributeType("TSK_MSI_DISPLAY_NAME", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "MSI Display Name")
except:
self.log(Level.INFO, "Attributes Creation Error, MSI Display Name ==> ")
try:
attID_efn = skCase.addArtifactAttributeType("TSK_PRODUCT_CODE", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "Product Code")
except:
self.log(Level.INFO, "Attributes Creation Error, Product Code ==> ")
try:
attID_efn = skCase.addArtifactAttributeType("TSK_SOFTWARE_PROP_HASH", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "Software Property Hash")
except:
self.log(Level.INFO, "Attributes Creation Error, Software Property Hash ==> ")
try:
attID_efn = skCase.addArtifactAttributeType("TSK_PROD_LANG", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "Product Language")
except:
self.log(Level.INFO, "Attributes Creation Error, Product Language ==> ")
try:
attID_efn = skCase.addArtifactAttributeType("TSK_FILE_PROP_HASH", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "File Property Hash")
except:
self.log(Level.INFO, "Attributes Creation Error, File Property Hash ==> ")
try:
attID_efn = skCase.addArtifactAttributeType("TSK_MSI_PUBLISHER", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "MSI Publisher")
except:
self.log(Level.INFO, "Attributes Creation Error, MSI Publisher ==> ")
for file in files:
if (file.getName() == "OBJECTS.DATA"):
# Open the DB using JDBC
lclDbPath = os.path.join(temp_dir, "recentlyUsedApps.db3")
self.log(Level.INFO, "Path the recentlyUsedApps.db3 database file created ==> " + lclDbPath)
try:
Class.forName("org.sqlite.JDBC").newInstance()
dbConn = DriverManager.getConnection("jdbc:sqlite:%s" % lclDbPath)
except SQLException as e:
self.log(Level.INFO, "Could not open database file (not SQLite) recentlyUsedApps.db3 (" + e.getMessage() + ")")
return IngestModule.ProcessResult.OK
# Query the history_visits table in the database and get all columns.
try:
stmt = dbConn.createStatement()
recently_used_sql = "select FolderPath 'TSK_PATH', ExplorerFileName 'TSK_EXPLORER_FILE_NAME', " + \
"FileSize 'TSK_FILE_SIZE', LastUserName 'TSK_USER_ID', strftime('%s',LastUsedTime) " + \
"'TSK_LAST_USED_TIME', TimeZoneOffset 'TSK_TIME_ZONE_OFFSET', LaunchCount " + \
"'TSK_LAUNCH_COUNT', OriginalFileName 'TSK_ORIG_FILE_NAME', FileDescription " + \
"'TSK_FILE_DESC', CompanyName 'TSK_ORGANIZATION', ProductName 'TSK_PROD_NAME', " + \
"ProductVersion 'TSK_PROD_VERSION', FileVersion 'TSK_FILE_VERSION', " + \
"AdditionalProductCodes 'TSK_ADDITIONAL_PROD_CODES', msiVersion " + \
"'TSK_MSI_VERSION', msiDisplayName 'TSK_MSI_DISPLAY_NAME', " + \
"ProductCode 'TSK_PRODUCT_CODE', SoftwarePropertiesHash " + \
"'TSK_SOFTWARE_PROP_HASH', ProductLanguage 'TSK_PROD_LANG', " + \
"FilePropertiesHash 'TSK_FILE_PROP_HASH', msiPublisher 'TSK_MSI_PUBLISHER' " + \
"from recently_used;"
self.log(Level.INFO, recently_used_sql)
resultSet = stmt.executeQuery(recently_used_sql)
self.log(Level.INFO, "query recently_used table")
except SQLException as e:
self.log(Level.INFO, "Error querying database for recently_used table (" + e.getMessage() + ")")
return IngestModule.ProcessResult.OK
artID_hst = skCase.getArtifactTypeID("TSK_CCM_RECENTLY_USED_APPS")
artID_hst_evt = skCase.getArtifactType("TSK_CCM_RECENTLY_USED_APPS")
meta = resultSet.getMetaData()
columncount = meta.getColumnCount()
column_names = []
self.log(Level.INFO, "Number of Columns in the table ==> " + str(columncount))
for x in range (1, columncount + 1):
self.log(Level.INFO, "Column Name ==> " + meta.getColumnLabel(x))
column_names.append(meta.getColumnLabel(x))
self.log(Level.INFO, "All Columns ==> " + str(column_names))
# Cycle through each row and create artifacts
while resultSet.next():
try:
#self.log(Level.INFO, SQL_String_1)
self.log(Level.INFO, "Artifact Is ==> " + str(artID_hst))
art = file.newArtifact(artID_hst)
self.log(Level.INFO, "Inserting attribute URL")
for col_name in column_names:
attID_ex1 = skCase.getAttributeType(col_name)
self.log(Level.INFO, "Inserting attribute ==> " + str(attID_ex1))
self.log(Level.INFO, "Attribute Type ==> " + str(attID_ex1.getValueType()))
if attID_ex1.getValueType() == BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING:
try:
art.addAttribute(BlackboardAttribute(attID_ex1, ParseRecentlyUsedAppsIngestModuleFactory.moduleName, resultSet.getString(col_name)))
except:
self.log(Level.INFO, "Attributes String Creation Error, " + col_name + " ==> ")
elif attID_ex1.getValueType() == BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.INTEGER:
try:
art.addAttribute(BlackboardAttribute(attID_ex1, ParseRecentlyUsedAppsIngestModuleFactory.moduleName, resultSet.getInt(col_name)))
except:
self.log(Level.INFO, "Attributes Integer Creation Error, " + col_name + " ==> ")
elif attID_ex1.getValueType() == BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.LONG:
try:
art.addAttribute(BlackboardAttribute(attID_ex1, ParseRecentlyUsedAppsIngestModuleFactory.moduleName, resultSet.getInt(col_name)))
except:
self.log(Level.INFO, "Attributes Long Creation Error, " + col_name + " ==> ")
elif attID_ex1.getValueType() == BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.DOUBLE:
try:
art.addAttribute(BlackboardAttribute(attID_ex1, ParseRecentlyUsedAppsIngestModuleFactory.moduleName, resultSet.getInt(col_name)))
except:
self.log(Level.INFO, "Attributes Double Creation Error, " + col_name + " ==> ")
elif attID_ex1.getValueType() == BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.BYTE:
try:
art.addAttribute(BlackboardAttribute(attID_ex1, ParseRecentlyUsedAppsIngestModuleFactory.moduleName, resultSet.getString(col_name)))
except:
self.log(Level.INFO, "Attributes Byte Creation Error, " + col_name + " ==> ")
else:
try:
art.addAttribute(BlackboardAttribute(attID_ex1, ParseRecentlyUsedAppsIngestModuleFactory.moduleName, int(resultSet.getString(col_name))))
except:
self.log(Level.INFO, "Attributes Datatime Creation Error, " + col_name + " ==> ")
except SQLException as e:
self.log(Level.INFO, "Error getting values from web_history table (" + e.getMessage() + ")")
IngestServices.getInstance().fireModuleDataEvent(
ModuleDataEvent(ParseRecentlyUsedAppsIngestModuleFactory.moduleName, artID_hst_evt, None))
stmt.close()
dbConn.close()
# Clean up
try:
os.remove(lclDbPath)
except:
self.log(Level.INFO, "removal of Recently Used database failed ")
#Clean up EventLog directory and files
for file in files:
try:
os.remove(os.path.join(temp_dir, file.getName()))
except:
self.log(Level.INFO, "removal of Recently Used files failed " + temp_dir + "\\" + file.getName())
try:
os.rmdir(temp_dir)
except:
self.log(Level.INFO, "removal of recently used directory failed " + temp_dir)
