def main(args=None):
"""The main routine."""
if args is None:
args = sys.argv[1:]
print("WU TANG FINANCIAL")
print("C.R.E.A.M.\n")
core.execute()
def main():
parser = argparse.ArgumentParser(description="BIG Query Testing Framework")
parser.add_argument("-p",
"--path",
nargs=1,
metavar="path",
type=str,
help="test file path")
args = parser.parse_args()
if len(args.path) != 0:
path = args.path[0]
FileManager(path)
execute()
else:
raise Exception("File Path is required. --path")
def execute():
sql = request.form['sql']
result = core.execute(sql)
if type(result['payload']) == str:
result['type'] = 'str'
if type(result['payload']) == dict:
result['type'] = 'dict'
return render_template('result.html', result=result, sql=sql)
def run_sca(binary_path, binary_args, sca_options):
'''Run the SCA tool'''
if not core.cmdexists(FDPRPRO):
sys.stderr.write("fdpr-pro is not installed in the system.\n" +
"To install it download and install the required\n" +
"package from: " + SDK_DOWNLOAD_PAGE + '\n')
sys.exit(2)
elif not core.cmdexists(FDPR_WRAP):
sys.stderr.write("fdpr_wrap is not installed in the system.\n" +
"To install it download and install the required\n" +
"package from: " + SDK_DOWNLOAD_PAGE + '\n')
sys.exit(2)
else:
# Export fdpr flags in system environment
os.environ['FDPR_OPT_FLAGS'] = sca_options.get_fdpr_opt()
# Get binary absolute path
binary_path = os.path.realpath(binary_path)
status_cmd = FDPR_WRAP + " " + binary_path + " " + binary_args
status = core.execute(status_cmd)
check_exit_status(status)
jour_file = binary_path + "-jour.xml"
group_problems = core.run_xml_match(jour_file)
if not group_problems:
sys.stdout.write("\nSCA report: No reports found.")
sys.exit(0)
if sca_options.get_file_type_opt() is not None:
ok_val = core.save_sca(group_problems, sca_options.get_file_name(),
sca_options.get_file_type_opt())
if not ok_val:
print "\nSCA report: An error happened when saving the file.\n"
else:
print "\nSCA report was saved in file: ",
print sca_options.get_file_name() + '\n'
else:
core.print_sca(group_problems, sca_options.get_color_opt())
# set to false if you want to do an experiment and NOT a submission
SUBMISSION = False
COMPRESSION = 5
# load data in memory
if SUBMISSION:
images = np.concatenate((loaders.load_all_images(False, COMPRESSION),
loaders.load_all_images(True, COMPRESSION)))
else:
images = loaders.load_all_images(False, COMPRESSION)
while True:
try:
# execute your function
if SUBMISSION:
core.execute(images)
else:
experiments.execute(images)
except:
traceback.print_exc()
print "Press enter to re-run the script, CTRL-C to exit"
sys.stdin.readline()
print "Re-running..."
reload(features)
reload(loaders)
reload(experiments)
reload(core)
from setting import params
from core import execute
url = params['url']
tag = params['tag']
dataframe = execute(url, tag)
dataframe.to_csv('output.txt', sep='\t', header=False)
# A Guide tells you how to use an Appliance
statements += core.model("ApplianceE -- GuideEEcnn -- GuideE")
# A version in Git have additional properties.
statements += core.model("GitVersionE -- CommitEScn1")
statements += core.model("GitVersionE -- DateETcn1")
statements += core.model("GitVersionE -- CommentEBcn1")
# Graphical Presentation
statements += core.relate(["EntityE", "ShapeSS", "box"])
statements += core.relate(["EntityE", "RedSI", 255])
statements += core.relate(["EntityE", "GreenSI", 255])
statements += core.relate(["EntityE", "BlueSI", 255])
statements += core.relate(["ArtifactE", "RedSI", 128])
statements += core.relate(["VersionE", "RedSI", 192])
statements += core.relate(["MutableE", "BlueSI", 0])
statements += core.relate(["ContainerE", "BlueSI", 64])
statements += core.relate(["ConstantE", "BlueSI", 128])
statements += core.relate(["EmbeddedE", "BlueSI", 192])
statements += core.relate(["SpecificationE", "GreenSI", 128])
statements += core.relate(["ImplementationE", "GreenSI", 192])
statements += core.relate(["GitVersionE", "BlueSI", 64])
core.execute(statements)
def run(self):
'''
Run an instance of a tool
'''
run_tool = True
if db.check_if_tool_run(self.name, self.target):
print "Record found in run history for " +self.name + " on " + self.target
if core.prompt_tool_reruns:
response = raw_input("Would you like to re-run the tool? [n]")
if "y" in response or "Y" in response:
run_tool = True
else:
run_tool = False
else:
run_tool = False
if not run_tool:
return
#do not run aggressive tools if aggressive mode is not enabled
if self.aggressive and not core.aggressive:
return
if not os.path.exists(os.path.join(self.output_dir, self.output_subdir)):
os.makedirs(os.path.join(self.output_dir, self.output_subdir))
print "Running "+self.name+" on "+self.target
if self.command:
output_file_path = os.path.join(self.output_dir, self.output_subdir, self.name + "_" + self.target + "." + self.output_format)
self.command = self.command.replace("[TARGET]", self.target)
self.command = self.command.replace("[OUTPUT]", output_file_path)
print self.command
self.start_time = core.getTimestamp(True)
self.command_result = core.execute(self.command, self.suppress_out)
self.end_time = core.getTimestamp(True)
if self.cleanup_regex <> "":
clean_result = re.findall(self.cleanup_regex, self.command_result)
self.command_result = core.list_to_text(clean_result)
if not self.output_format:
db.add_run_to_db(self.name, self.target, self.command, self.command_result,'' ,'txt',self.start_time, self.end_time, self.output_subdir)
else:
db.add_run_to_db(self.name, self.target, self.command, self.command_result, output_file_path, self.output_format, self.start_time, self.end_time, self.output_subdir)
print ""
#if no output directory is specified or tool outputs to file itself, then only output to screen...
if self.output_dir and not self.output_format:
output.write_outfile(os.path.join(self.output_dir, self.output_subdir), self.name+ "_" + self.target + ".txt", self.command_result)
if self.email_regex:
self.emails = sorted(list(set(re.findall(self.email_regex, self.command_result))))
if self.email_domain_filter:
self.emails = [s for s in self.emails if self.email_domain_filter in s]
for email in self.emails:
email = email.lower()
db.add_person_to_db(email)
print "Emails discovered: " + str(self.emails)
if self.dns_regex:
self.dns = sorted(list(set(re.findall(self.dns_regex, self.command_result))))
for target in self.dns:
target = target.lower()
addresses = core.nslookup_fwd(target)
for address in addresses:
db.add_host_to_db(address,[target])
print "DNS entries discovered: " + str(self.dns)
if self.ip_regex:
self.ip = sorted(list(set(re.findall(self.ip_regex, self.command_result))))
for target in self.ip:
hostnames = core.nslookup_rev(target)
for hostname in hostnames:
hostname = hostname.lower()
db.add_host_to_db(target, [hostname])
print "IPs discovered: " + str(self.ip)
print "\n" + "-"*80 + "\n"
if self.url:
self.url = self.url.replace("[TARGET]", self.target)
output_file_path = os.path.join(self.output_dir, self.output_subdir, self.name + "_" + self.target + "." + self.website_output_format)
command = "cutycapt --url="+self.url+"--delay=1000 --out="+ output_file_path
core.execute(command, self.suppress_out)
db.add_run_to_db(self.name, self.target, self.command, self.command_result, output_file_path, self.website_output_format, self.start_time, self.end_time, self.output_subdir)
else:
statements += trackRequirement(container_artifact,
container_mutable,
container_constant, req, vnr,
title, body)
req = None
body = ""
return statements
###
### Track Pii project
(stmts, artifact, mutable, constant) = trackSpecification("./requirements.txt")
core.execute(stmts)
core.execute(trackRequirements(artifact, mutable, constant))
core.execute(trackPythonFile("./core.py"))
core.execute(trackPythonFile("./pii.py"))
core.execute(trackPythonFile("./pii_model.py"))
core.execute(trackPythonFile("./pii_tracker.py"))
core.execute(trackPythonFile("./q_files.py"))
core.execute(trackPythonFile("./q_spec.py"))
core.execute(trackPythonFile("./q_no_implementation.py"))
core.execute(trackPythonFile("./setversions.py"))
core.execute(trackJavascriptFile("./core.js"))
core.execute(trackGitVersions("./core.py", "core/python"))
core.execute(trackGitVersions("./pii.py", "pii/python"))
core.execute(trackGitVersions("./pii_model.py", "pii_model/python"))
import core
import sys
if __name__ == "__main__":
print("LabWork_1_PhoneBook. Danilov Sergey. 6th Group IAD")
if (len(sys.argv) > 1):
if sys.argv[1] != "debug":
core.init()
else:
print("Debug smoke testing only")
print("Database has been loaded sucessfully")
print("Type 'help' to open instruction of all functions "
"of the program or 'quit' to exit. Have fun!")
while True:
print(">>> ", end='')
command = ""
if (len(sys.argv) > 1):
print("Debug smoke testing only")
command = "quit"
else:
command = input()
if command == "quit" or command == "q":
sys.exit(0)
else:
core.execute(command)
def compute_file_paths():
csv_file_path = f"out/{directory_name}/csv/{file_name}.csv"
html_file_path = f"out/{directory_name}/html/{file_name}.html"
pdf_file_path = f"out/{directory_name}/pdf/{file_name}.pdf"
return csv_file_path, html_file_path, pdf_file_path
def create_directories(directory_name):
os.makedirs("out", exist_ok=True)
os.makedirs(f"out/{directory_name}", exist_ok=True)
os.makedirs(f"out/{directory_name}/csv", exist_ok=True)
os.makedirs(f"out/{directory_name}/html", exist_ok=True)
os.makedirs(f"out/{directory_name}/pdf", exist_ok=True)
picked_cal_names, picked_cal_ids = input_calendars()
from_date, to_date = input_range_option()
filter = input_filter()
output_format = input_format()
file_name = compute_file_name(from_date, to_date)
directory_name = compute_directory_name(picked_cal_names)
csv_file_path, html_file_path, pdf_file_path = compute_file_paths()
create_directories(directory_name)
result = core.execute(picked_cal_ids, from_date, to_date, filter)
print_csv(result, csv_file_path, output_format)
print_pdf_and_html(result, pdf_file_path, html_file_path)
print("From", from_date)
print("To", to_date)
print(f"Saved to: \n{csv_file_path} \n{pdf_file_path} \n{html_file_path}")
def execute_command(self):
core.execute(self, self.console_input.text())
def run(self):
'''
Run an instance of a tool
'''
run_tool = True
if db.check_if_tool_run(self.name, self.target):
print "Record found in run history for " + self.name + " on " + self.target
if core.prompt_tool_reruns:
response = raw_input("Would you like to re-run the tool? [n]")
if "y" in response or "Y" in response:
run_tool = True
else:
run_tool = False
else:
run_tool = False
if not run_tool:
return
#do not run aggressive tools if aggressive mode is not enabled
if self.aggressive and not core.aggressive:
return
if not os.path.exists(os.path.join(self.output_dir,
self.output_subdir)):
os.makedirs(os.path.join(self.output_dir, self.output_subdir))
print "Running " + self.name + " on " + self.target
if self.command:
output_file_path = os.path.join(
self.output_dir, self.output_subdir,
self.name + "_" + self.target + "." + self.output_format)
self.command = self.command.replace("[TARGET]", self.target)
self.command = self.command.replace("[OUTPUT]", output_file_path)
self.command = self.command.replace("[PROJECT]", core.projectname)
print self.command
self.start_time = core.getTimestamp(True)
self.command_result = core.execute(self.command, self.suppress_out)
self.end_time = core.getTimestamp(True)
if self.cleanup_regex <> "":
clean_result = re.findall(self.cleanup_regex,
self.command_result)
self.command_result = core.list_to_text(clean_result)
if not self.output_format:
db.add_run_to_db(self.name, self.target, self.command,
self.command_result, '', 'txt',
self.start_time, self.end_time,
self.output_subdir)
else:
db.add_run_to_db(self.name, self.target, self.command,
self.command_result, output_file_path,
self.output_format, self.start_time,
self.end_time, self.output_subdir)
print ""
#if no output directory is specified or tool outputs to file itself, then only output to screen...
if self.output_dir and not self.output_format:
output.write_outfile(
os.path.join(self.output_dir, self.output_subdir),
self.name + "_" + self.target + ".txt",
self.command_result)
if self.email_regex:
self.emails = sorted(
list(set(re.findall(self.email_regex,
self.command_result))))
if self.email_domain_filter:
self.emails = [
s for s in self.emails if self.email_domain_filter in s
]
for email in self.emails:
email = email.lower()
db.add_person_to_db(email)
print "Emails discovered: " + str(self.emails)
if self.dns_regex:
self.dns = sorted(
list(set(re.findall(self.dns_regex, self.command_result))))
for target in self.dns:
target = target.lower()
addresses = core.nslookup_fwd(target)
for address in addresses:
db.add_host_to_db(address, [target])
print "DNS entries discovered: " + str(self.dns)
if self.ip_regex:
self.ip = sorted(
list(set(re.findall(self.ip_regex, self.command_result))))
for target in self.ip:
hostnames = core.nslookup_rev(target)
for hostname in hostnames:
hostname = hostname.lower()
db.add_host_to_db(target, [hostname])
print "IPs discovered: " + str(self.ip)
print "\n" + "-" * 80 + "\n"
if self.url:
self.url = self.url.replace("[TARGET]", self.target)
output_file_path = os.path.join(
self.output_dir, self.output_subdir, self.name + "_" +
self.target + "." + self.website_output_format)
command = "cutycapt --url=" + self.url + " --delay=" + self.delay + " --out=" + output_file_path
#Check for $DISPLAY which returns null if no X server; required for cutycapt (cannot run in SSH / headless)
if os.environ.get('DISPLAY'):
core.execute(command, self.suppress_out)
db.add_run_to_db(self.name, self.target, self.command,
self.command_result, output_file_path,
self.website_output_format, self.start_time,
self.end_time, self.output_subdir)
else:
print "[!] No X server detected (maybe inside an SSH session?)"
print "[!] Cutycapt for screenshot requires X server...skipping... :("
#!/usr/bin/python
# -*- coding: utf-8 -*-
from core import execute
if __name__ == '__main__':
execute()
def __record(self, cpi_file_name, quiet=False):
""" Record the events and their values in a .cpi file
Parameters:
cpi_file_name - the path where the cpi file will be generated
quiet - if should suppress any message during the recording step
"""
ocount = "ocount"
core.supported_feature(core.get_processor(), "Breakdown")
if not os.path.isfile(self.__binary_path):
sys.stderr.write(self.__binary_path + ' binary file not found\n')
sys.exit(1)
timestamp = core.get_timestamp()
binary_name = self.__binary_path.split("/").pop(-1)
dir_current = os.getcwd()
ocount_out = dir_current + "/output"
if not core.cmdexists(ocount):
sys.stderr.write(ocount + " is not installed in the system. " +
"Install oprofile before continue." + "\n")
sys.exit(2)
reader = events_reader.EventsReader(core.get_processor())
if not cpi_file_name:
fname = dir_current + "/" + binary_name + "_" + timestamp + ".cpi"
cpi_file_name = fname
else:
dir_file = os.path.dirname(os.path.realpath(cpi_file_name))
if not os.path.exists(dir_file):
sys.stderr.write(dir_file + " directory not found\n")
return 1
elif os.path.isdir(cpi_file_name):
sys.stderr.write(cpi_file_name + " is not a file\n")
return 1
start_time = time.time()
exec_counter = 0
events = {}
# Run ocount for all events groups
for event in reader.get_events():
exec_counter = exec_counter + 1
ocount_cmd = ocount + " -b -f " + ocount_out
for item in event:
ocount_cmd += " -e " + item
if not quiet:
sys.stdout.write("\r Recording CPI Events: %d/%d "
"iterations (elapsed time: %d seconds)" %
(exec_counter, len(reader.get_events()),
(time.time() - start_time)))
sys.stdout.flush()
status, output = core.execute_stdout(ocount_cmd + ' ' +
self.__binary_path + ' ' +
self.__binary_args)
if status != 0:
sys.stderr.write(
"\n\nFailed to run {0} command.".format(ocount) + "\n" +
output + "\n")
sys.exit(1)
core.parse_file(ocount_out, events)
core.execute("rm " + ocount_out)
if not quiet:
print ""
core.save_events(events, cpi_file_name)
return events