def __revert_installation(self):
output = True
if len(self.urls_backup) > 0:
urls_data = ""
for line in self.urls_backup:
urls_data += line
output = write_to_file(self.urls_file, urls_data)
if len(self.settings_backup) > 0:
settings_data = ""
for line in self.settings_backup:
settings_data += line
output = write_to_file(self.settings_file, settings_data)
for app_path in self.delete_app_dir:
output = delete_dir(app_path)
if output:
for plugin_path in self.delete_plugins_dir:
if output:
output = delete_dir(plugin_path)
return output
def get_ssh_credentials(nels_id):
credentail = storage.get_ssh_credential(nels_id)
if not credentail:
return None
(host, username) = (credentail[0], credentail[1])
feed_utils.info("fetching keys. host: %s, username:%s " % (host, username))
key_file = path.join(config.TEMP_DIR, "%s.nels" % nels_id)
feed_utils.info("writing key file: %s" % key_file)
file_utils.write_to_file(key_file, credentail[2])
run_utils.launch_cmd("chmod 600 %s" % key_file)
return [credentail[0], credentail[1], key_file]
def __recover_app(self):
if len(self.settings_backup) > 0:
settings_data = ""
for line in self.settings_backup:
settings_data += line
write_to_file(self.settings_file, settings_data)
if len(self.urls_backup) > 0:
urls_data = ""
for line in self.urls_backup:
urls_data += line
write_to_file(self.urls_file, urls_data)
def calculate_accuracy_and_loss_and_write_report(self):
avg_acc = self.calculate_mean(self.accuracies)
avg_loss = self.calculate_mean(self.losses)
file_utils.write_to_file("\n" + str(self.losses),
"\n" + str(self.accuracies),
str(avg_loss),
str(avg_acc),
path="test/")
print("Accuracy ", avg_acc)
print("Loss", avg_loss)
def __edit_urls_py(self):
checker = "RedirectView.as_view(url='/katana/')"
data = read_json_data(self.wf_config_file)
if data["app"]["url"].startswith("/"):
app_url = data["app"]["url"][1:]
else:
app_url = data["app"]["url"]
self.urls_inclusions.append("url(r'^" + app_url + "', include('" +
data["app"]["include"] + "')),")
data = readlines_from_file(self.urls_file)
self.urls_backup = data
index = -1
for i in range(0, len(data)):
if checker in data[i]:
index = i + 1
break
white_space = data[index].split("url")
for i in range(0, len(self.urls_inclusions)):
self.urls_inclusions[
i] = white_space[0] + self.urls_inclusions[i] + "\n"
u_data = data[:index]
u_data.extend(self.urls_inclusions)
u_data.extend(data[index:])
urls_data = ""
for line in u_data:
urls_data += line
output = write_to_file(self.urls_file, urls_data)
return output
def __remove_app_from_urls(self):
print(self.include_urls)
data = readlines_from_file(self.urls_file)
urls_data = ""
for url in self.include_urls:
urls_data = ""
for line in data:
if url not in line:
urls_data += line
data = urls_data
output = write_to_file(self.urls_file, urls_data)
return output
def __remove_app_from_settings(self):
data = readlines_from_file(self.settings_file)
sf_data = []
for line in data:
if line.strip() != "'{0}',".format(self.pkg_in_settings):
sf_data.append(line)
settings_data = ""
for line in sf_data:
settings_data += line
output = write_to_file(self.settings_file, settings_data)
return output
def __edit_settings_py(self):
data = readlines_from_file(self.settings_file)
self.settings_backup = data
index = -1
for i in range(0, len(data)):
if "wui.core" in data[i]:
index = i
break
white_space = data[index].split("'")
self.pkg_in_settings = white_space[
0] + "'" + self.pkg_in_settings + "',\n"
sf_data = data[:index]
sf_data.append(self.pkg_in_settings)
sf_data.extend(data[index:])
settings_data = ""
for line in sf_data:
settings_data += line
output = write_to_file(self.settings_file, settings_data)
return output
def test_add_symlink(self):
test_link = self.new_temp_file('test_link')
src_file = self.new_temp_file('linktest.txt')
fu.write_to_file(src_file, "link test")
fu.add_symlink(test_link, src_file)
self.failUnless(os.path.lexists(test_link))
def test_write_to_file(self):
filename = self.new_temp_file('write_test.txt')
random_str = ut.rand_str(100)
fu.write_to_file(filename, random_str)
self.assertEqual(random_str, fu.read_file(filename))
def gen_crawl_report(db_file,
db_pass2=None,
db_other_profs=None,
prof_dir=None):
""" visits_cnt, cookies, localstorage, flash cookies, cache, indexeddb,
http reqs/resps
canvas: list distinct FPers, linked to the sites that include this FPer
evercookie: list potential evercookies by searching ID-like common strings
among different vectors"""
out_dir = os.path.dirname(db_file)
crawl_name = os.path.basename(os.path.dirname(db_file))
figs = [] # figures to be plotted, removed for now.
respawned = []
if db_pass2 and db_other_profs and prof_dir:
respawned = ev.get_flash_evercookies(db_file, db_pass2, db_other_profs,
prof_dir)
start, end = dbu.get_db_entry(db_file, dbu.DBCmd.GET_VISIT_DATES, False)
visits_cnt = dbu.get_db_entry(db_file, dbu.DBCmd.COUNT_VISITS, False)[0]
completed_visits_cnt = dbu.get_db_entry(db_file, dbu.DBCmd.COUNT_VISITS,
True)[0]
cookies = dbu.get_db_entry(db_file, dbu.DBCmd.COUNT_COOKIES, 0)
localstorage = dbu.get_db_entry(db_file, dbu.DBCmd.COUNT_LOCALSTORAGE, 0)
print "genreport len(localstorage)", len(localstorage)
xsite_flash_cookies = get_xsite_flash_cookies(db_file)
xsite_local_storage = get_xsite_local_storage(db_file)
try:
flash_cookie_count = dbu.get_db_entry(db_file, dbu.DBCmd.COUNT_LSO, 0)
except:
flash_cookie_count = [""]
canvas_meta_rows = dbu.get_db_entry(db_file, dbu.DBCmd.GET_CANVAS_META, 0)
canvas_scr_domains = {}
canvas_events_per_script = {}
canvas_url_counts = {}
canvas_domain_counts = {}
canvas_script_urls = dbu.get_db_entry(db_file,
dbu.DBCmd.GET_CANVAS_SCRIPTS, 0)
false_positives = []
for canvas_script_url_tup in canvas_script_urls:
canvas_script_url = canvas_script_url_tup[0]
canvas_events = dbu.get_db_entry(db_file,
dbu.DBCmd.GET_CANVAS_EVENTS_BY_SCRIPT,
canvas_script_url)
if not ca.is_canvas_false_positive(canvas_events):
scr_evs = dbu.get_db_entry(db_file,
dbu.DBCmd.GET_CANVAS_EVENTS_BY_SCRIPT,
canvas_script_url)
canvas_events_per_script[canvas_script_url] = scr_evs
url_cnts = dbu.get_db_entry(db_file,
dbu.DBCmd.COUNT_SITES_BY_CANVAS_SCRIPT,
canvas_script_url)
canvas_url_counts[canvas_script_url] = url_cnts
domain = cu.extract_domain(canvas_script_url)
if domain in canvas_scr_domains:
canvas_scr_domains[domain].append(canvas_script_url)
else:
canvas_scr_domains[domain] = [canvas_script_url]
else:
false_positives.append(canvas_script_url_tup)
# print canvas_script_url_tup
# Remove false positives
for false_positive in false_positives:
canvas_script_urls.remove(false_positive)
# total_canvas_fp_count = sum()
all_canvasfp_ranks = {}
all_canvasfp_ranks_urls = {}
for canvas_scr_domain, canvas_scr_urls in canvas_scr_domains.iteritems():
script_ranks_and_urls =\
dbu.get_db_entry(db_file,
dbu.DBCmd.GET_RANK_AND_URLS_BY_CANVAS_SCRIPTS,
canvas_scr_urls)
canvas_domain_counts[canvas_scr_domain] = len(script_ranks_and_urls)
all_canvasfp_ranks[canvas_scr_domain] = map(lambda x: x[0],
script_ranks_and_urls)
all_canvasfp_ranks_urls[canvas_scr_domain] = script_ranks_and_urls
# print all_canvasfp_ranks
# fu.write_to_file(j(out_dir, "%s-canvas.json" % crawl_name),
# json.dumps(all_canvasfp_ranks))
total_canvas_fp_count = sum(canvas_domain_counts.itervalues())
# print "Total canvas FP count", total_canvas_fp_count
rank_set = set()
for _, v in all_canvasfp_ranks.iteritems():
for rank in v:
rank_set.add(rank)
# print "Total canvas FP count - uniq", len(rank_set)
nameSpace = {
'title': "Crawl Report",
'visits_cnt': visits_cnt,
'completed_visits_cnt': completed_visits_cnt,
'cookies': cookies[0],
'localstorage': localstorage[0],
'flash_cookie_count': flash_cookie_count[0],
'canvas_meta_rows': canvas_meta_rows,
'start': start,
'end': end,
'canvas_domain_counts': canvas_domain_counts,
'canvas_url_counts': canvas_url_counts,
'canvas_events_per_script': canvas_events_per_script,
'canvas_scr_domains': canvas_scr_domains,
'total_canvas_fp_count': total_canvas_fp_count,
'canvas_script_urls': canvas_script_urls,
'get_tld': cu.extract_domain,
'xsite_flash_cookies': xsite_flash_cookies,
'xsite_local_storages': xsite_local_storage,
'respawned': respawned,
'figs': figs,
'canvasfp_ranks_urls': all_canvasfp_ranks_urls,
# '3rdp_cookies': 3rdp_cookies,
}
report_template = Template(template_str, searchList=[nameSpace])
fu.write_to_file(j(out_dir, "%s-report.html" % crawl_name),
str(report_template))
def gen_crawl_report(db_file, db_pass2=None, db_other_profs=None,
prof_dir=None):
""" visits_cnt, cookies, localstorage, flash cookies, cache, indexeddb,
http reqs/resps
canvas: list distinct FPers, linked to the sites that include this FPer
evercookie: list potential evercookies by searching ID-like common strings
among different vectors"""
out_dir = os.path.dirname(db_file)
crawl_name = os.path.basename(os.path.dirname(db_file))
figs = [] # figures to be plotted, removed for now.
respawned = []
if db_pass2 and db_other_profs and prof_dir:
respawned = ev.get_flash_evercookies(db_file, db_pass2,
db_other_profs, prof_dir)
start, end = dbu.get_db_entry(db_file, dbu.DBCmd.GET_VISIT_DATES, False)
visits_cnt = dbu.get_db_entry(db_file, dbu.DBCmd.COUNT_VISITS, False)[0]
completed_visits_cnt = dbu.get_db_entry(db_file,
dbu.DBCmd.COUNT_VISITS, True)[0]
cookies = dbu.get_db_entry(db_file, dbu.DBCmd.COUNT_COOKIES, 0)
localstorage = dbu.get_db_entry(db_file, dbu.DBCmd.COUNT_LOCALSTORAGE, 0)
print "genreport len(localstorage)", len(localstorage)
xsite_flash_cookies = get_xsite_flash_cookies(db_file)
xsite_local_storage = get_xsite_local_storage(db_file)
try:
flash_cookie_count = dbu.get_db_entry(db_file, dbu.DBCmd.COUNT_LSO, 0)
except:
flash_cookie_count = [""]
canvas_meta_rows = dbu.get_db_entry(db_file, dbu.DBCmd.GET_CANVAS_META, 0)
canvas_scr_domains = {}
canvas_events_per_script = {}
canvas_url_counts = {}
canvas_domain_counts = {}
canvas_script_urls = dbu.get_db_entry(db_file,
dbu.DBCmd.GET_CANVAS_SCRIPTS, 0)
false_positives = []
for canvas_script_url_tup in canvas_script_urls:
canvas_script_url = canvas_script_url_tup[0]
canvas_events = dbu.get_db_entry(db_file,
dbu.DBCmd.GET_CANVAS_EVENTS_BY_SCRIPT,
canvas_script_url)
if not ca.is_canvas_false_positive(canvas_events):
scr_evs = dbu.get_db_entry(db_file,
dbu.DBCmd.GET_CANVAS_EVENTS_BY_SCRIPT,
canvas_script_url)
canvas_events_per_script[canvas_script_url] = scr_evs
url_cnts = dbu.get_db_entry(db_file,
dbu.DBCmd.COUNT_SITES_BY_CANVAS_SCRIPT,
canvas_script_url)
canvas_url_counts[canvas_script_url] = url_cnts
domain = cu.extract_domain(canvas_script_url)
if domain in canvas_scr_domains:
canvas_scr_domains[domain].append(canvas_script_url)
else:
canvas_scr_domains[domain] = [canvas_script_url]
else:
false_positives.append(canvas_script_url_tup)
# print canvas_script_url_tup
# Remove false positives
for false_positive in false_positives:
canvas_script_urls.remove(false_positive)
# total_canvas_fp_count = sum()
all_canvasfp_ranks = {}
all_canvasfp_ranks_urls = {}
for canvas_scr_domain, canvas_scr_urls in canvas_scr_domains.iteritems():
script_ranks_and_urls =\
dbu.get_db_entry(db_file,
dbu.DBCmd.GET_RANK_AND_URLS_BY_CANVAS_SCRIPTS,
canvas_scr_urls)
canvas_domain_counts[canvas_scr_domain] = len(script_ranks_and_urls)
all_canvasfp_ranks[canvas_scr_domain] = map(lambda x: x[0],
script_ranks_and_urls)
all_canvasfp_ranks_urls[canvas_scr_domain] = script_ranks_and_urls
# print all_canvasfp_ranks
# fu.write_to_file(j(out_dir, "%s-canvas.json" % crawl_name),
# json.dumps(all_canvasfp_ranks))
total_canvas_fp_count = sum(canvas_domain_counts.itervalues())
# print "Total canvas FP count", total_canvas_fp_count
rank_set = set()
for _, v in all_canvasfp_ranks.iteritems():
for rank in v:
rank_set.add(rank)
# print "Total canvas FP count - uniq", len(rank_set)
nameSpace = {'title': "Crawl Report",
'visits_cnt': visits_cnt,
'completed_visits_cnt': completed_visits_cnt,
'cookies': cookies[0],
'localstorage': localstorage[0],
'flash_cookie_count': flash_cookie_count[0],
'canvas_meta_rows': canvas_meta_rows,
'start': start,
'end': end,
'canvas_domain_counts': canvas_domain_counts,
'canvas_url_counts': canvas_url_counts,
'canvas_events_per_script': canvas_events_per_script,
'canvas_scr_domains': canvas_scr_domains,
'total_canvas_fp_count': total_canvas_fp_count,
'canvas_script_urls': canvas_script_urls,
'get_tld': cu.extract_domain,
'xsite_flash_cookies': xsite_flash_cookies,
'xsite_local_storages': xsite_local_storage,
'respawned': respawned,
'figs': figs,
'canvasfp_ranks_urls': all_canvasfp_ranks_urls,
# '3rdp_cookies': 3rdp_cookies,
}
report_template = Template(template_str, searchList=[nameSpace])
fu.write_to_file(j(out_dir, "%s-report.html" % crawl_name),
str(report_template))
