def send_results_via_rtir(resultfile, watchlistfile, configfile):
config = configparser.ConfigParser()
config.read_file(configfile)
ticketing = rt.Rt(config['rt']['uri'], config['rt']['username'],
config['rt']['password'])
if not ticketing.login():
raise ValueError('Login to RT not successful.')
logging.debug('Grouping results by mail address...')
grouped = group_by_mail(read_data(resultfile),
read_string_to_tree(watchlistfile.read()))
for address, data in grouped.items():
logging.debug('Creating ticket for %s...', address)
text = '\n\n'.join([
'\n'.join(['%s: %s' % row for row in block.items()])
for block in data
])
ticket_id = ticketing.create_ticket(
Queue=config['rt']['queue'],
Subject='Certificate Transparency Log Information',
Owner=config['rt']['username'],
Requestor=address,
Status='resolved',
Text=text)
if not ticket_id:
raise ValueError('Creating RT ticket not successful.')
logging.debug('Created ticket %d.', ticket_id)
else:
logging.debug('Empty input data.')
def predictions():
ADMINS = ['*****@*****.**']
if not app.debug:
import logging
from logging.handlers import SMTPHandler
mail_handler = SMTPHandler('127.0.0.1',
'*****@*****.**',
ADMINS, 'Predictions Failed')
mail_handler.setLevel(logging.ERROR)
app.logger.addHandler(mail_handler)
res = results.read_data()
scores = all_predictions.score(res)
scores.sort(key=operator.itemgetter(1), reverse=True)
scores = [(name.title(), score) for name, score in scores]
if not scores:
return str(all_predictions.predictions)
return render_template('scores.html', scores = scores)
import matplotlib.pyplot as plt
from collections import defaultdict
from itertools import combinations
from pprint import pprint
from scipy import stats
import random
from itertools import chain
import results
def choose_points(data):
return [d.total_response() for d in data]
def check_data(data):
#print stats.describe(choose_points(data['2']))
min_len = min(map(len, data.values()))
print stats.kruskal(*[choose_points(d[:min_len]) for d in data.values()])
print stats.wilcoxon(choose_points(data['1']), choose_points(data['2']))
data = results.read_data(bucket=r'^/api/\w{3}(\w)\w+/config$',
filename='data/out.parsed')
check_data(data)
print "\nFound correct username: "******"Elapsed time: ", (time.time() - start_time) / 60, 'min'
print "Total Attempts: ", count
print "=" * 80
control_the_lights(hue_url, username)
exit()
#pprint(res.json())
if count % 100 == 0:
now = time.time()
elapsed = now - start_time
print count, elapsed, 100 / (now - interval)
interval = now
if count % 3000 == 0:
time.sleep(0.3) # let any existing connections finish
print "Current guess: ", current_guess
print "Parsing data:"
subprocess.call('./parse_pcap.py data/*.pcap',
shell=True) # I know, I know
print "Calculating next guess:"
data = results.read_data(bucket=r'^/api/(%s\w)\w+/config$' %
current_guess,
data_dir='data')
next_guess = calculate_guess.next_guess(data)
if next_guess is not None:
current_guess = next_guess
print "CHANGING GUESS: ", current_guess
subprocess.call('make clean', shell=True) # dump old data
print "Collecting data:"
time.sleep(0.005)
random.shuffle(username_generators)
def next_guess(data):
# this is tuned for my device what a charset len of 8. Modify as
# appropriate.
res = analyze_data(data, p_threshold=0.1)
pprint(res)
values = sorted(res.values())
if values and values[-1] >= 6:
if (values[-1] - values[-2]) >= 2:
if sum(values[:-1]) < 13: # ?
return max(res, key=res.get)
if __name__=='__main__':
prefix_len = 5
data = results.read_data(bucket=r'^/api/(\w{%s})\w+/config$' % prefix_len,
data_dir='data',
postfix='.parsed')
pprint(analyze_data(data, p_threshold=0.1))
# length = 1501
# incr_length = 101
# max_len = data.minlen()
# start = random.randint(0, max_len - length)
# while True:
# # print length, start
# # this_data = data.sample(length, start)
# this_data = data
# if next_guess(this_data):
# print "Exiting", length, start,
print "Total Attempts: ", count
print "=" * 80
control_the_lights(hue_url, username)
exit()
#pprint(res.json())
if count % 100 == 0:
now = time.time()
elapsed = now - start_time
print "{} {:.2f} {:.2f}".format(
count, elapsed, 100 / (now - interval))
interval = now
if count % 3000 == 0:
time.sleep(0.05) # let any existing connections finish
print "Current guess: ", current_guess
print "Parsing data..."
subprocess.call(
'./parse_pcap.py data/*.pcap', shell=True) # I know, I know
print "Calculating next guess..."
data = results.read_data(
bucket=r'^/api/(%s\w)\w+/config$' % current_guess,
data_dir='data')
next_guess = calculate_guess.next_guess(data)
if next_guess is not None:
current_guess = next_guess
print "CHANGING GUESS: ", current_guess
print "Guess time: ", time.time() - guess_time
subprocess.call('make clean', shell=True) # dump old data
print "Collecting data:"
# time.sleep(0.005)
random.shuffle(username_generators)
import matplotlib.pyplot as plt
from collections import defaultdict
from itertools import combinations
from pprint import pprint
from scipy import stats, interpolate, signal
import random
from itertools import chain
import results
def choose_points(qr_list):
return [d.total_response() for d in qr_list]
def check_data(data):
""" graph the values """
tsdata = data.all_as_timeseries()
res = defaultdict(int)
for x in tsdata:
res[x.response_count()] += 1
pprint(res)
data = results.read_data(bucket=r'^/api/\w{5}(\w)\w{4}/config$',
filename='data/out.parsed')
check_data(data)
def test_result_reader_fileobj_string(self):
result = list(results.read_data(io.StringIO(result_string)))
self.assertEqual(result, result_expected)
import random
from itertools import chain
import results
def choose_points(qr_list):
return [d.total_response() - getattr(d, 'median', 0) for d in qr_list]
def check_data(data):
""" graph the values """
# data.median_filter(choose_points)
for key, value in data.items():
plt.plot([x.response[0] for x in value],
choose_points(value),
'.',
alpha=0.5,
label=str(key),
)
# plt.plot([x.response[0] for x in data.all_as_timeseries()],
# [x.median for x in data.median_filter(choose_points)])
plt.legend()
plt.show()
data = results.read_data(bucket=r'^/api/(\w{5})\w+/config$',
data_dir='data')
check_data(data)
def test_result_reader_string(self):
result = list(results.read_data(result_string))
self.assertEqual(result, result_expected)
def test_result_reader_no_data(self):
result = list(results.read_data(''))
self.assertEqual(result, [])
raise ValueError('Creating RT ticket not successful.')
logging.debug('Created ticket %d.', ticket_id)
else:
logging.debug('Empty input data.')
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('mode', choices=['group', 'send-rt'], default='group')
parser.add_argument('results', type=argparse.FileType('r'))
parser.add_argument('watchlist',
type=argparse.FileType('r'),
default='%s/.certspotter/watchlist' % HOME)
parser.add_argument('--config',
type=argparse.FileType('r'),
default='%s/.config/certspotter_processing.ini' % HOME)
parser.add_argument('--verbose',
action='store_true',
help='Output debugging information')
args = parser.parse_args()
if args.verbose:
logging.basicConfig(format='%(message)s', level='DEBUG')
if args.mode == 'group':
retval = group_by_mail(read_data(args.results),
read_string_to_tree(args.watchlist.read()))
print(json.dumps(retval))
if args.mode == 'send-rt':
send_results_via_rtir(args.results, args.watchlist, args.config)
count += 1
if "whitelist" in res.json(): # we got back a full config string...
print "=" * 80
print "\nFound correct username: "******"Elapsed time: ", (time.time() - start_time) / 60, "min"
print "Total Attempts: ", count
print "=" * 80
control_the_lights(hue_url, username)
exit()
# pprint(res.json())
if count % 100 == 0:
now = time.time()
elapsed = now - start_time
print count, elapsed, 100 / (now - interval)
interval = now
if count % 3000 == 0:
time.sleep(0.3) # let any existing connections finish
print "Current guess: ", current_guess
print "Parsing data:"
subprocess.call("./parse_pcap.py data/*.pcap", shell=True) # I know, I know
print "Calculating next guess:"
data = results.read_data(bucket=r"^/api/(%s\w)\w+/config$" % current_guess, data_dir="data")
next_guess = calculate_guess.next_guess(data)
if next_guess is not None:
current_guess = next_guess
print "CHANGING GUESS: ", current_guess
subprocess.call("make clean", shell=True) # dump old data
print "Collecting data:"
time.sleep(0.005)
random.shuffle(username_generators)
def next_guess(data):
# this is tuned for my device what a charset len of 8. Modify as
# appropriate.
res = analyze_data(data, p_threshold=0.1)
pprint(res)
values = sorted(res.values())
if values and values[-1] >= 6:
if (values[-1] - values[-2]) >= 2:
if sum(values[:-1]) < 13: # ?
return max(res, key=res.get)
if __name__ == '__main__':
prefix_len = 5
data = results.read_data(bucket=r'^/api/(\w{%s})\w+/config$' % prefix_len,
data_dir='data',
postfix='.parsed')
pprint(analyze_data(data, p_threshold=0.1))
# length = 1501
# incr_length = 101
# max_len = data.minlen()
# start = random.randint(0, max_len - length)
# while True:
# # print length, start
# # this_data = data.sample(length, start)
# this_data = data
# if next_guess(this_data):
# print "Exiting", length, start,
# the weeds, where there are some obviously VERY different values
# (e.g. p=1.0e-40) at play, but just counting them at 0.1 doesn't
# every resolve. This cranks up the threshold, in an attempt to
# draw them out.
if values[-1] == 7:
if max(values[:-1]) >= 5:
return next_guess(data, p_threshold / 10)
if values[-1] >= 6:
if (values[-1] - values[-2]) >= 2:
if sum(values[:-1]) < 12: # sometimes there's low level dissent.
return max(res, key=res.get)
def trim_data(data):
for k in data:
data[k] = data[k][15000:20000]
if __name__=='__main__':
prefix_len = len(users.USERNAME_PREFIX) + 1
data = results.read_data(bucket=r'^/api/(\w{%d})\w+/config$' % prefix_len)
#trim_data(data)
# res = analyze_data(data, p_threshold=0.1)
print "Counts:"
print "Next Guess: ", next_guess(data)
for k, v in data.items():
print k, len(choose_points(v))
# print "Results:"
# for k, v in sorted(res.items()):
# print "{}: {}".format(k, v)
def test_result_reader_fileobj_bytes(self):
result = list(results.read_data(io.BytesIO(result_string.encode())))
self.assertEqual(result, result_expected)
""" combinatoric KS, add hits """
data = data.sample(5001)
# data.median_filter(choose_points)
data_roundup = defaultdict(int)
for k1, k2 in combinations(data.keys(), 2):
# DON'T EVER USE A SAMPLE SIZE THAT IS A MULTIPLE OF 100
d, p = stats.ks_2samp(choose_points(data[k1]),
choose_points(data[k2]))
print k1, k2, d, p
if p < p_threshold:
data_roundup[k1] += 1
data_roundup[k2] += 1
return dict(data_roundup)
data = results.read_data(bucket=r'^/api/\w{3}(\w)\w{6}/config$',
data_dir='more_recent_data')
pprint(check_data(data))
exit()
correct = 0
incorrect = 0
unclear = 0
shortened = []
shorten_error = 0
ANSWER = '0'
for x in range(1000):
print "Iteration: ", x
res = check_data(data)
if not res:
unclear += 1
from pprint import pprint
from scipy import stats, interpolate, signal
import random
from itertools import chain
import calculate_guess
import results
def check_data(data):
""" graph the values """
for key, value in data.items():
print key, len([x.total_response() for x in value
if calculate_guess.keep_point(x)])
plt.plot([x.response[0] for x in value
if calculate_guess.keep_point(x)],
[x.total() for x in value
if calculate_guess.keep_point(x)],
'.',
alpha=0.5,
label=str(key),
)
# plt.plot([x.response[0] for x in data.all_as_timeseries()],
# [x.median for x in data.median_filter(choose_points)])
plt.legend()
plt.show()
data = results.read_data(bucket=r'^/api/(\w{1})\w+/config$')
check_data(data)
