def send_results_via_rtir(resultfile, watchlistfile, configfile): config = configparser.ConfigParser() config.read_file(configfile) ticketing = rt.Rt(config['rt']['uri'], config['rt']['username'], config['rt']['password']) if not ticketing.login(): raise ValueError('Login to RT not successful.') logging.debug('Grouping results by mail address...') grouped = group_by_mail(read_data(resultfile), read_string_to_tree(watchlistfile.read())) for address, data in grouped.items(): logging.debug('Creating ticket for %s...', address) text = '\n\n'.join([ '\n'.join(['%s: %s' % row for row in block.items()]) for block in data ]) ticket_id = ticketing.create_ticket( Queue=config['rt']['queue'], Subject='Certificate Transparency Log Information', Owner=config['rt']['username'], Requestor=address, Status='resolved', Text=text) if not ticket_id: raise ValueError('Creating RT ticket not successful.') logging.debug('Created ticket %d.', ticket_id) else: logging.debug('Empty input data.')
def predictions(): ADMINS = ['*****@*****.**'] if not app.debug: import logging from logging.handlers import SMTPHandler mail_handler = SMTPHandler('127.0.0.1', '*****@*****.**', ADMINS, 'Predictions Failed') mail_handler.setLevel(logging.ERROR) app.logger.addHandler(mail_handler) res = results.read_data() scores = all_predictions.score(res) scores.sort(key=operator.itemgetter(1), reverse=True) scores = [(name.title(), score) for name, score in scores] if not scores: return str(all_predictions.predictions) return render_template('scores.html', scores = scores)
import matplotlib.pyplot as plt from collections import defaultdict from itertools import combinations from pprint import pprint from scipy import stats import random from itertools import chain import results def choose_points(data): return [d.total_response() for d in data] def check_data(data): #print stats.describe(choose_points(data['2'])) min_len = min(map(len, data.values())) print stats.kruskal(*[choose_points(d[:min_len]) for d in data.values()]) print stats.wilcoxon(choose_points(data['1']), choose_points(data['2'])) data = results.read_data(bucket=r'^/api/\w{3}(\w)\w+/config$', filename='data/out.parsed') check_data(data)
print "\nFound correct username: "******"Elapsed time: ", (time.time() - start_time) / 60, 'min' print "Total Attempts: ", count print "=" * 80 control_the_lights(hue_url, username) exit() #pprint(res.json()) if count % 100 == 0: now = time.time() elapsed = now - start_time print count, elapsed, 100 / (now - interval) interval = now if count % 3000 == 0: time.sleep(0.3) # let any existing connections finish print "Current guess: ", current_guess print "Parsing data:" subprocess.call('./parse_pcap.py data/*.pcap', shell=True) # I know, I know print "Calculating next guess:" data = results.read_data(bucket=r'^/api/(%s\w)\w+/config$' % current_guess, data_dir='data') next_guess = calculate_guess.next_guess(data) if next_guess is not None: current_guess = next_guess print "CHANGING GUESS: ", current_guess subprocess.call('make clean', shell=True) # dump old data print "Collecting data:" time.sleep(0.005) random.shuffle(username_generators)
def next_guess(data): # this is tuned for my device what a charset len of 8. Modify as # appropriate. res = analyze_data(data, p_threshold=0.1) pprint(res) values = sorted(res.values()) if values and values[-1] >= 6: if (values[-1] - values[-2]) >= 2: if sum(values[:-1]) < 13: # ? return max(res, key=res.get) if __name__=='__main__': prefix_len = 5 data = results.read_data(bucket=r'^/api/(\w{%s})\w+/config$' % prefix_len, data_dir='data', postfix='.parsed') pprint(analyze_data(data, p_threshold=0.1)) # length = 1501 # incr_length = 101 # max_len = data.minlen() # start = random.randint(0, max_len - length) # while True: # # print length, start # # this_data = data.sample(length, start) # this_data = data # if next_guess(this_data): # print "Exiting", length, start,
print "Total Attempts: ", count print "=" * 80 control_the_lights(hue_url, username) exit() #pprint(res.json()) if count % 100 == 0: now = time.time() elapsed = now - start_time print "{} {:.2f} {:.2f}".format( count, elapsed, 100 / (now - interval)) interval = now if count % 3000 == 0: time.sleep(0.05) # let any existing connections finish print "Current guess: ", current_guess print "Parsing data..." subprocess.call( './parse_pcap.py data/*.pcap', shell=True) # I know, I know print "Calculating next guess..." data = results.read_data( bucket=r'^/api/(%s\w)\w+/config$' % current_guess, data_dir='data') next_guess = calculate_guess.next_guess(data) if next_guess is not None: current_guess = next_guess print "CHANGING GUESS: ", current_guess print "Guess time: ", time.time() - guess_time subprocess.call('make clean', shell=True) # dump old data print "Collecting data:" # time.sleep(0.005) random.shuffle(username_generators)
import matplotlib.pyplot as plt from collections import defaultdict from itertools import combinations from pprint import pprint from scipy import stats, interpolate, signal import random from itertools import chain import results def choose_points(qr_list): return [d.total_response() for d in qr_list] def check_data(data): """ graph the values """ tsdata = data.all_as_timeseries() res = defaultdict(int) for x in tsdata: res[x.response_count()] += 1 pprint(res) data = results.read_data(bucket=r'^/api/\w{5}(\w)\w{4}/config$', filename='data/out.parsed') check_data(data)
def test_result_reader_fileobj_string(self): result = list(results.read_data(io.StringIO(result_string))) self.assertEqual(result, result_expected)
import random from itertools import chain import results def choose_points(qr_list): return [d.total_response() - getattr(d, 'median', 0) for d in qr_list] def check_data(data): """ graph the values """ # data.median_filter(choose_points) for key, value in data.items(): plt.plot([x.response[0] for x in value], choose_points(value), '.', alpha=0.5, label=str(key), ) # plt.plot([x.response[0] for x in data.all_as_timeseries()], # [x.median for x in data.median_filter(choose_points)]) plt.legend() plt.show() data = results.read_data(bucket=r'^/api/(\w{5})\w+/config$', data_dir='data') check_data(data)
def test_result_reader_string(self): result = list(results.read_data(result_string)) self.assertEqual(result, result_expected)
def test_result_reader_no_data(self): result = list(results.read_data('')) self.assertEqual(result, [])
raise ValueError('Creating RT ticket not successful.') logging.debug('Created ticket %d.', ticket_id) else: logging.debug('Empty input data.') if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('mode', choices=['group', 'send-rt'], default='group') parser.add_argument('results', type=argparse.FileType('r')) parser.add_argument('watchlist', type=argparse.FileType('r'), default='%s/.certspotter/watchlist' % HOME) parser.add_argument('--config', type=argparse.FileType('r'), default='%s/.config/certspotter_processing.ini' % HOME) parser.add_argument('--verbose', action='store_true', help='Output debugging information') args = parser.parse_args() if args.verbose: logging.basicConfig(format='%(message)s', level='DEBUG') if args.mode == 'group': retval = group_by_mail(read_data(args.results), read_string_to_tree(args.watchlist.read())) print(json.dumps(retval)) if args.mode == 'send-rt': send_results_via_rtir(args.results, args.watchlist, args.config)
count += 1 if "whitelist" in res.json(): # we got back a full config string... print "=" * 80 print "\nFound correct username: "******"Elapsed time: ", (time.time() - start_time) / 60, "min" print "Total Attempts: ", count print "=" * 80 control_the_lights(hue_url, username) exit() # pprint(res.json()) if count % 100 == 0: now = time.time() elapsed = now - start_time print count, elapsed, 100 / (now - interval) interval = now if count % 3000 == 0: time.sleep(0.3) # let any existing connections finish print "Current guess: ", current_guess print "Parsing data:" subprocess.call("./parse_pcap.py data/*.pcap", shell=True) # I know, I know print "Calculating next guess:" data = results.read_data(bucket=r"^/api/(%s\w)\w+/config$" % current_guess, data_dir="data") next_guess = calculate_guess.next_guess(data) if next_guess is not None: current_guess = next_guess print "CHANGING GUESS: ", current_guess subprocess.call("make clean", shell=True) # dump old data print "Collecting data:" time.sleep(0.005) random.shuffle(username_generators)
def next_guess(data): # this is tuned for my device what a charset len of 8. Modify as # appropriate. res = analyze_data(data, p_threshold=0.1) pprint(res) values = sorted(res.values()) if values and values[-1] >= 6: if (values[-1] - values[-2]) >= 2: if sum(values[:-1]) < 13: # ? return max(res, key=res.get) if __name__ == '__main__': prefix_len = 5 data = results.read_data(bucket=r'^/api/(\w{%s})\w+/config$' % prefix_len, data_dir='data', postfix='.parsed') pprint(analyze_data(data, p_threshold=0.1)) # length = 1501 # incr_length = 101 # max_len = data.minlen() # start = random.randint(0, max_len - length) # while True: # # print length, start # # this_data = data.sample(length, start) # this_data = data # if next_guess(this_data): # print "Exiting", length, start,
# the weeds, where there are some obviously VERY different values # (e.g. p=1.0e-40) at play, but just counting them at 0.1 doesn't # every resolve. This cranks up the threshold, in an attempt to # draw them out. if values[-1] == 7: if max(values[:-1]) >= 5: return next_guess(data, p_threshold / 10) if values[-1] >= 6: if (values[-1] - values[-2]) >= 2: if sum(values[:-1]) < 12: # sometimes there's low level dissent. return max(res, key=res.get) def trim_data(data): for k in data: data[k] = data[k][15000:20000] if __name__=='__main__': prefix_len = len(users.USERNAME_PREFIX) + 1 data = results.read_data(bucket=r'^/api/(\w{%d})\w+/config$' % prefix_len) #trim_data(data) # res = analyze_data(data, p_threshold=0.1) print "Counts:" print "Next Guess: ", next_guess(data) for k, v in data.items(): print k, len(choose_points(v)) # print "Results:" # for k, v in sorted(res.items()): # print "{}: {}".format(k, v)
def test_result_reader_fileobj_bytes(self): result = list(results.read_data(io.BytesIO(result_string.encode()))) self.assertEqual(result, result_expected)
""" combinatoric KS, add hits """ data = data.sample(5001) # data.median_filter(choose_points) data_roundup = defaultdict(int) for k1, k2 in combinations(data.keys(), 2): # DON'T EVER USE A SAMPLE SIZE THAT IS A MULTIPLE OF 100 d, p = stats.ks_2samp(choose_points(data[k1]), choose_points(data[k2])) print k1, k2, d, p if p < p_threshold: data_roundup[k1] += 1 data_roundup[k2] += 1 return dict(data_roundup) data = results.read_data(bucket=r'^/api/\w{3}(\w)\w{6}/config$', data_dir='more_recent_data') pprint(check_data(data)) exit() correct = 0 incorrect = 0 unclear = 0 shortened = [] shorten_error = 0 ANSWER = '0' for x in range(1000): print "Iteration: ", x res = check_data(data) if not res: unclear += 1
from pprint import pprint from scipy import stats, interpolate, signal import random from itertools import chain import calculate_guess import results def check_data(data): """ graph the values """ for key, value in data.items(): print key, len([x.total_response() for x in value if calculate_guess.keep_point(x)]) plt.plot([x.response[0] for x in value if calculate_guess.keep_point(x)], [x.total() for x in value if calculate_guess.keep_point(x)], '.', alpha=0.5, label=str(key), ) # plt.plot([x.response[0] for x in data.all_as_timeseries()], # [x.median for x in data.median_filter(choose_points)]) plt.legend() plt.show() data = results.read_data(bucket=r'^/api/(\w{1})\w+/config$') check_data(data)