def main(): init_logger() parser = create_parser() args = parser.parse_args() setup_logging(args.loglevel) logger.debug(args) # set ctlogs, type: [<ctutlz.ctlog.Log>, ...] logs_dict = args.fetch_ctlogs() set_operator_names(logs_dict) ctlogs = Logs(logs_dict['logs']) if args.log_list_filename: logs_dict = read_log_list(args.log_list_filename) set_operator_names(logs_dict) ctlogs = Logs(logs_dict['logs']) for host in args.hostname: scrape_and_verify_scts(host, args.verification_tasks, ctlogs)
def ctloglist(print_json=None): '''Gather ct-log lists and print the merged log list. Args: print_json(boolean): If True, print merged log list as json data. Else print as markdown. ''' if not print_json: today = datetime.date.today() now = datetime.datetime.now() logger.info('# Known Certificate Transparency (CT) Logs\n') logger.verbose('Created with [ctloglist]' '(https://github.com/theno/ctutlz#ctloglist)\n') logger.verbose( '* [all_logs_list.json](' 'https://www.gstatic.com/ct/log_list/v2/all_logs_list.json)' '\n') logger.info('Version (Date): %s\n' % today) logger.verbose('Datetime: %s\n' % now) logger.info('') # formatting: insert empty line # all_logs_list.json all_dict = download_log_list(URL_ALL_LOGS) orig_all_dict = dict(all_dict) set_operator_names(all_dict) all_logs = Logs([all_dict]) if print_json: json_str = json.dumps(orig_all_dict, indent=4, sort_keys=True) print(json_str) else: show_logs(all_logs, '')
def ctloglist(print_json=None): '''Gather ct-log lists and print the merged log list. Args: print_json(boolean): If True, print merged log list as json data. Else print as markdown. ''' if not print_json: today = datetime.date.today() now = datetime.datetime.now() logger.info('# Known Certificate Transparency (CT) Logs\n') logger.verbose('Created with [ctloglist]' '(https://github.com/theno/ctutlz#ctloglist)\n') logger.verbose('Merged log lists:') logger.verbose("* webpage [known logs]" '(https://www.certificate-transparency.org/known-logs)') logger.verbose('* [log_list.json]' '(https://www.gstatic.com/ct/log_list/log_list.json)') logger.verbose( '* [all_logs_list.json](' 'https://www.gstatic.com/ct/log_list/all_logs_list.json)' '\n') logger.info(flo('Version (Date): {today}\n')) logger.verbose(flo('Datetime: {now}\n')) logger.info('') # formatting: insert empty line # from webpage webpage_dict = logs_dict_from_webpage() all_from_webpage = Logs([ log_dict for log_list in [webpage_dict[key] for key in webpage_dict] for log_dict in log_list ]) included_from_webpage = Logs(webpage_dict.get('included_in_chrome', [])) try: webpage_dict.pop('included_in_chrome') except KeyError: pass frozen_from_webpage = Logs(webpage_dict.get('frozen_logs', [])) try: webpage_dict.pop('frozen_logs') except KeyError: pass pending_from_webpage = Logs( webpage_dict.get('pending_inclusion_in_chrome', [])) try: webpage_dict.pop('pending_inclusion_in_chrome') except KeyError: pass disqualified_from_webpage = \ Logs(webpage_dict.get('disqualified_from_chrome', [])) try: webpage_dict.pop('disqualified_from_chrome') except KeyError: pass rejected_from_webpage = Logs(webpage_dict.get('rejected_by_chrome', [])) try: webpage_dict.pop('rejected_by_chrome') except KeyError: pass distrusted_from_webpage = Logs( webpage_dict.get('completely_distrusted_by_chrome', [])) try: webpage_dict.pop('completely_distrusted_by_chrome') except KeyError: pass other_from_webpage = Logs(webpage_dict.get('other_logs', [])) try: webpage_dict.pop('other_logs') except KeyError: pass special_from_webpage = Logs(webpage_dict.get('special_purpose_logs', [])) try: webpage_dict.pop('special_purpose_logs') except KeyError: pass unknown_log_titles = [key for key in webpage_dict.keys()] if unknown_log_titles: logger.error( red( flo('unknown log titles (i.e. log states): {unknown_log_titles}' ))) # log_list.json: chrome ct policy compliant logs log_list_dict = download_log_list(URL_LOG_LIST) set_operator_names(log_list_dict) for log_dict in log_list_dict['logs']: if 'disqualified_at' in log_dict.keys(): log_dict['chrome_state'] = ChromeStates.DISQUALIFIED elif 'final_sth' in log_dict.keys(): log_dict['chrome_state'] = ChromeStates.FROZEN else: log_dict['chrome_state'] = ChromeStates.INCLUDED log_list_logs = Logs(log_list_dict['logs']) # all_logs_list.json all_dict = download_log_list(URL_ALL_LOGS) set_operator_names(all_dict) all_logs = Logs(all_dict['logs']) # merge lists and show the result log_lists = merge_log_lists(**locals()) if print_json: data = { 'operators': all_dict['operators'], 'logs': list_from_lists(log_lists) } unset_operator_names(data) json_str = json.dumps(data, indent=4, sort_keys=True) print(json_str) else: for item in log_lists: show_logs(item['logs'], item['heading'])