def show_logs(logs, heading, order=2):
if len(logs) <= 0:
return
logger.info('#' * order + '%s\n' % ' ' + heading if heading else '')
s_or_not = 's'
if len(logs) == 1:
s_or_not = ''
# show log size
logger.info('%i log%s\n' % (len(logs), s_or_not))
# list log urls
for log in logs:
if logger.level < logging.INFO:
anchor = log.url.replace('/', '')
logger.verbose('* [%s](#%s)' % (log.url, anchor))
else:
logger.info('* %s' % log.url)
logger.info('')
for log in logs:
show_log(log)
logger.info('End of list')
def show_logs(logs, heading, order=2):
if len(logs) > 0:
logger.info('#' * order + flo(' {heading}\n'))
s_or_not = 's'
if len(logs) == 1:
s_or_not = ''
# show log size
logger.info('%i log%s\n' % (len(logs), s_or_not))
# list log urls
for log in logs:
if logger.level < logging.INFO:
anchor = log.url.replace('/', '')
logger.verbose(flo('* [{log.url}](#{anchor})'))
else:
logger.info(flo('* {log.url}'))
logger.info('')
for log in logs:
show_log(log)
def ctloglist(print_json=None):
'''Gather ct-log lists and print the merged log list.
Args:
print_json(boolean): If True, print merged log list as json data.
Else print as markdown.
'''
if not print_json:
today = datetime.date.today()
now = datetime.datetime.now()
logger.info('# Known Certificate Transparency (CT) Logs\n')
logger.verbose('Created with [ctloglist]'
'(https://github.com/theno/ctutlz#ctloglist)\n')
logger.verbose(
'* [all_logs_list.json]('
'https://www.gstatic.com/ct/log_list/v2/all_logs_list.json)'
'\n')
logger.info('Version (Date): %s\n' % today)
logger.verbose('Datetime: %s\n' % now)
logger.info('') # formatting: insert empty line
# all_logs_list.json
all_dict = download_log_list(URL_ALL_LOGS)
orig_all_dict = dict(all_dict)
set_operator_names(all_dict)
all_logs = Logs([all_dict])
if print_json:
json_str = json.dumps(orig_all_dict, indent=4, sort_keys=True)
print(json_str)
else:
show_logs(all_logs, '')
def ctloglist(print_json=None):
'''Gather ct-log lists and print the merged log list.
Args:
print_json(boolean): If True, print merged log list as json data.
Else print as markdown.
'''
if not print_json:
today = datetime.date.today()
now = datetime.datetime.now()
logger.info('# Known Certificate Transparency (CT) Logs\n')
logger.verbose('Created with [ctloglist]'
'(https://github.com/theno/ctutlz#ctloglist)\n')
logger.verbose('Merged log lists:')
logger.verbose("* webpage [known logs]"
'(https://www.certificate-transparency.org/known-logs)')
logger.verbose('* [log_list.json]'
'(https://www.gstatic.com/ct/log_list/log_list.json)')
logger.verbose(
'* [all_logs_list.json]('
'https://www.gstatic.com/ct/log_list/all_logs_list.json)'
'\n')
logger.info(flo('Version (Date): {today}\n'))
logger.verbose(flo('Datetime: {now}\n'))
logger.info('') # formatting: insert empty line
# from webpage
webpage_dict = logs_dict_from_webpage()
all_from_webpage = Logs([
log_dict for log_list in [webpage_dict[key] for key in webpage_dict]
for log_dict in log_list
])
included_from_webpage = Logs(webpage_dict.get('included_in_chrome', []))
try:
webpage_dict.pop('included_in_chrome')
except KeyError:
pass
frozen_from_webpage = Logs(webpage_dict.get('frozen_logs', []))
try:
webpage_dict.pop('frozen_logs')
except KeyError:
pass
pending_from_webpage = Logs(
webpage_dict.get('pending_inclusion_in_chrome', []))
try:
webpage_dict.pop('pending_inclusion_in_chrome')
except KeyError:
pass
disqualified_from_webpage = \
Logs(webpage_dict.get('disqualified_from_chrome', []))
try:
webpage_dict.pop('disqualified_from_chrome')
except KeyError:
pass
rejected_from_webpage = Logs(webpage_dict.get('rejected_by_chrome', []))
try:
webpage_dict.pop('rejected_by_chrome')
except KeyError:
pass
distrusted_from_webpage = Logs(
webpage_dict.get('completely_distrusted_by_chrome', []))
try:
webpage_dict.pop('completely_distrusted_by_chrome')
except KeyError:
pass
other_from_webpage = Logs(webpage_dict.get('other_logs', []))
try:
webpage_dict.pop('other_logs')
except KeyError:
pass
special_from_webpage = Logs(webpage_dict.get('special_purpose_logs', []))
try:
webpage_dict.pop('special_purpose_logs')
except KeyError:
pass
unknown_log_titles = [key for key in webpage_dict.keys()]
if unknown_log_titles:
logger.error(
red(
flo('unknown log titles (i.e. log states): {unknown_log_titles}'
)))
# log_list.json: chrome ct policy compliant logs
log_list_dict = download_log_list(URL_LOG_LIST)
set_operator_names(log_list_dict)
for log_dict in log_list_dict['logs']:
if 'disqualified_at' in log_dict.keys():
log_dict['chrome_state'] = ChromeStates.DISQUALIFIED
elif 'final_sth' in log_dict.keys():
log_dict['chrome_state'] = ChromeStates.FROZEN
else:
log_dict['chrome_state'] = ChromeStates.INCLUDED
log_list_logs = Logs(log_list_dict['logs'])
# all_logs_list.json
all_dict = download_log_list(URL_ALL_LOGS)
set_operator_names(all_dict)
all_logs = Logs(all_dict['logs'])
# merge lists and show the result
log_lists = merge_log_lists(**locals())
if print_json:
data = {
'operators': all_dict['operators'],
'logs': list_from_lists(log_lists)
}
unset_operator_names(data)
json_str = json.dumps(data, indent=4, sort_keys=True)
print(json_str)
else:
for item in log_lists:
show_logs(item['logs'], item['heading'])
def scrape_and_verify_scts(hostname, verification_tasks, ctlogs):
logger.info(flo('# {hostname}\n'))
res = do_handshake(hostname,
scts_tls=(verify_scts_by_tls in verification_tasks),
scts_ocsp=(verify_scts_by_ocsp in verification_tasks))
if res.ee_cert_der:
logger.debug('got certificate\n')
if res.ee_cert.is_ev_cert:
logger.info('* EV cert')
else:
logger.info('* no EV cert')
if res.ee_cert.is_letsencrypt_cert:
logger.info("* issued by Let's Encrypt\n")
else:
logger.info("* not issued by Let's Encrypt\n")
if res.err:
logger.warn(res.err)
else:
for verification_task in verification_tasks:
logger.info(flo('## {verification_task.__name__}\n'))
verifications = verification_task(res, ctlogs)
if verifications:
for verification in verifications:
show_verification(verification)
elif res.ee_cert_der is not None:
logger.info('no SCTs\n')
def show_verification(verification):
'''
Args:
verification(ctutlz.sct.verification.SctVerificationResult)
'''
sct = verification.sct
sct_log_id1, sct_log_id2 = [to_hex(val)
for val
in struct.unpack("!16s16s", sct.log_id.tdf)]
logger.info('```')
logger.verbose('=' * 59)
logger.verbose(flo('Version : {sct.version_hex}'))
logger.verbose(flo('LogID : {sct_log_id1}'))
logger.verbose(flo(' {sct_log_id2}'))
logger.info(flo('LogID b64 : {sct.log_id_b64}'))
logger.verbose(flo('Timestamp : {sct.timestamp} ({sct.timestamp_hex})'))
logger.verbose(flo(
'Extensions: {sct.extensions_len} ({sct.extensions_len_hex})'))
logger.verbose(flo('Algorithms: {sct.signature_alg_hash_hex}/'
'{sct.signature_algorithm_signature} (hash/sign)'))
show_signature_verbose(sct.signature)
prefix = 'Sign. b64 : '
logger.info(prefix + text_with_newlines(sct.signature_b64, line_length=16*3,
newline='\n' + ' '*len(prefix)))
logger.verbose('--') # visual gap between sct infos and verification result
log = verification.log
if log is None:
logger.info('Log not found\n')
else:
logger.info(flo('Log found : {log.description}'))
logger.verbose('Operator : ' + ', '.join(log.operated_by))
logger.info(flo('Chrome : {log.chrome_state}'))
if verification.verified:
logger.info(flo('Result : Verified OK'))
else:
logger.info(flo('Result : Verification Failure'))
logger.info('```\n')
def show_verification(verification):
'''
Args:
verification(ctutlz.sct.verification.SctVerificationResult)
'''
sct = verification.sct
sct_log_id1, sct_log_id2 = [
to_hex(val) for val in struct.unpack("!16s16s", sct.log_id.tdf)
]
logger.info('```')
logger.verbose('=' * 59)
logger.verbose('Version : %s' % sct.version_hex)
logger.verbose('LogID : %s' % sct_log_id1)
logger.verbose(' %s' % sct_log_id2)
logger.info('LogID b64 : %s' % sct.log_id_b64)
logger.verbose('Timestamp : %s (%s)' % (sct.timestamp, sct.timestamp_hex))
logger.verbose('Extensions: %d (%s)' %
(sct.extensions_len, sct.extensions_len_hex))
logger.verbose(
'Algorithms: %s/%s (hash/sign)' %
(sct.signature_alg_hash_hex, sct.signature_algorithm_signature))
show_signature_verbose(sct.signature)
prefix = 'Sign. b64 : '
logger.info(prefix + text_with_newlines(sct.signature_b64,
line_length=16 * 3,
newline='\n' + ' ' * len(prefix)))
logger.verbose(
'--') # visual gap between sct infos and verification result
log = verification.log
if log is None:
logger.info('Log not found\n')
else:
logger.info('Log found : %s' % log.description)
logger.verbose('Operator : %s' % log.operated_by['name'])
logger.info('Chrome : %s' % log.scts_accepted_by_chrome)
if verification.verified:
logger.info('Result : Verified OK')
else:
logger.info('Result : Verification Failure')
logger.info('```\n')