def show_signature_verbose(signature):
'''Print out signature as hex string to logger.verbose.
Args:
signature(bytes)
'''
sig_offset = 0
while sig_offset < len(signature):
if len(signature) - sig_offset > 16:
bytes_to_read = 16
else:
bytes_to_read = len(signature) - sig_offset
sig_bytes = struct.unpack_from(flo('!{bytes_to_read}s'), signature,
sig_offset)[0]
if sig_offset == 0:
logger.verbose('Signature : %s' % to_hex(sig_bytes))
else:
logger.verbose(' %s' % to_hex(sig_bytes))
sig_offset = sig_offset + bytes_to_read
def show_logs(logs, heading, order=2):
if len(logs) > 0:
logger.info('#' * order + flo(' {heading}\n'))
s_or_not = 's'
if len(logs) == 1:
s_or_not = ''
# show log size
logger.info('%i log%s\n' % (len(logs), s_or_not))
# list log urls
for log in logs:
if logger.level < logging.INFO:
anchor = log.url.replace('/', '')
logger.verbose(flo('* [{log.url}](#{anchor})'))
else:
logger.info(flo('* {log.url}'))
logger.info('')
for log in logs:
show_log(log)
def show_logs(logs, heading, order=2):
if len(logs) <= 0:
return
logger.info('#' * order + '%s\n' % ' ' + heading if heading else '')
s_or_not = 's'
if len(logs) == 1:
s_or_not = ''
# show log size
logger.info('%i log%s\n' % (len(logs), s_or_not))
# list log urls
for log in logs:
if logger.level < logging.INFO:
anchor = log.url.replace('/', '')
logger.verbose('* [%s](#%s)' % (log.url, anchor))
else:
logger.info('* %s' % log.url)
logger.info('')
for log in logs:
show_log(log)
logger.info('End of list')
def ctloglist(print_json=None):
'''Gather ct-log lists and print the merged log list.
Args:
print_json(boolean): If True, print merged log list as json data.
Else print as markdown.
'''
if not print_json:
today = datetime.date.today()
now = datetime.datetime.now()
logger.info('# Known Certificate Transparency (CT) Logs\n')
logger.verbose('Created with [ctloglist]'
'(https://github.com/theno/ctutlz#ctloglist)\n')
logger.verbose(
'* [all_logs_list.json]('
'https://www.gstatic.com/ct/log_list/v2/all_logs_list.json)'
'\n')
logger.info('Version (Date): %s\n' % today)
logger.verbose('Datetime: %s\n' % now)
logger.info('') # formatting: insert empty line
# all_logs_list.json
all_dict = download_log_list(URL_ALL_LOGS)
orig_all_dict = dict(all_dict)
set_operator_names(all_dict)
all_logs = Logs([all_dict])
if print_json:
json_str = json.dumps(orig_all_dict, indent=4, sort_keys=True)
print(json_str)
else:
show_logs(all_logs, '')
def ctloglist(print_json=None):
'''Gather ct-log lists and print the merged log list.
Args:
print_json(boolean): If True, print merged log list as json data.
Else print as markdown.
'''
if not print_json:
today = datetime.date.today()
now = datetime.datetime.now()
logger.info('# Known Certificate Transparency (CT) Logs\n')
logger.verbose('Created with [ctloglist]'
'(https://github.com/theno/ctutlz#ctloglist)\n')
logger.verbose('Merged log lists:')
logger.verbose("* webpage [known logs]"
'(https://www.certificate-transparency.org/known-logs)')
logger.verbose('* [log_list.json]'
'(https://www.gstatic.com/ct/log_list/log_list.json)')
logger.verbose(
'* [all_logs_list.json]('
'https://www.gstatic.com/ct/log_list/all_logs_list.json)'
'\n')
logger.info(flo('Version (Date): {today}\n'))
logger.verbose(flo('Datetime: {now}\n'))
logger.info('') # formatting: insert empty line
# from webpage
webpage_dict = logs_dict_from_webpage()
all_from_webpage = Logs([
log_dict for log_list in [webpage_dict[key] for key in webpage_dict]
for log_dict in log_list
])
included_from_webpage = Logs(webpage_dict.get('included_in_chrome', []))
try:
webpage_dict.pop('included_in_chrome')
except KeyError:
pass
frozen_from_webpage = Logs(webpage_dict.get('frozen_logs', []))
try:
webpage_dict.pop('frozen_logs')
except KeyError:
pass
pending_from_webpage = Logs(
webpage_dict.get('pending_inclusion_in_chrome', []))
try:
webpage_dict.pop('pending_inclusion_in_chrome')
except KeyError:
pass
disqualified_from_webpage = \
Logs(webpage_dict.get('disqualified_from_chrome', []))
try:
webpage_dict.pop('disqualified_from_chrome')
except KeyError:
pass
rejected_from_webpage = Logs(webpage_dict.get('rejected_by_chrome', []))
try:
webpage_dict.pop('rejected_by_chrome')
except KeyError:
pass
distrusted_from_webpage = Logs(
webpage_dict.get('completely_distrusted_by_chrome', []))
try:
webpage_dict.pop('completely_distrusted_by_chrome')
except KeyError:
pass
other_from_webpage = Logs(webpage_dict.get('other_logs', []))
try:
webpage_dict.pop('other_logs')
except KeyError:
pass
special_from_webpage = Logs(webpage_dict.get('special_purpose_logs', []))
try:
webpage_dict.pop('special_purpose_logs')
except KeyError:
pass
unknown_log_titles = [key for key in webpage_dict.keys()]
if unknown_log_titles:
logger.error(
red(
flo('unknown log titles (i.e. log states): {unknown_log_titles}'
)))
# log_list.json: chrome ct policy compliant logs
log_list_dict = download_log_list(URL_LOG_LIST)
set_operator_names(log_list_dict)
for log_dict in log_list_dict['logs']:
if 'disqualified_at' in log_dict.keys():
log_dict['chrome_state'] = ChromeStates.DISQUALIFIED
elif 'final_sth' in log_dict.keys():
log_dict['chrome_state'] = ChromeStates.FROZEN
else:
log_dict['chrome_state'] = ChromeStates.INCLUDED
log_list_logs = Logs(log_list_dict['logs'])
# all_logs_list.json
all_dict = download_log_list(URL_ALL_LOGS)
set_operator_names(all_dict)
all_logs = Logs(all_dict['logs'])
# merge lists and show the result
log_lists = merge_log_lists(**locals())
if print_json:
data = {
'operators': all_dict['operators'],
'logs': list_from_lists(log_lists)
}
unset_operator_names(data)
json_str = json.dumps(data, indent=4, sort_keys=True)
print(json_str)
else:
for item in log_lists:
show_logs(item['logs'], item['heading'])
def show_log(log, order=3):
logger.verbose('#' * order + flo(' {log.url}\n'))
logdict = log._asdict()
for key, value in logdict.items():
if key == 'id_b64_non_calculated' and value == log.id_b64:
value = None # don't log this value
if key == 'operated_by':
value = ', '.join(value)
# avoid markdown syntax interpretation and improve readablity
key = key.replace('_', ' ')
if value is not None:
logger.verbose(flo('* __{key}__: `{value}`'))
logger.verbose(
flo('* __scts accepted by chrome__: '
'{log.scts_accepted_by_chrome}'))
if log.key is not None:
logger.verbose(flo('* __id b64__: `{log.id_b64}`'))
logger.verbose(flo('* __pubkey__:\n```\n{log.pubkey}\n```'))
logger.verbose('')
def show_verification(verification):
'''
Args:
verification(ctutlz.sct.verification.SctVerificationResult)
'''
sct = verification.sct
sct_log_id1, sct_log_id2 = [to_hex(val)
for val
in struct.unpack("!16s16s", sct.log_id.tdf)]
logger.info('```')
logger.verbose('=' * 59)
logger.verbose(flo('Version : {sct.version_hex}'))
logger.verbose(flo('LogID : {sct_log_id1}'))
logger.verbose(flo(' {sct_log_id2}'))
logger.info(flo('LogID b64 : {sct.log_id_b64}'))
logger.verbose(flo('Timestamp : {sct.timestamp} ({sct.timestamp_hex})'))
logger.verbose(flo(
'Extensions: {sct.extensions_len} ({sct.extensions_len_hex})'))
logger.verbose(flo('Algorithms: {sct.signature_alg_hash_hex}/'
'{sct.signature_algorithm_signature} (hash/sign)'))
show_signature_verbose(sct.signature)
prefix = 'Sign. b64 : '
logger.info(prefix + text_with_newlines(sct.signature_b64, line_length=16*3,
newline='\n' + ' '*len(prefix)))
logger.verbose('--') # visual gap between sct infos and verification result
log = verification.log
if log is None:
logger.info('Log not found\n')
else:
logger.info(flo('Log found : {log.description}'))
logger.verbose('Operator : ' + ', '.join(log.operated_by))
logger.info(flo('Chrome : {log.chrome_state}'))
if verification.verified:
logger.info(flo('Result : Verified OK'))
else:
logger.info(flo('Result : Verification Failure'))
logger.info('```\n')
def show_verification(verification):
'''
Args:
verification(ctutlz.sct.verification.SctVerificationResult)
'''
sct = verification.sct
sct_log_id1, sct_log_id2 = [
to_hex(val) for val in struct.unpack("!16s16s", sct.log_id.tdf)
]
logger.info('```')
logger.verbose('=' * 59)
logger.verbose('Version : %s' % sct.version_hex)
logger.verbose('LogID : %s' % sct_log_id1)
logger.verbose(' %s' % sct_log_id2)
logger.info('LogID b64 : %s' % sct.log_id_b64)
logger.verbose('Timestamp : %s (%s)' % (sct.timestamp, sct.timestamp_hex))
logger.verbose('Extensions: %d (%s)' %
(sct.extensions_len, sct.extensions_len_hex))
logger.verbose(
'Algorithms: %s/%s (hash/sign)' %
(sct.signature_alg_hash_hex, sct.signature_algorithm_signature))
show_signature_verbose(sct.signature)
prefix = 'Sign. b64 : '
logger.info(prefix + text_with_newlines(sct.signature_b64,
line_length=16 * 3,
newline='\n' + ' ' * len(prefix)))
logger.verbose(
'--') # visual gap between sct infos and verification result
log = verification.log
if log is None:
logger.info('Log not found\n')
else:
logger.info('Log found : %s' % log.description)
logger.verbose('Operator : %s' % log.operated_by['name'])
logger.info('Chrome : %s' % log.scts_accepted_by_chrome)
if verification.verified:
logger.info('Result : Verified OK')
else:
logger.info('Result : Verification Failure')
logger.info('```\n')
