def write_report(report): data_path = "%s/%s/%s/report.json" % (report['inspector'], report['year'], report['report_id']) utils.write( utils.json_for(report), "%s/%s" % (utils.data_dir(), data_path) ) return data_path
def run(options): # Input: A path to the root of a domain-scan output directory. scan_path = options.get("scan", ".") results_path = os.path.join(scan_path, "results") # Output: Where to put the post-processed results. output_path = options.get("output", ".") # Take the uswds scan data and cut it down to just the rows for # which the bad banner text is enabled, and only the data needed for it, # in JSON form. uswds_csv = os.path.join(results_path, "uswds.csv") # collect list of dicts to convert into JSON bad_banner = [] uswds_present = [] with open(uswds_csv, newline='') as csvfile: for dict_row in csv.DictReader(csvfile): has_bad_banner = utils.boolean_for(dict_row["USWDS Bad Banner Text"]) is_uswds_present = utils.boolean_for(dict_row["USWDS Present"]) if (has_bad_banner): bad_banner.append({ 'hostname': dict_row["Domain"], 'base_domain': dict_row["Base Domain"], 'scanned_url': dict_row["Scanned URL"] }) if (is_uswds_present): uswds_present.append({ 'hostname': dict_row["Domain"], 'base_domain': dict_row["Base Domain"], 'scanned_url': dict_row["Scanned URL"] }) # Save resulting JSON. bad_banner_data = utils.json_for(bad_banner) bad_banner_output = os.path.join(output_path, "bad_banner.json") utils.write(bad_banner_data, bad_banner_output) uswds_present_data = utils.json_for(uswds_present) uswds_present_output = os.path.join(output_path, "uswds_present.json") utils.write(uswds_present_data, uswds_present_output)
def save_meta_result(result): path = meta_path_for(result['type'], result['agency'], result['year'], result['id']) # for paged metadata, don't overwrite if we've got it already, # we don't keep anything that should change. if os.path.exists(path): logging.debug("[%s][%s] Knew about it, skipping." % (result['id'], result['type'])) else: logging.warn("[%s][%s] Newly discovered, saving metadata." % (result['id'], result['type'])) utils.write(utils.json_for(result), path)
def save_meta_result(result): path = meta_path_for(result['type'], result['agency'], result['year'], result['id']) # for paged metadata, don't overwrite if we've got it already, # we don't keep anything that should change. if os.path.exists(path): logging.debug("[%s][%s] Knew about it, skipping." % (result['id'], result['type'])) else: logging.warn("[%s][%s] Newly discovered, saving metadata." % (result['id'], result['type'])) utils.write(utils.json_for(result), path)
def main(): args = docopt.docopt(__doc__, version='v0.0.1') utils.configure_logging(args['--debug']) out_file = args['--output'] # Read from a .csv, or allow domains on the command line. domains = [] if args['INPUT'][0].endswith(".csv"): domains = utils.load_domains(args['INPUT'][0]) else: domains = args['INPUT'] # If the user wants to sort them, sort them in place. if args['--sorted']: domains.sort() options = { 'user_agent': args['--user-agent'], 'timeout': args['--timeout'], 'preload_cache': args['--preload-cache'], 'cache': args['--cache'] } results = pshtt.inspect_domains(domains, options) # JSON can go to STDOUT, or to a file. if args['--json']: output = utils.json_for(results) if out_file is None: print(output) else: utils.write(output, out_file) logging.warn("Wrote results to %s." % out_file) # Markdwon can go to STDOUT, or to a file elif args['--markdown']: output = sys.stdout if out_file is not None: output = open(out_file, 'w') pshtt.md_for(results, output) if out_file is not None: output.close() # CSV always goes to a file. else: if args['--output'] is None: out_file = 'results.csv' pshtt.csv_for(results, out_file) logging.warn("Wrote results to %s." % out_file)
def scan(domain, options): logging.debug("[%s][pshtt]" % domain) # cache output from pshtt cache_pshtt = utils.cache_path(domain, "pshtt", ext="json") force = options.get("force", False) data = None if (force is False) and (os.path.exists(cache_pshtt)): logging.debug("\tCached.") raw = open(cache_pshtt).read() data = json.loads(raw) if (data.__class__ is dict) and data.get('invalid'): return None else: logging.debug("\t %s %s" % (command, domain)) raw = utils.scan([ command, domain, '--json', '--user-agent', '\"%s\"' % user_agent, '--timeout', str(timeout), '--preload-cache', preload_cache ]) if not raw: utils.write(utils.invalid({}), cache_pshtt) logging.warn("\tBad news scanning, sorry!") return None data = json.loads(raw) utils.write(utils.json_for(data), utils.cache_path(domain, "pshtt")) # pshtt scanner uses JSON arrays, even for single items data = data[0] row = [] for field in headers: value = data[field] # TODO: Fix this upstream if (field != "HSTS Header") and (field != "HSTS Max Age") and ( field != "Redirect To"): if value is None: value = False row.append(value) yield row
def create_preload_list(): preload_json = None if PRELOAD_CACHE and os.path.exists(PRELOAD_CACHE): logging.debug("Using cached Chrome preload list.") preload_json = json.loads(open(PRELOAD_CACHE).read()) else: logging.debug("Fetching Chrome preload list from source...") # Downloads the chromium preloaded domain list and sets it to a global set file_url = 'https://chromium.googlesource.com/chromium/src/net/+/master/http/transport_security_state_static.json?format=TEXT' # TODO: proper try/except around this network request request = requests.get(file_url) raw = request.content # To avoid parsing the contents of the file out of the source tree viewer's # HTML, we download it as a raw file. googlesource.com Base64-encodes the # file to avoid potential content injection issues, so we need to decode it # before using it. https://code.google.com/p/gitiles/issues/detail?id=7 raw = base64.b64decode(raw).decode('utf-8') # The .json file contains '//' comments, which are not actually valid JSON, # and confuse Python's JSON decoder. Begone, foul comments! raw = ''.join( [re.sub(r'^\s*//.*$', '', line) for line in raw.splitlines()]) preload_json = json.loads(raw) if PRELOAD_CACHE: logging.debug("Caching preload list at %s" % PRELOAD_CACHE) utils.write(utils.json_for(preload_json), PRELOAD_CACHE) # For our purposes, we only care about entries that includeSubDomains fully_preloaded = [] for entry in preload_json['entries']: if entry.get('include_subdomains', False) is True: fully_preloaded.append(entry['name']) return fully_preloaded
def do_document(result, page, options): if result.get('pdfLink') is None: print("\tERROR, no pdfLink for document.") return False document = clean_document(result) # can limit to a particular known document ID, for debugging limit_id = options.get('document_id') if limit_id and (limit_id != document['document_id']): print("\tSkipping, not requested.") return False # 1) write JSON to disk at predictable path json_path = path_for(page, document['document_id'], "json") utils.write(utils.json_for(document), json_path) # 2) download pdfLink (unless dry run) if options.get('dry_run') is None: print("\t%s" % document['document_id']) pdf_path = path_for(page, document['document_id'], document['file_type']) result = utils.download( document['url'], pdf_path, { 'binary': (document['file_type'].lower() == 'pdf'), 'cache': not (options.get('force', False)) } ) if result: utils.text_from_pdf(pdf_path) return True
def get_record(agency, year, doc_id, options): # meta_path = meta_path_for("record", agency, year, doc_id) json_path = data_path_for("record", agency, year, doc_id, "json") ## Special: resume mode # # since downloading documents requires re-downloading landing pages, # the only way to do a resumable mass download of docs is to # check for whether the doc has been downloaded yet. this, in turn, # requires checking for the presence of [doc_type].[file_type], and the # file_type needs to be loaded from [doc_type].json. # # So, if --resume is on, before re-downloading anything, check if we # have a parsed .json, and if so, load the file_type and check for # the doc itself. If present, return True and move on. if options.get("resume"): if os.path.exists(json_path): data = json.load(open(json_path)) # it's an unreleased doc, move on anyway if data["unreleased"]: logging.warn("[%s][%s][%s][%s] Unreleased, skipping." % ("record", agency, year, doc_id)) return True doc_path = data_path_for("record", agency, year, doc_id, data["file_type"]) if os.path.exists(doc_path): logging.warn("[%s][%s][%s][%s] Already done, skipping." % ("record", agency, year, doc_id)) return True logging.warn("[%s][%s][%s][%s] Getting record..." % ("record", agency, year, doc_id)) # meta = json.load(open(meta_path)) # download landing page for record url = "https://foiaonline.regulations.gov/foia/action/public/view/record?objectId=%s" % doc_id # save the landing page no matter what, but only use it as a cache # if we're skipping the docs (download links are ephemeral :( ) body = utils.download(url, cache_path_for("record", agency, year, doc_id), {'cache': options.get('skip_doc', False)} ) # assume released unreleased = False doc = BeautifulSoup(body) main = doc.select("#mainForm") if main: main = main[0] else: logging.warn("[%s][%s][%s][%s] Landing page is not available, skipping." % ("record", agency, year, doc_id)) return True # get some other metadata about the record headers = record_headers_from(doc) # now clear the labels so text can be more easily extracted for label in main.select("fieldset .formitem label"): label.extract() links = main.select("fieldset .formitem") # get the actual document download link/ID download_link = links[headers["title"]].select("a") if len(download_link) > 0: download_url = download_link[0]['href'] download_url = "https://foiaonline.regulations.gov" + download_url # no link means it's not released else: unreleased = True title = links[headers["title"]].text.strip() author = links[headers["author"]].text.strip() if author == "N/A": author = None released_date = links[headers["released_on"]].text.strip() if released_date == "N/A": released_on = None else: try: released_at = parse(released_date) released_on = released_at.strftime("%Y-%m-%d") except TypeError: released_on = None request_id = links[headers["request"]].text.strip() file_type = links[headers["file_type"]].text.strip().lower() if file_type == "text": file_type = "txt" # for untyped binary files, just save .mystery and we'll worry later if (not file_type) or (file_type.strip() == ""): file_type = "mystery" # TODO: handle unexpected file types more gracefully # right now, it accepts any extension and dl's them. # it should choke on unexpected types, and email admin. # this should correspond with it being unreleased if file_type.startswith("contact"): unreleased = True exemptions = links[headers["exemptions"]].text.strip() if exemptions == "N/A": exemptions = None retention = links[headers["retention"]].text.strip() if retention == "N/A": retention = None file_size = links[headers["file_size"]].text.strip() record = { "type": "record", "landing_id": doc_id, "landing_url": url, "agency": agency, "year": year, "request_id": request_id, "title": title, "released_on": released_on, "released_original": released_date, "author": author, "exemptions": exemptions, "retention": retention } if unreleased: record["unreleased"] = True else: record["unreleased"] = False record["file_size"] = file_size record["file_type"] = file_type # ephemeral, used below to download, and kept for record-keeping record["download_url"] = download_url # 1) write JSON to disk at predictable path utils.write(utils.json_for(record), json_path) # 2) download the associated record doc (unless dry run) if unreleased: logging.warn("\tUnreleased doc, moving on.") elif options.get('skip_doc') is None: logging.warn("\tDownloading...") text_types = ('txt') binary_types = ('pdf', 'doc', 'docx', 'xls', 'xlsx', 'mystery', '') doc_path = data_path_for("record", agency, year, doc_id, record['file_type']) result = utils.download( download_url, doc_path, { 'binary': True, 'cache': not (options.get('force', False)) } ) # PDF extraction is easy enough if result and (record['file_type'] == 'pdf'): logging.warn("\tExtracting text from PDF...") utils.text_from_pdf(doc_path) return True
def paginated_mode(suffix, options, uid, api_key): # Cache hostnames in a dict for de-duping. hostnames_map = {} certificate_api = certificates.CensysCertificates(uid, api_key) if 'query' in options and options['query']: query = options['query'] else: query = "parsed.subject.common_name:\"%s\" or parsed.extensions.subject_alt_name.dns_names:\"%s\"" % ( suffix, suffix) logging.debug("Censys query:\n%s\n" % query) # time to sleep between requests (defaults to 5s) delay = int(options.get("delay", 5)) # Censys page size, fixed page_size = 100 # Start page defaults to 1. start_page = int(options.get("start", 1)) # End page defaults to whatever the API says is the last one. end_page = options.get("end", None) if end_page is None: end_page = get_end_page(query, certificate_api) if end_page is None: logging.warn("Error looking up number of pages.") exit(1) else: end_page = int(end_page) max_records = ((end_page - start_page) + 1) * page_size fields = [ "parsed.subject.common_name", "parsed.extensions.subject_alt_name.dns_names" ] current_page = start_page logging.warn("Fetching up to %i records, starting at page %i." % (max_records, start_page)) last_cached = False force = options.get("force", False) while current_page <= end_page: if (not last_cached) and (current_page > start_page): logging.debug("(Waiting %is before fetching page %i.)" % (delay, current_page)) last_cached = False time.sleep(delay) logging.debug("Fetching page %i." % current_page) cache_page = utils.cache_path(str(current_page), "censys") if (force is False) and (os.path.exists(cache_page)): logging.warn("\t[%i] Cached page." % current_page) last_cached = True certs_raw = open(cache_page).read() certs = json.loads(certs_raw) if (certs.__class__ is dict) and certs.get('invalid'): continue else: try: certs = list( certificate_api.search(query, fields=fields, page=current_page, max_records=page_size)) utils.write(utils.json_for(certs), cache_page) except censys.base.CensysException: logging.warn(utils.format_last_exception()) logging.warn("Censys error, skipping page %i." % current_page) utils.write(utils.invalid({}), cache_page) continue except: logging.warn(utils.format_last_exception()) logging.warn("Unexpected error, skipping page %i." % current_page) utils.write(utils.invalid({}), cache_page) exit(1) for cert in certs: # Common name + SANs names = cert.get('parsed.subject.common_name', []) + cert.get( 'parsed.extensions.subject_alt_name.dns_names', []) logging.debug(names) for name in names: hostnames_map[sanitize_name(name)] = None current_page += 1 logging.debug("Done fetching from API.") return hostnames_map
def get_record(agency, year, doc_id, options): # meta_path = meta_path_for("record", agency, year, doc_id) json_path = data_path_for("record", agency, year, doc_id, "json") ## Special: resume mode # # since downloading documents requires re-downloading landing pages, # the only way to do a resumable mass download of docs is to # check for whether the doc has been downloaded yet. this, in turn, # requires checking for the presence of [doc_type].[file_type], and the # file_type needs to be loaded from [doc_type].json. # # So, if --resume is on, before re-downloading anything, check if we # have a parsed .json, and if so, load the file_type and check for # the doc itself. If present, return True and move on. if options.get("resume"): if os.path.exists(json_path): data = json.load(open(json_path)) # it's an unreleased doc, move on anyway if data["unreleased"]: logging.warn("[%s][%s][%s][%s] Unreleased, skipping." % ("record", agency, year, doc_id)) return True doc_path = data_path_for("record", agency, year, doc_id, data["file_type"]) if os.path.exists(doc_path): logging.warn("[%s][%s][%s][%s] Already done, skipping." % ("record", agency, year, doc_id)) return True logging.warn("[%s][%s][%s][%s] Getting record..." % ("record", agency, year, doc_id)) # meta = json.load(open(meta_path)) # download landing page for record url = "https://foiaonline.regulations.gov/foia/action/public/view/record?objectId=%s" % doc_id # save the landing page no matter what, but only use it as a cache # if we're skipping the docs (download links are ephemeral :( ) body = utils.download(url, cache_path_for("record", agency, year, doc_id), {'cache': options.get('skip_doc', False)}) # assume released unreleased = False doc = BeautifulSoup(body) main = doc.select("#mainForm") if main: main = main[0] else: logging.warn( "[%s][%s][%s][%s] Landing page is not available, skipping." % ("record", agency, year, doc_id)) return True # get some other metadata about the record headers = record_headers_from(doc) # now clear the labels so text can be more easily extracted for label in main.select("fieldset .formitem label"): label.extract() links = main.select("fieldset .formitem") # get the actual document download link/ID download_link = links[headers["title"]].select("a") if len(download_link) > 0: download_url = download_link[0]['href'] download_url = "https://foiaonline.regulations.gov" + download_url # no link means it's not released else: unreleased = True title = links[headers["title"]].text.strip() author = links[headers["author"]].text.strip() if author == "N/A": author = None released_date = links[headers["released_on"]].text.strip() if released_date == "N/A": released_on = None else: try: released_at = parse(released_date) released_on = released_at.strftime("%Y-%m-%d") except TypeError: released_on = None request_id = links[headers["request"]].text.strip() file_type = links[headers["file_type"]].text.strip().lower() if file_type == "text": file_type = "txt" # for untyped binary files, just save .mystery and we'll worry later if (not file_type) or (file_type.strip() == ""): file_type = "mystery" # TODO: handle unexpected file types more gracefully # right now, it accepts any extension and dl's them. # it should choke on unexpected types, and email admin. # this should correspond with it being unreleased if file_type.startswith("contact"): unreleased = True exemptions = links[headers["exemptions"]].text.strip() if exemptions == "N/A": exemptions = None retention = links[headers["retention"]].text.strip() if retention == "N/A": retention = None file_size = links[headers["file_size"]].text.strip() record = { "type": "record", "landing_id": doc_id, "landing_url": url, "agency": agency, "year": year, "request_id": request_id, "title": title, "released_on": released_on, "released_original": released_date, "author": author, "exemptions": exemptions, "retention": retention } if unreleased: record["unreleased"] = True else: record["unreleased"] = False record["file_size"] = file_size record["file_type"] = file_type # ephemeral, used below to download, and kept for record-keeping record["download_url"] = download_url # 1) write JSON to disk at predictable path utils.write(utils.json_for(record), json_path) # 2) download the associated record doc (unless dry run) if unreleased: logging.warn("\tUnreleased doc, moving on.") elif options.get('skip_doc') is None: logging.warn("\tDownloading...") text_types = ('txt') binary_types = ('pdf', 'doc', 'docx', 'xls', 'xlsx', 'mystery', '') doc_path = data_path_for("record", agency, year, doc_id, record['file_type']) result = utils.download(download_url, doc_path, { 'binary': True, 'cache': not (options.get('force', False)) }) # PDF extraction is easy enough if result and (record['file_type'] == 'pdf'): logging.warn("\tExtracting text from PDF...") utils.text_from_pdf(doc_path) return True