def __init__(self, args):
urls = None
if args.input_file is not None:
args.quiet = True
with open(args.input_file, 'r') as input_file:
urls = []
for url in input_file.readlines():
url = url.strip()
urls.append(url if '://' in url else 'http://' + url)
else:
args.url = args.url.lower()
if '://' not in args.url:
args.url = 'http://' + args.url
text_printer = Printer(args.verbosity)
cache = Cache()
cache.set_location(args.cache_dir)
cache.printer = text_printer
self.options = {
'url': args.url,
'urls': urls,
'quiet': args.quiet,
'prefix': '',
'user_agent': args.user_agent,
'proxy': args.proxy,
'verbosity': args.verbosity,
'insecure': args.insecure,
'threads': 10,
'batch_size': 20,
'run_all': args.run_all,
'match_all': args.match_all,
'stop_after': args.stop_after,
'no_cache_load': args.no_cache_load,
'no_cache_save': args.no_cache_save,
'write_file': args.output_file,
'subdomains': args.subdomains
}
self.data = {
'cache': cache,
'results': Results(self.options),
'fingerprints': Fingerprints(),
'matcher': Match(),
'printer': text_printer,
'detected_cms': set(),
'error_pages': set(),
'requested': queue.Queue()
}
if self.options['write_file'] is not None:
self.json_outputter = OutputJSON(self.options, self.data)
self.data['printer'].print_logo()
self.results = None
def run_fingerprint(place_id):
client = MongoClient(dbhost, dbport)
db = client.prospectinator
places = db.places
query_url = place_url % (place_id, gmaps_apikey)
# retrieve a record by the place id. If it is not present, create a new one.
place_doc = places.find_one({'result.place_id': place_id}) or json.loads(
requests.get(query_url).text)
if 'website' in place_doc['result']:
if 'fingerprint' not in place_doc and site_is_up(
place_doc['result']['website']):
print("scanning url %-40s for place ID %s" %
(place_doc['result']['website'], place_id))
w = wig(url=place_doc['result']['website'])
w.run()
results = OutputJSON(w.options, w.data)
results.add_results()
place_doc['fingerprint'] = results.json_data[0]
places.save(place_doc)
return
