def crtshQuery(domain):
found = list()
if os.path.isfile(domain + ".sub.crtsh") == False or os.path.getsize(
domain + ".sub.crtsh") == 0:
try:
rawlist = crtshAPI().search(domain)[0]
except:
rawlist = list()
for item in rawlist:
item = json.loads(item)
for k, v in item.items():
if k == "name_value":
if '@' in v:
continue
if '*' in v:
continue
if checkFqdn(v) == False:
continue
if '\n' in v:
for tok in v.split('\n'):
found.append(tok)
else:
found.append(v)
found = list(set(found))
saveFile(domain + ".sub.crtsh", found)
else:
temp = readFile(domain + ".sub.crtsh")
for item in temp:
if len(item) > 2:
found.append(item.rstrip("\n"))
return found
def dynamic_main(self, queue_dict):
"""
Main entry point for process to call.
core_serialization.SubDomain Attributes:
name: long name of method
module_name: name of the module that performed collection
source: source of the subdomain or resource of collection
module_version: version from meta
source: source of the collection
time: time the result obj was built
subdomain: subdomain to use
valid: is domain valid
:return:
"""
core_args = self.json_entry['args']
task_output_queue = queue_dict['task_output_queue']
cs = core_scrub.Scrub()
rd = []
data = crtshAPI().search(str(core_args.DOMAIN))
for d in data:
cs.subdomain = d['issuer']
# check if domain name is valid
valid = cs.validate_domain()
# build the SubDomain Object to pass
sub_obj = core_serialization.SubDomain(self.info["Name"],
self.info["Module"],
"https://crt.sh",
self.info["Version"],
time.time(), d['issuer'],
valid)
# populate queue with return data objects
task_output_queue.put(sub_obj)
def query_crtsh(args):
""" query crt.sh for the given domain
Params:
args - arguments provided at runtime
Returns:
certs - a list of certificates returned from the crt.sh query
"""
domain = args.domain
result = crtshAPI().search(domain)
#print(result[0])
now = datetime.now()
recent_certs = []
print(f'[+] Querying crt.sh for: {domain}')
for item in result[0]:
result_timestamp = parse(item["entry_timestamp"])
if now - timedelta(days=args.timeframe) <= result_timestamp <= now:
#print(f'[+] Found one! {result[item]["entry_timestamp"]}')
# check for any duplicate serial numbers
if not any(s["serial_number"] == item["serial_number"]
for s in recent_certs):
if len(item["serial_number"]) >= 32:
recent_certs.append(item)
print(
f'[+] Found {len(recent_certs)} certificates in last {args.timeframe} days for query: {domain}'
)
return recent_certs
def query(domain):
results = []
full_result = crtshAPI().search(domain,expired=False)
if len(full_result) == 0:
results.append(
Result(
message='No results found.'
)
)
else:
for res in full_result:
results.append(
Result(
message="Found certificate info.",
json_result={
'common_name': res['common_name'],
'valid_from': res['not_before']
}
)
)
return results
def ca_issuer(domain):
"""Returns the name of the Certificate authority (CA) issuer of the URL
Issue with this feature: It takes so long to execute large amount of data.
"""
ca_issuer = crtshAPI().search(domain)
if not ca_issuer:
return None
else:
return ca_issuer[0]['issuer_name']
def exploit(workspace):
n_tab = 0
global output
now = datetime.now()
dt_string = now.strftime("%d_%m_%Y_%H_%M_%S")
file = "{}_misc_crtsh".format(dt_string)
dfile = "{}_dns_file".format(dt_string)
filename = "./workspaces/{}/{}".format(workspace, file)
domain = variables['DOMAIN']['value']
json_data = crtshAPI().search(domain)
for certs in json_data:
nv = (certs['name_value']).split("\n")
certs['name_value'] = nv
yn = variables['GENERATE-LIST']['value']
if not yn == "":
if not yn.lower() == 'true' and not yn.lower() == 'false':
print(
colored("[*] 'GENERATE-LIST' should be either True or False.")
)
elif yn.lower() == 'true':
dns_names = []
for cert in json_data:
for dns in cert['name_value']:
dns_names.append(dns)
uniquedns = set(dns_names)
dns_list = list(uniquedns)
dfilename = "./workspaces/{}/{}".format(workspace, dfile)
dnsfile = open(dfilename, "w")
for dns in dns_list:
dnsfile.write(dns+"\n")
dnsfile.close()
print(colored("[*] DNS List saved on '{}'.".format(dfilename), "green"))
with open(filename, 'w') as outfile:
json.dump(json_data, outfile, indent=4, default=str)
print(colored("[*] Content dumped on file '{}'.".format(filename), "green"))
output += colored("---------------------------------\n", "yellow", attrs=['bold'])
for data in json_data:
output += colored("{}: {}\n".format("Common Name", data["common_name"]), "yellow", attrs=['bold'])
list_dictionary(data, n_tab)
output += colored("---------------------------------\n", "yellow", attrs=['bold'])
pipepager(output, "less -R")
output = ""
def crtsh_lookup(domain):
domains = []
print("\n[*] Performing crtsh lookup\n")
result = crtshAPI().search(domain)
for record in result:
for r in record:
# Account for newline characters in some name records
if '\n' in r['name_value']:
domain_names = r['name_value'].split("\n")
for d in domain_names:
domains.append(d.strip())
else:
domains.append(r['name_value'].strip())
domains = list(dict.fromkeys(domains))
for d in domains:
print("[+] %s" % d)
return result, domains
def subdomain_search(value):
"""
helper function for interacting with crt.sh
"""
return json.dumps(crtsh.crtshAPI().search(str(value)))
columns[k].append(v) # append the value into the appropriate list
# based on column name k
fuzzed_domain = columns['domain-name']
print fuzzed_domain
print "==" * 50
print "**" * 50
while True:
for item in fuzzed_domain:
print item
a = item
# a = raw_input("ENTER THE NAME FOR THE DOMAIN TO BE SEARCHED : ")
print colored('SEARCHING CERTIFICATES FOR ', 'red'), item
jason_data = json.dumps(crtshAPI().search(a))
jason_data = json.loads(jason_data)
# print jason_data
# print len(jason_data)
epoch_now = int(datetime.datetime.now().strftime('%s'))
for item in jason_data[0]:
# print item
headers = {
'Content-type': 'application/json',
'Accept': 'text/plain'
}
hash_object = hashlib.sha1(json.dumps(item)).hexdigest()
creation_date = (datetime.datetime.strptime(
def subdomain_search(value):
return json.dumps(crtshAPI().search(str(value)))
from crtsh import crtshAPI
import sys
import json
if len(sys.argv) > 1:
print(json.dumps(crtshAPI().search(sys.argv[1]), indent=2))
else:
pass
##monitoring for the certificate transpirincy by www.crt.sh
#we are using the unoficial crtsh api and the elastic search
#elastic search databse stores the data and when the program runs after every 12 hrs
#it checks if there is new certificate has been generated it will prompt new certificate found or else it will sleep for next 12 hrs.
from crtsh import crtshAPI
import json
import requests
from pprint import pprint
import hashlib
import time
data_feed = json.dumps(crtshAPI().search('cyware.com'))
while True:
cyware = json.loads(data_feed)
es_url = "http://127.0.0.1:9200/cert_new/cywar/"
dummp_post = requests.get(es_url)
for item in cyware[0]:
headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
hash_object = hashlib.sha1(json.dumps(item)).hexdigest()
min_cert_id = item['min_cert_id']
#print json.dumps(item)
dummp_post = requests.post(es_url + str(item['min_cert_id']),
headers=headers,
data=json.dumps(item))
view = json.loads(dummp_post.text)
if view["_version"] <= 1:
print "=" * 40
print "NEW CERTIFICATE FOUND"
print "=" * 40
def api(d):
results = json.dumps(crtshAPI().search(d))
return results
url = "https://api.macvendors.com/" + param
erg = shell("curl -sK -X GET %s" % url)
for i in erg.output():
print "\033[0;249;0m" + i + "\033[0;249;0m"
elif choice == "4":
param = raw_input("Enter a domain or ip: ")
erg = shell("ping -c 4 %s" % param)
for i in erg.output():
print "\033[0;249;0m" + i + "\033[0;249;0m"
elif choice == "5":
param = raw_input("Enter a domain name: ")
print "\n -- This check can last some minutes! -- \n"
try:
print "\033[0;249;0m" + json.dumps(crtshAPI().search(param), indent=2) + "\033[0;249;0m"
except:
print " The service crt.sh is not available"
print "Certificates download is ongoing!"
shell("./crt.sh %s" % param)
print "\033[0;249;0mThe certificate are stored as *.html files under output/crt!\033[0;249;0m"
elif choice == "6":
print "\033[0;249;0mYou can enter a name (<john+doe>) or an emailaddress\033[0;249;0m"
param = raw_input("Enter your searchparameter: ")
reverse_whois.do_reverse_whois(param)
elif choice == "7":
param = raw_input("Enter a domain or an IP: ")
reverse_ip.do_call(param)
from crtsh import crtshAPI
from pprint import pprint
from dns import resolver
all_hosts = []
certs = crtshAPI().search('DOMAIN_TO_SEARCH')
for cert in certs[0]:
all_hosts.append(cert['name_value'])
hosts = set(all_hosts)
for host in hosts:
try:
a = resolver.query(host)
print("{0}: {1}".format(host, a.canonical_name))
except:
print("{0} failed".format(host))