def dynamic_main(self, queue_dict):
"""
Main entry point for process to call.
core_serialization.SubDomain Attributes:
name: long name of method
module_name: name of the module that performed collection
source: source of the subdomain or resource of collection
module_version: version from meta
source: source of the collection
time: time the result obj was built
subdomain: subdomain to use
valid: is domain valid
:return:
"""
core_args = self.json_entry['args']
task_output_queue = queue_dict['task_output_queue']
cs = core_scrub.Scrub()
rd = []
data = crtshAPI().search(str(core_args.DOMAIN))
for d in data:
cs.subdomain = d['issuer']
# check if domain name is valid
valid = cs.validate_domain()
# build the SubDomain Object to pass
sub_obj = core_serialization.SubDomain(self.info["Name"],
self.info["Module"],
"https://crt.sh",
self.info["Version"],
time.time(), d['issuer'],
valid)
# populate queue with return data objects
task_output_queue.put(sub_obj)
def dynamic_main(self, queue_dict):
"""
Main entry point for process to call.
core_serialization.SubDomain Attributes:
name: long name of method
module_name: name of the module that performed collection
source: source of the subdomain or resource of collection
module_version: version from meta
source: source of the collection
time: time the result obj was built
subdomain: subdomain to use
valid: is domain valid
:return: NONE
"""
foundsubdomains = []
core_args = self.json_entry['args']
task_output_queue = queue_dict['task_output_queue']
cs = core_scrub.Scrub()
start_count = int(self.json_entry['bing_search']['start_count'])
end_count = int(self.json_entry['bing_search']['end_count'])
while start_count <= end_count:
domain = "http://www.bing.com/search?q=site%3A" + \
str(core_args.DOMAIN) + "&first=" + str(start_count)
data, status = self.request_content(domain)
soup = BeautifulSoup(data, 'html.parser')
for i in soup.find_all('a', href=True):
possiblesubdomain = i['href']
if "." + str(core_args.DOMAIN) in possiblesubdomain:
parsed = urlparse(possiblesubdomain)
if parsed.netloc not in foundsubdomains:
foundsubdomains.append(str(parsed.netloc))
if parsed.hostname not in foundsubdomains:
foundsubdomains.append(str(parsed.hostname))
for sub in foundsubdomains:
cs.subdomain = sub
# check if domain name is valid
valid = cs.validate_domain()
# build the SubDomain Object to pass
sub_obj = core_serialization.SubDomain(
self.info["Name"],
self.info["Module"],
self.options['url'],
domain,
time.time(),
sub,
valid
)
task_output_queue.put(sub_obj)
# results inc at rate of 10 per page
start_count += 10
time.sleep(0.5)
def _dns_result_callback(self, name, future):
"""Handles the result passed by the _dns_lookup function."""
# Record processed we can now release the lock
self.sem.release()
# Handle known exceptions, barf on other ones
if future.exception() is not None:
try:
err_num = future.exception().args[0]
err_text = future.exception().args[1]
except IndexError:
self.logger("Couldn't parse exception: {}".format(
future.exception()), 'err')
if (err_num == 4): # This is domain name not found, ignore it.
pass
elif (err_num == 12): # Timeout from DNS server
self.logger("Timeout for {}".format(name), 'warn', 2)
elif (err_num == 1): # Server answered with no data
pass
else:
self.logger('{} generated an unexpected exception: {}'.format(
name, future.exception()), 'err')
self.errors.append({'hostname': name, 'error': err_text})
# Output result
else:
self.cs.subdomain = name
# check if domain name is valid
valid = self.cs.validate_domain()
# build the SubDomain Object to pass
sub_obj = core_serialization.SubDomain(
self.info["Name"],
self.info["Module"],
"",
self.info["Version"],
time.time(),
name,
valid
)
if valid:
if not self._check_wildcard_domain(name):
self.runtime_queue.append(name)
self.task_output_queue.put(sub_obj)
ip = ', '.join([ip.host for ip in future.result()])
self.fqdn.append((name, ip))
# self.logger("{:<30}\t{}".format(name, ip), 'pos')
# self.logger(future.result(), 'dbg', 3)
self.tasks.remove(future)
# TODO: enable verbose
if not self.silent:
self.pbar.update()
def dynamic_main(self, queue_dict):
"""
Main entry point for process to call.
core_serialization.SubDomain Attributes:
name: long name of method
module_name: name of the module that performed collection
source: source of the subdomain or resource of collection
module_version: version from meta
source: source of the collection
time: time the result obj was built
subdomain: subdomain to use
valid: is domain valid
:return: NONE
"""
core_args = self.json_entry['args']
task_output_queue = queue_dict['task_output_queue']
cs = core_scrub.Scrub()
domain = self.options['url'] % (str(core_args.DOMAIN))
data, status = self.request_json(domain)
data = json.loads(data)
if status:
for d in data['data']:
cs.subdomain = d['id']
# check if domain name is valid
valid = cs.validate_domain()
# build the SubDomain Object to pass
sub_obj = core_serialization.SubDomain(
self.info["Name"],
self.info["Module"],
self.options['url'],
self.info["Version"],
time.time(),
d['id'],
valid
)
# populate queue with return data object
task_output_queue.put(sub_obj)
