def update_ports(self, command):
utils.print_good('Cleaning for {0}:{1}'.format(command.get('banner'),
command.get('pre_run')))
csv_data = utils.just_read(command.get('requirement'), get_list=True)
if not csv_data:
utils.print_bad('Requirement not found: {0}'.format(
command.get('requirement')))
return False
result = {}
for line in csv_data[1:]:
host = line.split(',')[0]
port = line.split(',')[3]
if result.get(host, None):
result[host] += "," + str(port).strip(',')
else:
result[host] = port
# store it as format can submit to summaries
final_result = []
for host, ports in result.items():
item = "ip_address|{0};;ports|{1}".format(host, ports)
final_result.append(item)
utils.just_write(command.get('cleaned_output'),
"\n".join(final_result))
summary.push_with_file(self.options, command.get('cleaned_output'))
def clean_massdns(self, command):
utils.print_good('Cleaning for {0}:{1}'.format(
command.get('banner'), command.get('post_run')))
output = utils.just_read(command.get('output_path'), get_list=True)
if not output:
utils.print_bad('Output not found: {0}'.format(
command.get('output_path')))
return False
only_A_record, summaries, resolved = [], [], []
for line in output:
if '. A ' in line:
only_A_record.append(line.split('. A ')[1].strip())
resolved.append(line.split('. A ')[0])
summary = "domain|{0};;ip_address|{1}".format(
line.split('. A ')[0],
line.split('. A ')[1])
summaries.append(summary)
elif '. CNAME ' in line:
resolved.append(line.split('. CNAME ')[0])
cleaned_output = utils.just_write(command.get('cleaned_output'),
"\n".join(only_A_record))
resolved_path = utils.replace_argument(
self.options, '$WORKSPACE/probing/resolved-$OUTPUT.txt')
resolved_output = utils.just_write(resolved_path, "\n".join(resolved))
if cleaned_output:
utils.check_output(command.get('cleaned_output'))
if resolved_output:
utils.check_output(resolved_path)
self.update_summaries(summaries)
def update_ports(self, command):
utils.print_good('Cleaning for {0}:{1}'.format(
command.get('banner'), command.get('post_run')))
csv_data = utils.just_read(command.get('output_path'), get_list=True)
if not csv_data:
utils.print_bad('Output not found: {0}'.format(
command.get('output_path')))
return False
result = {}
for line in csv_data[1:]:
host = line.split(',')[0]
port = line.split(',')[3]
if result.get('host', None):
result[host] += "," + str(port).strip(',')
else:
result[host] = port
final_result = []
for host, ports in result.items():
item = "ip_address|{0};;ports|{1}".format(host, ports)
final_result.append(item)
utils.just_write(command.get('cleaned_output'),
"\n".join(final_result))
def get_scheme(self, command):
utils.print_good('Cleaning for {0}:{1}'.format(
command.get('banner'), command.get('post_run')))
csv_data = utils.just_read(command.get('requirement'), get_list=True)
if not csv_data:
utils.print_bad('Output not found: {0}'.format(
command.get('output_path')))
return False
summaries, result = [], []
for line in csv_data[1:]:
# print(line)
if ',' not in line or len(line.split(',')) < 3:
continue
_results = line.split(',')
host = _results[0].strip('"')
port = _results[2].strip('"')
service = _results[4].strip('"') + "/" + _results[5].strip('"')
result.append("http://" + host + ":" + port)
result.append("https://" + host + ":" + port)
sum_line = f"domain|{host};;ip_address|{host};;ports|{port};;technologies|{service}"
summaries.append(sum_line)
# print(sum_line)
scheme_path = utils.replace_argument(
self.options, '$WORKSPACE/vulnscan/scheme-$OUTPUT.txt')
utils.just_write(scheme_path, "\n".join(result))
# update summaries table
formatted_summary = utils.replace_argument(
self.options, '$WORKSPACE/vulnscan/formatted-summary-$OUTPUT.txt')
utils.just_write(formatted_summary, "\n".join(summaries))
summary.push_with_file(self.options, formatted_summary)
def clean_gowitness(self, command):
utils.print_good('Cleaning for {0}:{1}'.format(
command.get('banner'), command.get('post_run')))
real_cmd = utils.resolve_command(
self.options, {
"banner":
"gowitness gen report",
"cmd":
"$GO_PATH/gowitness report generate -n $WORKSPACE/portscan/screenshot/$OUTPUT-raw-gowitness.html --destination $WORKSPACE/portscan/screenshot/raw-gowitness/ --db $WORKSPACE/portscan/screenshot/gowitness.db",
"output_path":
"$WORKSPACE/portscan/screenshot/$OUTPUT-raw-gowitness.html",
})
execute.send_cmd(self.options, real_cmd)
raw_html = utils.just_read(real_cmd.get('output_path'))
if not raw_html:
utils.print_bad('Requirement not found: {0}'.format(
real_cmd.get('output_path')))
return False
local_path = utils.replace_argument(self.options,
'$WORKSPACE/portscan/')
real_html = raw_html.replace(local_path, '')
utils.just_write(command.get('cleaned_output'), real_html)
utils.check_output(command.get('cleaned_output'))
def clean_gowitness(self, command):
utils.print_good('Cleaning for {0}:{1}'.format(
command.get('banner'), command.get('post_run')))
raw_html = utils.just_read(command.get('output_path'))
if not raw_html:
utils.print_bad('Output not found: {0}'.format(
command.get('output_path')))
return False
local_path = utils.replace_argument(
self.options, '$WORKSPACE/screenshot/')
real_html = raw_html.replace(local_path, '')
utils.just_write(command.get('cleaned_output'), real_html)
def get_domain(self, command):
utils.print_good('Cleaning for {0}:{1}'.format(
command.get('banner'), command.get('post_run')))
output = utils.just_read(command.get('output_path'))
if not output:
utils.print_bad('Output not found: {0}'.format(
command.get('output_path')))
return False
no_scheme = output.replace('https://', '').replace('http://', '')
utils.just_write(command.get('cleaned_output'), no_scheme)
if command.get('cleaned_output'):
utils.check_output(command.get('cleaned_output'))
def gen_summary(self, command):
summary_path = utils.replace_argument(
self.options, '$WORKSPACE/vulnscan/summary-$OUTPUT.csv')
sum_head = '"IP","FQDN","PORT","PROTOCOL","SERVICE","VERSION"'
details_folder = utils.replace_argument(
self.options, '$WORKSPACE/vulnscan/details/')
details = utils.list_files(details_folder, '.csv')
summary_data = [sum_head]
for detail in details:
really_detail = utils.just_read(detail, get_list=True)
if really_detail:
summary_data.append("\n".join(really_detail[1:]))
utils.just_write(summary_path, "\n".join(summary_data))
def gen_summary(self, command):
summary_path = utils.replace_argument(
self.options, '$WORKSPACE/vulnscan/summary-$OUTPUT.csv')
sum_head = 'IP,Host,OS,Proto,Port,Service,Product,Service FP,NSE Script ID,NSE Script Output,Notes'
details_folder = utils.replace_argument(
self.options, '$WORKSPACE/vulnscan/details/')
details = utils.list_files(details_folder, '.csv')
summary_data = [sum_head]
for detail in details:
really_detail = utils.just_read(detail, get_list=True)
if really_detail:
summary_data.append("\n".join(really_detail[1:]))
utils.just_write(summary_path, "\n".join(summary_data))
def update_tech(self, command):
utils.print_good('Cleaning for {0}:{1}'.format(
command.get('banner'), command.get('post_run')))
output = utils.just_read(command.get('output_path'), get_list=True)
if not output:
utils.print_bad('Output not found: {0}'.format(
command.get('output_path')))
return False
tech_summaries = []
for line in output:
try:
jsonl = utils.get_json(line)
if jsonl.get('matches'):
domain = utils.get_domain(jsonl.get('hostname'))
techs = [x.get('app_name') for x in jsonl.get('matches')]
item = "domain|{0};;technologies|{1}".format(
domain, ",".join(techs))
tech_summaries.append(item)
except:
pass
cleaned_output = utils.just_write(command.get('cleaned_output'),
"\n".join(tech_summaries))
if cleaned_output:
utils.check_output(command.get('cleaned_output'))
# update technologies to db
summary.push_with_file(self.options, command.get('cleaned_output'))
def join_ip(self, command):
cleaned_output = utils.just_read(command.get('cleaned_output'), get_list=True)
raw_input = utils.just_read(
command.get('requirement'), get_list=True)
result = []
for line in raw_input:
if utils.valid_ip(line.strip()):
result.append(line)
if cleaned_output:
result = list(set(result + cleaned_output))
else:
result = list(set(result))
if result:
utils.just_write(command.get('cleaned_output'), "\n".join(result))
summaries = []
for item in result:
summary = f"domain|{item};;ip_address|{item}"
summaries.append(summary)
self.update_summaries(summaries)
def get_scheme(self, command):
utils.print_good('Preparing for {0}:{1}'.format(
command.get('banner'), command.get('pre_run')))
scheme_path = utils.replace_argument(
self.options, '$WORKSPACE/portscan/scheme-$OUTPUT.txt')
csv_data = utils.just_read(command.get('requirement'), get_list=True)
if not csv_data:
utils.print_bad('Requirement not found: {0}'.format(
command.get('requirement')))
return False
result = []
for line in csv_data[1:]:
host = line.split(',')[0]
port = line.split(',')[3]
result.append("http://" + host + ":" + port)
result.append("https://" + host + ":" + port)
utils.just_write(scheme_path, "\n".join(result))
utils.check_output(scheme_path)
def clean_gobuster(self, command):
utils.print_good('Cleaning for {0}:{1}'.format(
command.get('banner'), command.get('post_run')))
output = utils.just_read(command.get('output_path'))
if not output:
utils.print_bad('Output not found: {0}'.format(
command.get('output_path')))
return False
cleaned_output = utils.just_write(command.get('cleaned_output'),
output.replace('Found: ', ''))
if cleaned_output:
utils.check_output(command.get('cleaned_output'))
def clean_massdns(self, command):
utils.print_good('Cleaning for {0}:{1}'.format(command.get('banner'), command.get('post_run')))
output = utils.just_read(command.get('output_path'), get_list=True)
if not output:
utils.print_bad('Output not found: {0}'.format(
command.get('output_path')))
return False
# only get A record
only_A_record = "\n".join([x.split('. A ')[0] for x in output if '. A ' in x])
cleaned_output = utils.just_write(command.get(
'cleaned_output'), only_A_record)
if cleaned_output:
utils.check_output(command.get('cleaned_output'))
def clean_vhosts_gobuster(self, command):
final_output = utils.replace_argument(
self.options, "$WORKSPACE/vhosts/vhosts-$OUTPUT.txt")
raw_outputs = utils.replace_argument(
self.options, "$WORKSPACE/vhosts/raw-summary-$OUTPUT.txt")
content = utils.just_read(raw_outputs)
if not content:
return
result = utils.regex_strip("\\s\\(Status.*", content)
cleaned_output = utils.just_write(
final_output, result.replace('Found: ', ''))
if cleaned_output:
utils.check_output(command.get(
'cleaned_output'))
def clean_multi_gobuster(self, command):
final_output = utils.replace_argument(
self.options, "$WORKSPACE/vhosts/vhost-$OUTPUT.txt")
# simple hack here
raw_outputs = utils.list_files(final_output + '/../raw/',
'-gobuster.txt')
utils.join_files(raw_outputs, final_output)
# content = final_output
content = utils.just_read(final_output)
if content:
result = utils.regex_strip("\\s\\(Status.*", content)
cleaned_output = utils.just_write(final_output,
result.replace('Found: ', ''))
if cleaned_output:
utils.check_output(command.get('cleaned_output'))
def clean_findomain(self, command):
utils.print_good('Cleaning for {0}:{1}'.format(
command.get('banner'), command.get('post_run')))
output = utils.just_read(command.get('output_path'), get_list=True)
if not output:
utils.print_bad('Output not found: {0}'.format(
command.get('output_path')))
return False
result = []
for line in output:
if '>>' in line.strip():
domain = line.strip().strip('>> ').split(' => ')[0]
ip = line.strip().strip('>> ').split(' => ')[0]
result.append(domain)
cleaned_output = utils.just_write(command.get('cleaned_output'),
"\n".join(result))
if cleaned_output:
utils.check_output(command.get('cleaned_output'))
def clean_massdns(self, command):
utils.print_good('Cleaning for {0}:{1}'.format(
command.get('banner'), command.get('post_run')))
output = utils.just_read(command.get('output_path'), get_list=True)
if output:
only_A_record, resolved, ips = [], [], []
for line in output:
if '. A ' in line:
ip = line.split('. A ')[1].strip()
domain = line.split('. A ')[0]
only_A_record.append(domain)
ips.append(ip)
resolved.append(line.split('. A ')[0])
cleaned_output = utils.just_write(command.get(
'cleaned_output'), "\n".join(ips))
if cleaned_output:
utils.check_output(command.get('cleaned_output'))
self.join_ip(command)
def get_amass(self, command):
ips, ip_range, asns = [], [], []
output_path = command.get('output_path')
content = utils.just_read(output_path, get_list=True)
if content:
ip_range.extend(content)
amass_output = utils.replace_argument(
self.options, '$WORKSPACE/subdomain/amass-$OUTPUT/amass.json')
if not utils.not_empty_file(amass_output):
return
real_ips = summary.get_ip(self.options)
if real_ips:
ips.extend(real_ips)
# parsed output from amass result
jsonl = utils.just_read(amass_output, get_list=True)
for line in jsonl:
json_data = utils.get_json(line)
for item in json_data.get('addresses'):
ips.append(item.get('ip'))
ip_range.append(item.get('cidr'))
asns.append(str(item.get('asn')))
ips_file = utils.replace_argument(
self.options, '$WORKSPACE/ipspace/summary-$OUTPUT.txt')
asn_file = utils.replace_argument(
self.options, '$WORKSPACE/ipspace/asn-$OUTPUT.txt')
range_file = utils.replace_argument(
self.options, '$WORKSPACE/ipspace/range-$OUTPUT.txt')
utils.just_write(ips_file, "\n".join(utils.unique_list(ips)))
utils.just_write(asn_file, "\n".join(utils.unique_list(asns)))
utils.just_write(range_file, "\n".join(utils.unique_list(ip_range)))
def update_summaries(self, summaries):
content = "\n".join(summaries)
formatted = utils.replace_argument(
self.options, '$WORKSPACE/probing/formatted-all-$OUTPUT.txt')
utils.just_write(formatted, content)
summary.push_with_file(self.options, formatted)
