def update_ports(self, command): utils.print_good('Cleaning for {0}:{1}'.format(command.get('banner'), command.get('pre_run'))) csv_data = utils.just_read(command.get('requirement'), get_list=True) if not csv_data: utils.print_bad('Requirement not found: {0}'.format( command.get('requirement'))) return False result = {} for line in csv_data[1:]: host = line.split(',')[0] port = line.split(',')[3] if result.get(host, None): result[host] += "," + str(port).strip(',') else: result[host] = port # store it as format can submit to summaries final_result = [] for host, ports in result.items(): item = "ip_address|{0};;ports|{1}".format(host, ports) final_result.append(item) utils.just_write(command.get('cleaned_output'), "\n".join(final_result)) summary.push_with_file(self.options, command.get('cleaned_output'))
def clean_massdns(self, command): utils.print_good('Cleaning for {0}:{1}'.format( command.get('banner'), command.get('post_run'))) output = utils.just_read(command.get('output_path'), get_list=True) if not output: utils.print_bad('Output not found: {0}'.format( command.get('output_path'))) return False only_A_record, summaries, resolved = [], [], [] for line in output: if '. A ' in line: only_A_record.append(line.split('. A ')[1].strip()) resolved.append(line.split('. A ')[0]) summary = "domain|{0};;ip_address|{1}".format( line.split('. A ')[0], line.split('. A ')[1]) summaries.append(summary) elif '. CNAME ' in line: resolved.append(line.split('. CNAME ')[0]) cleaned_output = utils.just_write(command.get('cleaned_output'), "\n".join(only_A_record)) resolved_path = utils.replace_argument( self.options, '$WORKSPACE/probing/resolved-$OUTPUT.txt') resolved_output = utils.just_write(resolved_path, "\n".join(resolved)) if cleaned_output: utils.check_output(command.get('cleaned_output')) if resolved_output: utils.check_output(resolved_path) self.update_summaries(summaries)
def update_ports(self, command): utils.print_good('Cleaning for {0}:{1}'.format( command.get('banner'), command.get('post_run'))) csv_data = utils.just_read(command.get('output_path'), get_list=True) if not csv_data: utils.print_bad('Output not found: {0}'.format( command.get('output_path'))) return False result = {} for line in csv_data[1:]: host = line.split(',')[0] port = line.split(',')[3] if result.get('host', None): result[host] += "," + str(port).strip(',') else: result[host] = port final_result = [] for host, ports in result.items(): item = "ip_address|{0};;ports|{1}".format(host, ports) final_result.append(item) utils.just_write(command.get('cleaned_output'), "\n".join(final_result))
def get_scheme(self, command): utils.print_good('Cleaning for {0}:{1}'.format( command.get('banner'), command.get('post_run'))) csv_data = utils.just_read(command.get('requirement'), get_list=True) if not csv_data: utils.print_bad('Output not found: {0}'.format( command.get('output_path'))) return False summaries, result = [], [] for line in csv_data[1:]: # print(line) if ',' not in line or len(line.split(',')) < 3: continue _results = line.split(',') host = _results[0].strip('"') port = _results[2].strip('"') service = _results[4].strip('"') + "/" + _results[5].strip('"') result.append("http://" + host + ":" + port) result.append("https://" + host + ":" + port) sum_line = f"domain|{host};;ip_address|{host};;ports|{port};;technologies|{service}" summaries.append(sum_line) # print(sum_line) scheme_path = utils.replace_argument( self.options, '$WORKSPACE/vulnscan/scheme-$OUTPUT.txt') utils.just_write(scheme_path, "\n".join(result)) # update summaries table formatted_summary = utils.replace_argument( self.options, '$WORKSPACE/vulnscan/formatted-summary-$OUTPUT.txt') utils.just_write(formatted_summary, "\n".join(summaries)) summary.push_with_file(self.options, formatted_summary)
def clean_gowitness(self, command): utils.print_good('Cleaning for {0}:{1}'.format( command.get('banner'), command.get('post_run'))) real_cmd = utils.resolve_command( self.options, { "banner": "gowitness gen report", "cmd": "$GO_PATH/gowitness report generate -n $WORKSPACE/portscan/screenshot/$OUTPUT-raw-gowitness.html --destination $WORKSPACE/portscan/screenshot/raw-gowitness/ --db $WORKSPACE/portscan/screenshot/gowitness.db", "output_path": "$WORKSPACE/portscan/screenshot/$OUTPUT-raw-gowitness.html", }) execute.send_cmd(self.options, real_cmd) raw_html = utils.just_read(real_cmd.get('output_path')) if not raw_html: utils.print_bad('Requirement not found: {0}'.format( real_cmd.get('output_path'))) return False local_path = utils.replace_argument(self.options, '$WORKSPACE/portscan/') real_html = raw_html.replace(local_path, '') utils.just_write(command.get('cleaned_output'), real_html) utils.check_output(command.get('cleaned_output'))
def clean_gowitness(self, command): utils.print_good('Cleaning for {0}:{1}'.format( command.get('banner'), command.get('post_run'))) raw_html = utils.just_read(command.get('output_path')) if not raw_html: utils.print_bad('Output not found: {0}'.format( command.get('output_path'))) return False local_path = utils.replace_argument( self.options, '$WORKSPACE/screenshot/') real_html = raw_html.replace(local_path, '') utils.just_write(command.get('cleaned_output'), real_html)
def get_domain(self, command): utils.print_good('Cleaning for {0}:{1}'.format( command.get('banner'), command.get('post_run'))) output = utils.just_read(command.get('output_path')) if not output: utils.print_bad('Output not found: {0}'.format( command.get('output_path'))) return False no_scheme = output.replace('https://', '').replace('http://', '') utils.just_write(command.get('cleaned_output'), no_scheme) if command.get('cleaned_output'): utils.check_output(command.get('cleaned_output'))
def gen_summary(self, command): summary_path = utils.replace_argument( self.options, '$WORKSPACE/vulnscan/summary-$OUTPUT.csv') sum_head = '"IP","FQDN","PORT","PROTOCOL","SERVICE","VERSION"' details_folder = utils.replace_argument( self.options, '$WORKSPACE/vulnscan/details/') details = utils.list_files(details_folder, '.csv') summary_data = [sum_head] for detail in details: really_detail = utils.just_read(detail, get_list=True) if really_detail: summary_data.append("\n".join(really_detail[1:])) utils.just_write(summary_path, "\n".join(summary_data))
def gen_summary(self, command): summary_path = utils.replace_argument( self.options, '$WORKSPACE/vulnscan/summary-$OUTPUT.csv') sum_head = 'IP,Host,OS,Proto,Port,Service,Product,Service FP,NSE Script ID,NSE Script Output,Notes' details_folder = utils.replace_argument( self.options, '$WORKSPACE/vulnscan/details/') details = utils.list_files(details_folder, '.csv') summary_data = [sum_head] for detail in details: really_detail = utils.just_read(detail, get_list=True) if really_detail: summary_data.append("\n".join(really_detail[1:])) utils.just_write(summary_path, "\n".join(summary_data))
def update_tech(self, command): utils.print_good('Cleaning for {0}:{1}'.format( command.get('banner'), command.get('post_run'))) output = utils.just_read(command.get('output_path'), get_list=True) if not output: utils.print_bad('Output not found: {0}'.format( command.get('output_path'))) return False tech_summaries = [] for line in output: try: jsonl = utils.get_json(line) if jsonl.get('matches'): domain = utils.get_domain(jsonl.get('hostname')) techs = [x.get('app_name') for x in jsonl.get('matches')] item = "domain|{0};;technologies|{1}".format( domain, ",".join(techs)) tech_summaries.append(item) except: pass cleaned_output = utils.just_write(command.get('cleaned_output'), "\n".join(tech_summaries)) if cleaned_output: utils.check_output(command.get('cleaned_output')) # update technologies to db summary.push_with_file(self.options, command.get('cleaned_output'))
def join_ip(self, command): cleaned_output = utils.just_read(command.get('cleaned_output'), get_list=True) raw_input = utils.just_read( command.get('requirement'), get_list=True) result = [] for line in raw_input: if utils.valid_ip(line.strip()): result.append(line) if cleaned_output: result = list(set(result + cleaned_output)) else: result = list(set(result)) if result: utils.just_write(command.get('cleaned_output'), "\n".join(result)) summaries = [] for item in result: summary = f"domain|{item};;ip_address|{item}" summaries.append(summary) self.update_summaries(summaries)
def get_scheme(self, command): utils.print_good('Preparing for {0}:{1}'.format( command.get('banner'), command.get('pre_run'))) scheme_path = utils.replace_argument( self.options, '$WORKSPACE/portscan/scheme-$OUTPUT.txt') csv_data = utils.just_read(command.get('requirement'), get_list=True) if not csv_data: utils.print_bad('Requirement not found: {0}'.format( command.get('requirement'))) return False result = [] for line in csv_data[1:]: host = line.split(',')[0] port = line.split(',')[3] result.append("http://" + host + ":" + port) result.append("https://" + host + ":" + port) utils.just_write(scheme_path, "\n".join(result)) utils.check_output(scheme_path)
def clean_gobuster(self, command): utils.print_good('Cleaning for {0}:{1}'.format( command.get('banner'), command.get('post_run'))) output = utils.just_read(command.get('output_path')) if not output: utils.print_bad('Output not found: {0}'.format( command.get('output_path'))) return False cleaned_output = utils.just_write(command.get('cleaned_output'), output.replace('Found: ', '')) if cleaned_output: utils.check_output(command.get('cleaned_output'))
def clean_massdns(self, command): utils.print_good('Cleaning for {0}:{1}'.format(command.get('banner'), command.get('post_run'))) output = utils.just_read(command.get('output_path'), get_list=True) if not output: utils.print_bad('Output not found: {0}'.format( command.get('output_path'))) return False # only get A record only_A_record = "\n".join([x.split('. A ')[0] for x in output if '. A ' in x]) cleaned_output = utils.just_write(command.get( 'cleaned_output'), only_A_record) if cleaned_output: utils.check_output(command.get('cleaned_output'))
def clean_vhosts_gobuster(self, command): final_output = utils.replace_argument( self.options, "$WORKSPACE/vhosts/vhosts-$OUTPUT.txt") raw_outputs = utils.replace_argument( self.options, "$WORKSPACE/vhosts/raw-summary-$OUTPUT.txt") content = utils.just_read(raw_outputs) if not content: return result = utils.regex_strip("\\s\\(Status.*", content) cleaned_output = utils.just_write( final_output, result.replace('Found: ', '')) if cleaned_output: utils.check_output(command.get( 'cleaned_output'))
def clean_multi_gobuster(self, command): final_output = utils.replace_argument( self.options, "$WORKSPACE/vhosts/vhost-$OUTPUT.txt") # simple hack here raw_outputs = utils.list_files(final_output + '/../raw/', '-gobuster.txt') utils.join_files(raw_outputs, final_output) # content = final_output content = utils.just_read(final_output) if content: result = utils.regex_strip("\\s\\(Status.*", content) cleaned_output = utils.just_write(final_output, result.replace('Found: ', '')) if cleaned_output: utils.check_output(command.get('cleaned_output'))
def clean_findomain(self, command): utils.print_good('Cleaning for {0}:{1}'.format( command.get('banner'), command.get('post_run'))) output = utils.just_read(command.get('output_path'), get_list=True) if not output: utils.print_bad('Output not found: {0}'.format( command.get('output_path'))) return False result = [] for line in output: if '>>' in line.strip(): domain = line.strip().strip('>> ').split(' => ')[0] ip = line.strip().strip('>> ').split(' => ')[0] result.append(domain) cleaned_output = utils.just_write(command.get('cleaned_output'), "\n".join(result)) if cleaned_output: utils.check_output(command.get('cleaned_output'))
def clean_massdns(self, command): utils.print_good('Cleaning for {0}:{1}'.format( command.get('banner'), command.get('post_run'))) output = utils.just_read(command.get('output_path'), get_list=True) if output: only_A_record, resolved, ips = [], [], [] for line in output: if '. A ' in line: ip = line.split('. A ')[1].strip() domain = line.split('. A ')[0] only_A_record.append(domain) ips.append(ip) resolved.append(line.split('. A ')[0]) cleaned_output = utils.just_write(command.get( 'cleaned_output'), "\n".join(ips)) if cleaned_output: utils.check_output(command.get('cleaned_output')) self.join_ip(command)
def get_amass(self, command): ips, ip_range, asns = [], [], [] output_path = command.get('output_path') content = utils.just_read(output_path, get_list=True) if content: ip_range.extend(content) amass_output = utils.replace_argument( self.options, '$WORKSPACE/subdomain/amass-$OUTPUT/amass.json') if not utils.not_empty_file(amass_output): return real_ips = summary.get_ip(self.options) if real_ips: ips.extend(real_ips) # parsed output from amass result jsonl = utils.just_read(amass_output, get_list=True) for line in jsonl: json_data = utils.get_json(line) for item in json_data.get('addresses'): ips.append(item.get('ip')) ip_range.append(item.get('cidr')) asns.append(str(item.get('asn'))) ips_file = utils.replace_argument( self.options, '$WORKSPACE/ipspace/summary-$OUTPUT.txt') asn_file = utils.replace_argument( self.options, '$WORKSPACE/ipspace/asn-$OUTPUT.txt') range_file = utils.replace_argument( self.options, '$WORKSPACE/ipspace/range-$OUTPUT.txt') utils.just_write(ips_file, "\n".join(utils.unique_list(ips))) utils.just_write(asn_file, "\n".join(utils.unique_list(asns))) utils.just_write(range_file, "\n".join(utils.unique_list(ip_range)))
def update_summaries(self, summaries): content = "\n".join(summaries) formatted = utils.replace_argument( self.options, '$WORKSPACE/probing/formatted-all-$OUTPUT.txt') utils.just_write(formatted, content) summary.push_with_file(self.options, formatted)