def pretty_print(self, show_all=False): """Outputs the translation stats to a table formatted view.""" if not self.untranslated and not self.untagged: self.log('\nNo untranslated strings found.\n') return # Most frequent untranslated and untagged messages. if self.untagged: table = texttable.Texttable(max_width=120) table.set_deco(texttable.Texttable.HEADER) table.set_cols_dtype(['t', 'i', 't']) table.set_cols_align(['l', 'r', 'l']) rows = [] missing = self.missing for locale in missing: for message in missing[locale]: rows.append( [str(locale), missing[locale][message], message]) rows = sorted(rows, key=lambda x: -x[1]) if not show_all: num_rows = len(rows) rows = rows[:self.ROW_COUNT] if num_rows > self.ROW_COUNT: rows.append([ '', num_rows - self.ROW_COUNT, '+ Additional untranslated strings...' ]) table.add_rows( [['Locale', '#', 'Untagged and Untranslated Message']] + rows) self.log('\n' + table.draw() + '\n') # Most frequent untranslated messages. if self.untranslated: table = texttable.Texttable(max_width=120) table.set_deco(texttable.Texttable.HEADER) table.set_cols_dtype(['t', 'i', 't']) table.set_cols_align(['l', 'r', 'l']) rows = [] for locale in self.untranslated: for message in self.untranslated[locale]: rows.append([ str(locale), self.untranslated[locale][message], message ]) rows = sorted(rows, key=lambda x: -x[1]) if not show_all: num_rows = len(rows) rows = rows[:self.ROW_COUNT] if num_rows > self.ROW_COUNT: rows.append([ '', num_rows - self.ROW_COUNT, '+ Additional untranslated strings...' ]) table.add_rows([['Locale', '#', 'Untranslated Message']] + rows) self.log('\n' + table.draw() + '\n') # Untranslated messages per locale. table = texttable.Texttable(max_width=120) table.set_deco(texttable.Texttable.HEADER) table.set_cols_dtype(['t', 'i']) table.set_cols_align(['l', 'r']) rows = [] for locale in self.untranslated: rows.append([str(locale), len(self.untranslated[locale])]) rows = sorted(rows, key=lambda x: -x[1]) table.add_rows([['Locale', 'Untranslated']] + rows) self.log('\n' + table.draw() + '\n')
def get_table(): table = texttable.Texttable(max_width=140) table.set_deco(0) return table
In this project, I was able to build two different tables with prefilled user data. I was able to store all of the information into the appropriate variables as lists of information. I then was able to gain user input for the person's name. I then performed the logic of earnings or losses and built a new list with those given values by using the Python append() function. I imported a library known as texttable to build the first formatted table of information and have also built a custom table beneath it. I used a proper header with the name value and the format() functions for appending all of these values to a string of text. Attached to this submission as well is a table that is built completely off of user input. I wanted to give it a try with functions in Python. """ # Import Text Table Module/Library import texttable # Set table = texttable.Texttable() # Pre-defined variables of stock information name = input("What is your full name? ") titles = ["Stock Symbol", "NO Shares", "Purchase Price", "Current Value", "Earnings/Losses"] stock_symbols = ["GOOGL", "MSFT", "RDS-A", "AIG", "FB"] number_shares = [125, 85, 400, 235, 150] purchase_prices = [772.88, 56.60, 49.58, 54.21, 124.31] current_prices = [941.53, 73.04, 55.74, 65.27, 172.45] earns_losses = [] full_list = [] # The formula is (Shares own * current price) - (shares own * bought price)
def list_all(self, jails): """List all jails.""" table = texttable.Texttable(max_width=0) jail_list = [] for jail in jails: jail = jail.properties["mountpoint"].value conf = iocage.lib.ioc_json.IOCJson(jail).json_load() uuid = conf["host_hostuuid"] full_ip4 = conf["ip4_addr"] ip6 = conf["ip6_addr"] jail_root = f"{self.pool}/iocage/jails/{uuid}/root" try: short_ip4 = full_ip4.split("|")[1].split("/")[0] except IndexError: short_ip4 = "-" tag = conf["tag"] boot = conf["boot"] jail_type = conf["type"] full_release = conf["release"] if "HBSD" in full_release: full_release = re.sub(r"\W\w.", "-", full_release) full_release = full_release.replace("--SD", "-STABLE-HBSD") short_release = full_release.rstrip("-HBSD") else: short_release = "-".join(full_release.rsplit("-")[:2]) if full_ip4 == "none": full_ip4 = "-" status, jid = self.list_get_jid(uuid) if status: state = "up" else: state = "down" if conf["type"] == "template": template = "-" else: try: template = iocage.lib.ioc_common.checkoutput([ "zfs", "get", "-H", "-o", "value", "origin", jail_root ]).split("/")[3] except IndexError: template = "-" if "release" in template.lower() or "stable" in template.lower(): template = "-" # Append the JID and the UUID to the table if self.full: jail_list.append([ jid, uuid, boot, state, tag, jail_type, full_release, full_ip4, ip6, template ]) else: jail_list.append( [jid, uuid[:8], state, tag, short_release, short_ip4]) list_type = "list_full" if self.full else "list_short" sort = iocage.lib.ioc_common.ioc_sort(list_type, self.sort, data=jail_list) jail_list.sort(key=sort) # Prints the table if self.header: if self.full: # We get an infinite float otherwise. table.set_cols_dtype( ["t", "t", "t", "t", "t", "t", "t", "t", "t", "t"]) jail_list.insert(0, [ "JID", "UUID", "BOOT", "STATE", "TAG", "TYPE", "RELEASE", "IP4", "IP6", "TEMPLATE" ]) else: # We get an infinite float otherwise. table.set_cols_dtype(["t", "t", "t", "t", "t", "t"]) jail_list.insert( 0, ["JID", "UUID", "STATE", "TAG", "RELEASE", "IP4"]) table.add_rows(jail_list) return table.draw() else: flat_jail = [j for j in jail_list] return flat_jail
def run(argv): fixtures = requests.get(URL, headers=headers).json()['fixtures'] fixturesR16_json = list(filter(lambda x: x['matchday'] == 4, fixtures)) fixturesR8_json_QF = list(filter(lambda x: x['matchday'] == 5, fixtures)) fixturesR4_json_SF = list(filter(lambda x: x['matchday'] == 6, fixtures)) fixturesR2_json_F = list(filter(lambda x: x['matchday'] == 7, fixtures)) fixturesR2_json_TP = list(filter(lambda x: x['matchday'] == 8, fixtures)) fixturesR16_knockOutMatch = matchResults(fixturesR16_json) fixturesR8_knockOutMatch_QF = matchResults(fixturesR8_json_QF) fixturesR4_knockOutMatch_SF = matchResults(fixturesR4_json_SF) fixturesR2_knockOutMatch_F = matchResults(fixturesR2_json_F) fixturesR2_knockOutMatch_TP = matchResults(fixturesR2_json_TP) # fixturesR16_knockOutMatch = list(map( # lambda x: knockOutMatch(x['homeTeamName'], x['awayTeamName'], str(x['result']['goalsHomeTeam']) + ' ('+ str(x['result'].get('penaltyShootout', {0:0}).get('goalsHomeTeam', '') ) + ')', # x['result']['goalsAwayTeam'], x['date']).__str__(), fixturesR16_json)) # fixturesR8_knockOutMatch_QF = list(map( # lambda x: knockOutMatch(x['homeTeamName'], x['awayTeamName'], x['result']['goalsHomeTeam'], # x['result']['goalsAwayTeam'], x['date']).__str__(), fixturesR8_json_QF)) # fixturesR4_knockOutMatch_SF = list(map( # lambda x: knockOutMatch(x['homeTeamName'], x['awayTeamName'], x['result']['goalsHomeTeam'], # x['result']['goalsAwayTeam'], x['date']).__str__(), fixturesR4_json_SF)) # fixturesR2_knockOutMatch_F = list(map( # lambda x: knockOutMatch(x['homeTeamName'], x['awayTeamName'], x['result']['goalsHomeTeam'], # x['result']['goalsAwayTeam'], x['date']).__str__(), fixturesR2_json_F)) # fixturesR2_knockOutMatch_TP = list(map( # lambda x: knockOutMatch(x['homeTeamName'], x['awayTeamName'], x['result']['goalsHomeTeam'], # x['result']['goalsAwayTeam'], x['date']).__str__(), fixturesR2_json_TP)) rightDownArrow = "\n\n_ _ _ _ _ _ .\n|\n|\n|\n|\n\u2193" rightUpArrow = "\u2191\n|\n|\n|\n|\n_ _ _ _ _ _ .\n\n" downDownArrow = "\n|\n|\n|\n|\n|\n|\n|\n\u2193" upUpArrow = "\u2191\n|\n|\n|\n|\n|\n|\n|" # r16 = [fixturesR16_knockOutMatch[0], '', fixturesR16_knockOutMatch[1], '', fixturesR16_knockOutMatch[2], '', fixturesR16_knockOutMatch[3], '', fixturesR16_knockOutMatch[4], '', fixturesR16_knockOutMatch[5], '', fixturesR16_knockOutMatch[6], '', fixturesR16_knockOutMatch[7] ] qf = [rightDownArrow, fixturesR8_knockOutMatch_QF[0], rightUpArrow,'',rightDownArrow, fixturesR8_knockOutMatch_QF[1], rightUpArrow,'',rightDownArrow, fixturesR8_knockOutMatch_QF[2], rightUpArrow,'',rightDownArrow, fixturesR8_knockOutMatch_QF[3], rightUpArrow ] sf = ['',rightDownArrow, downDownArrow, fixturesR4_knockOutMatch_SF[0], upUpArrow, rightUpArrow, '','','',rightDownArrow, downDownArrow, fixturesR4_knockOutMatch_SF[1], upUpArrow, rightUpArrow, '' ] f = ['', '', '',rightDownArrow, downDownArrow, downDownArrow, downDownArrow, fixturesR2_knockOutMatch_F[0], upUpArrow, upUpArrow, upUpArrow, rightUpArrow, '', '', '' ] l_all_matches_transpose = [r16, qf, sf, f] l_all_matches = list(zip(*l_all_matches_transpose)) print('+---------------------------------------------------------------------------------------------------+') print('---------------------------------FIFA WORLDCUP 2018 KNOCK-OUT STAGE----------------------------------') table = texttable.Texttable() table.set_cols_align(["c", "c", "c", "c"]) table.set_cols_width([23, 23, 23, 23]) table.set_deco(table.BORDER) table.add_rows(l_all_matches, header=False) print(table.draw()) print('---------------------------------created by harleen mann on 30-6-18----------------------------------')
async def start(): parser = argparse.ArgumentParser( description= 'theHarvester is used to gather open source intelligence (OSINT) on a company or domain.' ) parser.add_argument('-d', '--domain', help='Company name or domain to search.', required=True) parser.add_argument( '-l', '--limit', help='Limit the number of search results, default=500.', default=500, type=int) parser.add_argument('-S', '--start', help='Start with result number X, default=0.', default=0, type=int) parser.add_argument('-g', '--google-dork', help='Use Google Dorks for Google search.', default=False, action='store_true') parser.add_argument( '-p', '--proxies', help='Use proxies for requests, enter proxies in proxies.yaml.', default=False, action='store_true') parser.add_argument('-s', '--shodan', help='Use Shodan to query discovered hosts.', default=False, action='store_true') parser.add_argument( '--screenshot', help= 'Take screenshots of resolved domains specify output directory: --screenshot output_directory', default="", type=str) parser.add_argument( '-v', '--virtual-host', help= 'Verify host name via DNS resolution and search for virtual hosts.', action='store_const', const='basic', default=False) parser.add_argument('-e', '--dns-server', help='DNS server to use for lookup.') parser.add_argument( '-t', '--dns-tld', help='Perform a DNS TLD expansion discovery, default False.', default=False) parser.add_argument('-r', '--take-over', help='Check for takeovers.', default=False, action='store_true') parser.add_argument('-n', '--dns-lookup', help='Enable DNS server lookup, default False.', default=False, action='store_true') parser.add_argument('-c', '--dns-brute', help='Perform a DNS brute force on the domain.', default=False, action='store_true') parser.add_argument('-f', '--filename', help='Save the results to an HTML and/or XML file.', default='', type=str) parser.add_argument( '-b', '--source', help= '''baidu, bing, bingapi, bufferoverun, certspotter, crtsh, dnsdumpster, duckduckgo, exalead, github-code, google, hackertarget, hunter, intelx, linkedin, linkedin_links, netcraft, otx, pentesttools, qwant, rapiddns, securityTrails, spyse, sublist3r, threatcrowd, threatminer, trello, twitter, urlscan, virustotal, yahoo''') args = parser.parse_args() filename: str = args.filename dnsbrute = (args.dns_brute, False) try: db = stash.StashManager() await db.do_init() except Exception: pass all_emails: list = [] all_hosts: list = [] all_ip: list = [] dnslookup = args.dns_lookup dnsserver = args.dns_server dnstld = args.dns_tld engines = [] # If the user specifies full: list = [] ips: list = [] google_dorking = args.google_dork host_ip: list = [] limit: int = args.limit shodan = args.shodan start: int = args.start all_urls: list = [] vhost: list = [] virtual = args.virtual_host word: str = args.domain takeover_status = args.take_over use_proxy = args.proxies async def store(search_engine: Any, source: str, process_param: Any = None, store_host: bool = False, store_emails: bool = False, store_ip: bool = False, store_people: bool = False, store_links: bool = False, store_results: bool = False) -> None: """ Persist details into the database. The details to be stored is controlled by the parameters passed to the method. :param search_engine: search engine to fetch details from :param source: source against which the details (corresponding to the search engine) need to be persisted :param process_param: any parameters to be passed to the search engine eg: Google needs google_dorking :param store_host: whether to store hosts :param store_emails: whether to store emails :param store_ip: whether to store IP address :param store_people: whether to store user details :param store_links: whether to store links :param store_results: whether to fetch details from get_results() and persist """ await search_engine.process(use_proxy) if process_param is None else await \ search_engine.process(process_param, use_proxy) db_stash = stash.StashManager() if source == 'suip': print( f'\033[94m[*] Searching {source[0].upper() + source[1:]} this module can take 10+ min but is worth ' f'it. \033[0m') else: print( f'\033[94m[*] Searching {source[0].upper() + source[1:]}. \033[0m' ) if store_host: host_names = [ host for host in filter(await search_engine.get_hostnames()) if f'.{word}' in host ] if source != 'hackertarget' and source != 'pentesttools' and source != 'rapiddns': # If source is inside this conditional it means the hosts returned must be resolved to obtain ip full_hosts_checker = hostchecker.Checker(host_names) temp_hosts, temp_ips = await full_hosts_checker.check() ips.extend(temp_ips) full.extend(temp_hosts) else: full.extend(host_names) all_hosts.extend(host_names) await db_stash.store_all(word, all_hosts, 'host', source) if store_emails: email_list = filter(await search_engine.get_emails()) all_emails.extend(email_list) await db_stash.store_all(word, email_list, 'email', source) if store_ip: ips_list = await search_engine.get_ips() all_ip.extend(ips_list) await db_stash.store_all(word, all_ip, 'ip', source) if store_results: email_list, host_names, urls = await search_engine.get_results() all_emails.extend(email_list) host_names = [ host for host in filter(host_names) if f'.{word}' in host ] all_urls.extend(filter(urls)) all_hosts.extend(host_names) await db.store_all(word, all_hosts, 'host', source) await db.store_all(word, all_emails, 'email', source) if store_people: people_list = await search_engine.get_people() await db_stash.store_all(word, people_list, 'people', source) if len(people_list) == 0: print('\n[*] No users found.\n\n') else: print('\n[*] Users found: ' + str(len(people_list))) print('---------------------') for usr in sorted(list(set(people_list))): print(usr) if store_links: links = await search_engine.get_links() await db.store_all(word, links, 'name', engineitem) if len(links) == 0: print('\n[*] No links found.\n\n') else: print(f'\n[*] Links found: {len(links)}') print('---------------------') for link in sorted(list(set(links))): print(link) stor_lst = [] if args.source is not None: if args.source.lower() != 'all': engines = sorted(set(map(str.strip, args.source.split(',')))) else: engines = Core.get_supportedengines() # Iterate through search engines in order if set(engines).issubset(Core.get_supportedengines()): print(f'\033[94m[*] Target: {word} \n \033[0m') for engineitem in engines: if engineitem == 'baidu': from theHarvester.discovery import baidusearch try: baidu_search = baidusearch.SearchBaidu(word, limit) stor_lst.append( store(baidu_search, engineitem, store_host=True, store_emails=True)) except Exception: pass elif engineitem == 'bing' or engineitem == 'bingapi': from theHarvester.discovery import bingsearch try: bing_search = bingsearch.SearchBing(word, limit, start) bingapi = '' if engineitem == 'bingapi': bingapi += 'yes' else: bingapi += 'no' stor_lst.append( store(bing_search, 'bing', process_param=bingapi, store_host=True, store_emails=True)) except Exception as e: if isinstance(e, MissingKey): print(e) else: print(e) elif engineitem == 'bufferoverun': from theHarvester.discovery import bufferoverun try: bufferoverun_search = bufferoverun.SearchBufferover( word) stor_lst.append( store(bufferoverun_search, engineitem, store_host=True, store_ip=True)) except Exception as e: print(e) elif engineitem == 'certspotter': from theHarvester.discovery import certspottersearch try: certspotter_search = certspottersearch.SearchCertspoter( word) stor_lst.append( store(certspotter_search, engineitem, None, store_host=True)) except Exception as e: print(e) elif engineitem == 'crtsh': try: from theHarvester.discovery import crtsh crtsh_search = crtsh.SearchCrtsh(word) stor_lst.append( store(crtsh_search, 'CRTsh', store_host=True)) except Exception as e: print( f'\033[93m[!] A timeout occurred with crtsh, cannot find {args.domain}\n {e}\033[0m' ) elif engineitem == 'dnsdumpster': try: from theHarvester.discovery import dnsdumpster dns_dumpster_search = dnsdumpster.SearchDnsDumpster( word) stor_lst.append( store(dns_dumpster_search, engineitem, store_host=True)) except Exception as e: print( f'\033[93m[!] An error occurred with dnsdumpster: {e} \033[0m' ) elif engineitem == 'duckduckgo': from theHarvester.discovery import duckduckgosearch duckduckgo_search = duckduckgosearch.SearchDuckDuckGo( word, limit) stor_lst.append( store(duckduckgo_search, engineitem, store_host=True, store_emails=True)) elif engineitem == 'exalead': from theHarvester.discovery import exaleadsearch exalead_search = exaleadsearch.SearchExalead( word, limit, start) stor_lst.append( store(exalead_search, engineitem, store_host=True, store_emails=True)) elif engineitem == 'github-code': try: from theHarvester.discovery import githubcode github_search = githubcode.SearchGithubCode( word, limit) stor_lst.append( store(github_search, engineitem, store_host=True, store_emails=True)) except MissingKey as ex: print(ex) else: pass elif engineitem == 'google': from theHarvester.discovery import googlesearch google_search = googlesearch.SearchGoogle( word, limit, start) stor_lst.append( store(google_search, engineitem, process_param=google_dorking, store_host=True, store_emails=True)) elif engineitem == 'hackertarget': from theHarvester.discovery import hackertarget hackertarget_search = hackertarget.SearchHackerTarget(word) stor_lst.append( store(hackertarget_search, engineitem, store_host=True)) elif engineitem == 'hunter': from theHarvester.discovery import huntersearch # Import locally or won't work. try: hunter_search = huntersearch.SearchHunter( word, limit, start) stor_lst.append( store(hunter_search, engineitem, store_host=True, store_emails=True)) except Exception as e: if isinstance(e, MissingKey): print(e) else: pass elif engineitem == 'intelx': from theHarvester.discovery import intelxsearch # Import locally or won't work. try: intelx_search = intelxsearch.SearchIntelx(word, limit) stor_lst.append( store(intelx_search, engineitem, store_host=True, store_emails=True)) except Exception as e: if isinstance(e, MissingKey): print(e) else: print( f'An exception has occurred in Intelx search: {e}' ) elif engineitem == 'linkedin': from theHarvester.discovery import linkedinsearch linkedin_search = linkedinsearch.SearchLinkedin( word, limit) stor_lst.append( store(linkedin_search, engineitem, store_people=True)) elif engineitem == 'linkedin_links': from theHarvester.discovery import linkedinsearch linkedin_links_search = linkedinsearch.SearchLinkedin( word, limit) stor_lst.append( store(linkedin_links_search, 'linkedin', store_links=True)) elif engineitem == 'netcraft': from theHarvester.discovery import netcraft netcraft_search = netcraft.SearchNetcraft(word) stor_lst.append( store(netcraft_search, engineitem, store_host=True)) elif engineitem == 'otx': from theHarvester.discovery import otxsearch try: otxsearch_search = otxsearch.SearchOtx(word) stor_lst.append( store(otxsearch_search, engineitem, store_host=True, store_ip=True)) except Exception as e: print(e) elif engineitem == 'pentesttools': from theHarvester.discovery import pentesttools try: pentesttools_search = pentesttools.SearchPentestTools( word) stor_lst.append( store(pentesttools_search, engineitem, store_host=True)) except Exception as e: if isinstance(e, MissingKey): print(e) else: print( f'An exception has occurred in PentestTools search: {e}' ) elif engineitem == 'qwant': from theHarvester.discovery import qwantsearch qwant_search = qwantsearch.SearchQwant(word, start, limit) stor_lst.append( store(qwant_search, engineitem, store_host=True, store_emails=True)) elif engineitem == 'rapiddns': from theHarvester.discovery import rapiddns try: rapiddns_search = rapiddns.SearchRapidDns(word) stor_lst.append( store(rapiddns_search, engineitem, store_host=True)) except Exception as e: print(e) elif engineitem == 'securityTrails': from theHarvester.discovery import securitytrailssearch try: securitytrails_search = securitytrailssearch.SearchSecuritytrail( word) stor_lst.append( store(securitytrails_search, engineitem, store_host=True, store_ip=True)) except Exception as e: if isinstance(e, MissingKey): print(e) else: pass elif engineitem == 'sublist3r': from theHarvester.discovery import sublist3r try: sublist3r_search = sublist3r.SearchSublist3r(word) stor_lst.append( store(sublist3r_search, engineitem, store_host=True)) except Exception as e: print(e) elif engineitem == 'spyse': from theHarvester.discovery import spyse try: spyse_search = spyse.SearchSpyse(word) stor_lst.append( store(spyse_search, engineitem, store_host=True, store_ip=True)) except Exception as e: print(e) elif engineitem == 'threatcrowd': from theHarvester.discovery import threatcrowd try: threatcrowd_search = threatcrowd.SearchThreatcrowd( word) stor_lst.append( store(threatcrowd_search, engineitem, store_host=True)) except Exception as e: print(e) elif engineitem == 'threatminer': from theHarvester.discovery import threatminer try: threatminer_search = threatminer.SearchThreatminer( word) stor_lst.append( store(threatminer_search, engineitem, store_host=True)) except Exception as e: print(e) elif engineitem == 'trello': from theHarvester.discovery import trello # Import locally or won't work. trello_search = trello.SearchTrello(word) stor_lst.append( store(trello_search, engineitem, store_results=True)) elif engineitem == 'twitter': from theHarvester.discovery import twittersearch twitter_search = twittersearch.SearchTwitter(word, limit) stor_lst.append( store(twitter_search, engineitem, store_people=True)) elif engineitem == 'urlscan': from theHarvester.discovery import urlscan try: urlscan_search = urlscan.SearchUrlscan(word) stor_lst.append( store(urlscan_search, engineitem, store_host=True, store_ip=True)) except Exception as e: print(e) elif engineitem == 'virustotal': from theHarvester.discovery import virustotal virustotal_search = virustotal.SearchVirustotal(word) stor_lst.append( store(virustotal_search, engineitem, store_host=True)) elif engineitem == 'yahoo': from theHarvester.discovery import yahoosearch yahoo_search = yahoosearch.SearchYahoo(word, limit) stor_lst.append( store(yahoo_search, engineitem, store_host=True, store_emails=True)) else: print('\033[93m[!] Invalid source.\n\n \033[0m') sys.exit(1) async def worker(queue): while True: # Get a "work item" out of the queue. stor = await queue.get() try: await stor queue.task_done() # Notify the queue that the "work item" has been processed. except Exception: queue.task_done() async def handler(lst): queue = asyncio.Queue() for stor_method in lst: # enqueue the coroutines queue.put_nowait(stor_method) # Create five worker tasks to process the queue concurrently. tasks = [] for i in range(5): task = asyncio.create_task(worker(queue)) tasks.append(task) # Wait until the queue is fully processed. await queue.join() # Cancel our worker tasks. for task in tasks: task.cancel() # Wait until all worker tasks are cancelled. await asyncio.gather(*tasks, return_exceptions=True) await handler(lst=stor_lst) # Sanity check to see if all_emails and all_hosts are defined. try: all_emails except NameError: print( '\n\n\033[93m[!] No emails found because all_emails is not defined.\n\n \033[0m' ) sys.exit(1) try: all_hosts except NameError: print( '\n\n\033[93m[!] No hosts found because all_hosts is not defined.\n\n \033[0m' ) sys.exit(1) # Results if len(all_ip) == 0: print('\n[*] No IPs found.') else: print('\n[*] IPs found: ' + str(len(all_ip))) print('-------------------') # use netaddr as the list may contain ipv4 and ipv6 addresses ip_list = sorted([netaddr.IPAddress(ip.strip()) for ip in set(all_ip)]) print('\n'.join(map(str, ip_list))) if len(all_emails) == 0: print('\n[*] No emails found.') else: print('\n[*] Emails found: ' + str(len(all_emails))) print('----------------------') print(('\n'.join(sorted(list(set(all_emails)))))) if len(all_hosts) == 0: print('\n[*] No hosts found.\n\n') else: print('\n[*] Hosts found: ' + str(len(all_hosts))) print('---------------------') all_hosts = sorted(list(set(all_hosts))) db = stash.StashManager() full = [ host if ':' in host and word in host else word in host.split(':')[0] and host for host in full ] full = list({host for host in full if host}) full.sort(key=lambda el: el.split(':')[0]) for host in full: print(host) host_ip = [ netaddr_ip.format() for netaddr_ip in sorted([netaddr.IPAddress(ip) for ip in ips]) ] await db.store_all(word, host_ip, 'ip', 'DNS-resolver') length_urls = len(all_urls) if length_urls == 0: if len(engines) >= 1 and 'trello' in engines: print('\n[*] No Trello URLs found.') else: total = length_urls print('\n[*] Trello URLs found: ' + str(total)) print('--------------------') for url in sorted(all_urls): print(url) # DNS brute force if dnsbrute and dnsbrute[0] is True: print('\n[*] Starting DNS brute force.') dns_force = dnssearch.DnsForce(word, dnsserver, verbose=True) hosts, ips = await dns_force.run() hosts = list({host for host in hosts if ':' in host}) hosts.sort(key=lambda el: el.split(':')[0]) print('\n[*] Hosts found after DNS brute force:') db = stash.StashManager() for host in hosts: print(host) full.append(host) await db.store_all(word, hosts, 'host', 'dns_bruteforce') # TakeOver Checking if takeover_status: print('\n[*] Performing subdomain takeover check') print('\n[*] Subdomain Takeover checking IS ACTIVE RECON') search_take = takeover.TakeOver(all_hosts) await search_take.process(proxy=use_proxy) # DNS reverse lookup dnsrev = [] if dnslookup is True: print('\n[*] Starting active queries.') # load the reverse dns tools from theHarvester.discovery.dnssearch import ( generate_postprocessing_callback, reverse_all_ips_in_range, serialize_ip_range) # reverse each iprange in a separate task __reverse_dns_tasks = {} for entry in host_ip: __ip_range = serialize_ip_range(ip=entry, netmask='24') if __ip_range and __ip_range not in set( __reverse_dns_tasks.keys()): print('\n[*] Performing reverse lookup on ' + __ip_range) __reverse_dns_tasks[__ip_range] = asyncio.create_task( reverse_all_ips_in_range( iprange=__ip_range, callback=generate_postprocessing_callback( target=word, local_results=dnsrev, overall_results=full), nameservers=list(map(str, dnsserver.split(','))) if dnsserver else None)) # run all the reversing tasks concurrently await asyncio.gather(*__reverse_dns_tasks.values()) # Display the newly found hosts print('\n[*] Hosts found after reverse lookup (in target domain):') print('--------------------------------------------------------') for xh in dnsrev: print(xh) # DNS TLD expansion dnstldres = [] if dnstld is True: print('[*] Starting DNS TLD expansion.') a = dnssearch.DnsTld(word, dnsserver, verbose=True) res = a.process() print('\n[*] Hosts found after DNS TLD expansion:') print('----------------------------------------') for y in res: print(y) dnstldres.append(y) if y not in full: full.append(y) # Virtual hosts search if virtual == 'basic': print('\n[*] Virtual hosts:') print('------------------') for data in host_ip: basic_search = bingsearch.SearchBing(data, limit, start) await basic_search.process_vhost() results = await basic_search.get_allhostnames() for result in results: result = re.sub(r'[[</?]*[\w]*>]*', '', result) result = re.sub('<', '', result) result = re.sub('>', '', result) print((data + '\t' + result)) vhost.append(data + ':' + result) full.append(data + ':' + result) vhost = sorted(set(vhost)) else: pass # Screenshots screenshot_tups = [] if len(args.screenshot) > 0: import time from aiomultiprocess import Pool from theHarvester.screenshot.screenshot import ScreenShotter screen_shotter = ScreenShotter(args.screenshot) path_exists = screen_shotter.verify_path() # Verify path exists if not create it or if user does not create it skip screenshot if path_exists: await screen_shotter.verify_installation() print( f'\nScreenshots can be found in: {screen_shotter.output}{screen_shotter.slash}' ) start_time = time.perf_counter() print('Filtering domains for ones we can reach') unique_resolved_domains = { url.split(':')[0] for url in full if ':' in url and 'www.' not in url } if len(unique_resolved_domains) > 0: # First filter out ones that didn't resolve print( 'Attempting to visit unique resolved domains, this is ACTIVE RECON' ) async with Pool(15) as pool: results = await pool.map(screen_shotter.visit, list(unique_resolved_domains)) # Filter out domains that we couldn't connect to unique_resolved_domains = list( sorted({tup[0] for tup in results if len(tup[1]) > 0})) async with Pool(3) as pool: print( f'Length of unique resolved domains: {len(unique_resolved_domains)} chunking now!\n' ) # If you have the resources you could make the function faster by increasing the chunk number chunk_number = 25 for chunk in screen_shotter.chunk_list( unique_resolved_domains, chunk_number): try: screenshot_tups.extend(await pool.map( screen_shotter.take_screenshot, chunk)) except Exception as ee: print( f'An exception has occurred while mapping: {ee}' ) end = time.perf_counter() # There is probably an easier way to do this total = end - start_time mon, sec = divmod(total, 60) hr, mon = divmod(mon, 60) total_time = "%02d:%02d" % (mon, sec) print(f'Finished taking screenshots in {total_time} seconds') print( '[+] Note there may be leftover chrome processes you may have to kill manually\n' ) # Shodan shodanres = [] if shodan is True: import texttable tab = texttable.Texttable() header = [ 'IP address', 'Hostname', 'Org', 'Services:Ports', 'Technologies' ] tab.header(header) tab.set_cols_align(['c', 'c', 'c', 'c', 'c']) tab.set_cols_valign(['m', 'm', 'm', 'm', 'm']) tab.set_chars(['-', '|', '+', '#']) tab.set_cols_width([15, 20, 15, 15, 18]) print('\033[94m[*] Searching Shodan. \033[0m') try: for ip in host_ip: print(('\tSearching for ' + ip)) shodan = shodansearch.SearchShodan() rowdata = await shodan.search_ip(ip) await asyncio.sleep(2) tab.add_row(rowdata) printedtable = tab.draw() print(printedtable) except Exception as e: print(f'\033[93m[!] An error occurred with Shodan: {e} \033[0m') else: pass # Here we need to add explosion mode. # We have to take out the TLDs to do this. if args.dns_tld is not False: counter = 0 for word in vhost: search = googlesearch.SearchGoogle(word, limit, counter) await search.process(google_dorking) emails = await search.get_emails() hosts = await search.get_hostnames() print(emails) print(hosts) else: pass # Reporting if filename != "": try: print('\n[*] Reporting started.') db = stash.StashManager() scanboarddata = await db.getscanboarddata() latestscanresults = await db.getlatestscanresults(word) previousscanresults = await db.getlatestscanresults( word, previousday=True) latestscanchartdata = await db.latestscanchartdata(word) scanhistorydomain = await db.getscanhistorydomain(word) pluginscanstatistics = await db.getpluginscanstatistics() generator = statichtmlgenerator.HtmlGenerator(word) html_code = await generator.beginhtml() html_code += await generator.generatedashboardcode(scanboarddata) html_code += await generator.generatelatestscanresults( latestscanresults) if len(screenshot_tups) > 0: html_code += await generator.generatescreenshots( screenshot_tups) html_code += await generator.generatepreviousscanresults( previousscanresults) graph = reportgraph.GraphGenerator(word) await graph.init_db() html_code += await graph.drawlatestscangraph( word, latestscanchartdata) html_code += await graph.drawscattergraphscanhistory( word, scanhistorydomain) html_code += await generator.generatepluginscanstatistics( pluginscanstatistics) html_code += '<p><span style="color: #000000;">Report generated on ' + str( datetime.datetime.now()) + '</span></p>' html_code += ''' </body> </html> ''' except Exception as e: print(e) print( '\n\033[93m[!] An error occurred while creating the output file.\n\n \033[0m' ) sys.exit(1) html_file = open( f'{filename}.html' if '.html' not in filename else filename, 'w') html_file.write(html_code) html_file.close() print('[*] Reporting finished.') print('[*] Saving files.') try: filename = filename.rsplit('.', 1)[0] + '.xml' with open(filename, 'w+') as file: file.write( '<?xml version="1.0" encoding="UTF-8"?><theHarvester>') for x in all_emails: file.write('<email>' + x + '</email>') for x in full: host, ip = x.split(':', 1) if ':' in x else (x, '') if ip and len(ip) > 3: file.write( f'<host><ip>{ip}</ip><hostname>{host}</hostname></host>' ) else: file.write(f'<host>{host}</host>') for x in vhost: host, ip = x.split(':', 1) if ':' in x else (x, '') if ip and len(ip) > 3: file.write( f'<vhost><ip>{ip} </ip><hostname>{host}</hostname></vhost>' ) else: file.write(f'<vhost>{host}</vhost>') if shodanres != []: shodanalysis = [] for x in shodanres: res = x.split('SAPO') file.write('<shodan>') file.write('<host>' + res[0] + '</host>') file.write('<port>' + res[2] + '</port>') file.write('<banner><!--' + res[1] + '--></banner>') reg_server = re.compile('Server:.*') temp = reg_server.findall(res[1]) if temp: shodanalysis.append(res[0] + ':' + temp[0]) file.write('</shodan>') if shodanalysis: shodanalysis = sorted(set(shodanalysis)) file.write('<servers>') for x in shodanalysis: file.write('<server>' + x + '</server>') file.write('</servers>') file.write('</theHarvester>') print('[*] Files saved.') except Exception as er: print( f'\033[93m[!] An error occurred while saving the XML file: {er} \033[0m' ) print('\n\n') sys.exit(0)
], [ Embedding(max_dict_words + skip_common, 500, input_length=max_sent_words), LSTM(64) ], [ Embedding(max_dict_words + skip_common, 1000, input_length=max_sent_words), LSTM(64) ], ] batch_sizes = [256] optimizers = [keras.optimizers.Adam()] epochs = [15] model = RNNGridSearch(archs, epochs, batch_sizes, optimizers, 'sentiment1.csv') histories = model.run(x_train, y_train, x_test, y_test) result = model.best_model() hist = result.pop('history_loss') result.pop('history_score') print('Best model is: ') tbl = texttable.Texttable() tbl.set_cols_align(["c", "c"]) tbl.set_cols_valign(["c", "c"]) tbl.add_rows([['Hyperparameter', 'Best value'], *list(result.items())]) print(tbl.draw())
def report_end(bids, pyps): bidders = sorted({b['bidder_name']: True for b in bids}.keys()) for bidder in bidders: address = "" item_counts = {} won_message = [] shipping = {} bb = [b for b in bids if b['bidder_name'] == bidder] userid = bb[0]['bidder_url'].split('=')[-1] contact_url = "https://truedungeon.com/component/uddeim/?task=new&recip=%s" % ( userid) for bid in sorted(bb, key=operator.itemgetter('item')): won_message.append("Auction %s: %s" % (bid['auction'], bid['message'])) if bid['item'] in item_counts: item_counts[bid['item']] += bid['won_quantity'] else: item_counts[bid['item']] = bid['won_quantity'] if bid['address']: address = bid['address'] pyp_choices = [] pyp_won = 0 pyp_text = "" for pyp in pyps: try: if pyp['bidder_url'] == bb[0]['bidder_url']: pyp_won += pyp['won_quantity'] for k, v in pyp.items(): if k.startswith('ur'): if v is not None and v != '': pyp_choices.append(v) except Exception as e: raise e #import pdb #pdb.set_trace() #import pdb #pdb.set_trace() #1+1 if pyp_won != 0 or pyp_choices != []: pyp_text = "%d PyP selections which were:\n%s" % ( pyp_won, "\n".join(sorted(pyp_choices))) dt = texttable.Texttable() dt.set_cols_align(['r', 'l']) dt.set_cols_dtype(['t', 't']) dt.set_deco(texttable.Texttable.HEADER) line_items = [['Qty', 'Item']] for ic in sorted(item_counts.keys()): line_items.append([item_counts[ic], ic]) dt.add_rows(line_items) # DEBUG - this is a nightmare - skip it - it will be faster to just print the ~50 labels one by one. ''' shipping['Order ID (required)'] = bidder shipping['Order Date'] = datetime.datetime.today().strftime( '%m/%d/%Y' ) # Dictates insurance? shipping['Order Value'] = 0 shipping['Requested Service'] = 'Standard Shipping' # I need to ensure my addresses are formatted as: # full name\nstreet\ncity, state zip address_parts = address.split( '\n' ) shipping['Ship To - Name'] = address_parts[0] shipping['Ship To - Address 1'] = address_parts[1] city, rest = address_parts[2].split( ', ' ) state = rest[:2] zipcode = rest[3:] shipping['Ship To - State/Province'] = state shipping['Ship To - City'] = city shipping['Ship To - Postal Code'] = zipcode ''' details = "\n".join(sorted(won_message)) print "-" * 80, "\n", contact_url, "\n", "Auction Items for: %s\n\nPlease verify your address and won items below, if everything is correct no need to respond. If not, please let me know!\n\n%s\n\n%s\n\n%s\n\nAuction Breakdown:\n%s\n\n" % ( bidder, address, pyp_text, dt.draw(), details)
def main(): """Main function.""" args = parse_arguments() start = time.time() print("Expected runtime: 20 s on ATIS dataset") grammar, sents = data_load(args.grammar_f, args.sents_f) test_sents = nltk.parse.util.extract_test_sentences(sents) # Display sentences and counts in a tabulated manner using texttable # References: https://pypi.python.org/pypi/texttable t = texttable.Texttable() length = [5, 70, 5, 10, 15] t.set_cols_width(length) rows = [[ "S.No.", "test sentence", "CFG", "parse tree counts", "simple parse tree counts (doesn't compute parse trees)" ]] cyk_runtime = 0 bp_runtime = 0 for index, sent in enumerate(test_sents): tokens = sent[0] sentence = " ".join(tokens) if args.show_chart or args.show_tree: print("(", index, ")", " ".join(tokens), "\n") root = grammar.start() cyk_start = time.time() chart, table = cky_parser(tokens, grammar) cyk_end = time.time() cyk_runtime += cyk_end - cyk_start tree_counts = 0 tree_counts_simple = 0 next_row = [ str(index + 1), str(sentence), str("False"), str(tree_counts), str(tree_counts_simple) ] if cky_recognizer(chart, root, args.show_chart, args.show_tree): if args.show_chart: print_chart(chart, tokens) bp_start = time.time() tree_counts, tree_counts_simple = print_parsed_tree( table, tokens, root, args.show_tree) bp_end = time.time() bp_runtime = bp_end - bp_start next_row = [ str(index + 1), str(sentence), str("True"), str(tree_counts), str(tree_counts_simple) ] if args.show_chart or args.show_tree: dash = "-=" * 40 print(dash, '\n') rows.append(next_row) if args.show_summary: t.add_rows(rows) print(t.draw()) end = time.time() print("Total runtime: %.3f s" % (end - start)) print("CYK parser runtime: %.3f s" % cyk_runtime) print("Backpointer runtime: %.6f s" % bp_runtime)
def format_diff_for_console(cls, list_of_results): """ create a table with io performance report for console """ tab = texttable.Texttable(max_width=200) tab.set_deco(tab.HEADER | tab.VLINES | tab.BORDER) header = [ cls.fiels_and_header_dct["name"].header, cls.fiels_and_header_dct["summ"].header, ] allign = ["l", "l"] header.append("IOPS ~ Cnf% ~ Dev%") allign.extend(["r"] * len(list_of_results)) header.extend("IOPS_{0} %".format(i + 2) for i in range(len(list_of_results[1:]))) header.append("BW") allign.extend(["r"] * len(list_of_results)) header.extend("BW_{0} %".format(i + 2) for i in range(len(list_of_results[1:]))) header.append("LAT") allign.extend(["r"] * len(list_of_results)) header.extend("LAT_{0}".format(i + 2) for i in range(len(list_of_results[1:]))) tab.header(header) sep = ["-" * 3] * len(header) processed_results = map(cls.prepare_data, list_of_results) key2results = [] for res in processed_results: key2results.append( dict(((item["name"], item["summ"]), item) for item in res)) prev_k = None iops_frmt = "{0[iops]} ~ {0[conf]:>2} ~ {0[dev]:>2}" for item in processed_results[0]: if prev_k is not None: if prev_k != item["key"]: tab.add_row(sep) prev_k = item["key"] key = (item['name'], item['summ']) line = list(key) base = key2results[0][key] line.append(iops_frmt.format(base)) for test_results in key2results[1:]: val = test_results.get(key) if val is None: line.append("-") elif base['iops'] == 0: line.append("Nan") else: prc_val = {'dev': val['dev'], 'conf': val['conf']} prc_val['iops'] = int(100 * val['iops'] / base['iops']) line.append(iops_frmt.format(prc_val)) line.append(base['bw']) for test_results in key2results[1:]: val = test_results.get(key) if val is None: line.append("-") elif base['bw'] == 0: line.append("Nan") else: line.append(int(100 * val['bw'] / base['bw'])) for test_results in key2results: val = test_results.get(key) if val is None: line.append("-") else: line.append("{0[lat_50]} - {0[lat_95]}".format(val)) tab.add_row(line) tab.set_cols_align(allign) return tab.draw()
async def stats(self, ctx): try: sql = "SELECT `messages_id` FROM `messages` WHERE action=0 ORDER BY `messages_id` DESC LIMIT 1" message_count = self.cursor.execute( sql).fetchall()[0]['messages_id'] sql = "SELECT COUNT(`action`) FROM `messages` WHERE action=1" message_delete_count = self.cursor.execute( sql).fetchall()[0]['COUNT(`action`)'] sql = "SELECT COUNT(`action`) FROM `messages` WHERE action=2" message_edit_count = self.cursor.execute( sql).fetchall()[0]['COUNT(`action`)'] sql = "SELECT COUNT(`command`) FROM `command_logs`" command_count = self.cursor.execute( sql).fetchall()[0]['COUNT(`command`)'] sql = 'SELECT COUNT(`command`) FROM `command_logs` WHERE server={0}'.format( ctx.message.server.id) server_command_count = self.cursor.execute( sql).fetchall()[0]['COUNT(`command`)'] sql = "SELECT `command`, COUNT(`command`) AS magnitude FROM `command_logs` GROUP BY `command` ORDER BY magnitude DESC LIMIT 6" command_magnitude = self.cursor.execute(sql).fetchall() sql = 'SELECT `server`, `server_name`, COUNT(`command`) AS magnitude FROM `command_logs` GROUP BY `server` ORDER BY magnitude DESC LIMIT 5' server_magnitude = self.cursor.execute(sql).fetchall() sql = 'SELECT * FROM `stats`' results = self.cursor.execute(sql).fetchall() server_list = [] for shard in results: server_list.append([ shard['largest_member_server_name'], shard['largest_member_server'] ]) count = 0 counts = [] for x in server_list: count = int(x[1]) counts.append(count) max_ = int(max(counts)) max_index = int(counts.index(max_)) biggest_server_name = server_list[max_index][0] biggest_server_count = server_list[max_index][1] magnitude_table = texttable.Texttable(max_width=90) # magnitude_table.set_cols_width([90, 90]) for x in server_magnitude: magnitude_table.add_rows( [["Server", "Commands"], [x['server_name'][:25], x['magnitude']]]) magnitude_msg = magnitude_table.draw() command_table = texttable.Texttable(max_width=90) # command_table.set_cols_width([90, 90]) for x in command_magnitude: command_table.add_rows([["Command", "Count"], [x['command'], x['magnitude']]]) command_msg = command_table.draw() command_stats_msg = magnitude_msg + '\n\n' + command_msg text_channels = 0 voice_channels = 0 for shard in results: text_channels += int(shard['text_channels']) voice_channels += int(shard['voice_channels']) user_count = 0 unique_users = 0 server_count = 0 for shard in results: server_count += int(shard['servers']) user_count += int(shard['users']) unique_users += int(shard['unique_users']) seconds = time.time() - start_time m, s = divmod(seconds, 60) h, m = divmod(m, 60) d, h = divmod(h, 24) w, d = divmod(d, 7) if s != 0: uptime = '**{0}** seconds.'.format(int(s)) if m != 0: uptime = ' : **{0}** minutes : '.format(int(m)) + uptime if h != 0: uptime = ' : **{0}** hours'.format(int(h)) + uptime if d != 0: uptime = ' : **{0}** days'.format(int(d)) + uptime if w != 0: uptime = ' : **{0}** weeks {1}'.format(int(w)) + uptime if m == 0: uptime = ' ' + uptime else: uptime = uptime[2:] msg = ":bar_chart: **User/Bot Statistics**\n" msg += "> Uptime: " + uptime + "\n" msg += "> On **{0}** Servers\n".format(server_count) msg += "> **{0}** Text channels | **{1}** Voice\n".format( text_channels, voice_channels) msg += "> Serving **{0}** Users\n".format(user_count) msg += "> Unique Users: **{0}**\n".format(unique_users) msg += "> Who've messaged **{0}** times ".format(message_count) msg += "where **{0}** of them have been edited ".format( message_edit_count) msg += "and **{0}** deleted.\n".format(message_delete_count) msg += "> In total **{0}** commands have been called, __{1}__ from this server.\n".format( command_count, server_command_count) msg += ':keyboard: **Command Statistics**\n' msg += "```\n{0}```".format(command_stats_msg) msg += ':desktop: **Server Statistics**\n' msg += '> Largest Server: **{0}** (Users: **{1}**)\n'.format( biggest_server_name, biggest_server_count) msg += '> Most used on: **{0}** (Commands: **{1}**/{2})\n'.format( server_magnitude[0]['server_name'], server_magnitude[0]['magnitude'], command_count) # msg += '> Server with most messages: *{0}* (Messages: **{1}/{2}**)' await self.bot.say(msg) except Exception as e: print(e)
def pretty_print_table(group_name, table): print('\n\n*** {} ***'.format(group_name)) tt = texttable.Texttable() tt.add_rows(table) print(tt.draw())
def to_tables(self): results = [] table = texttable.Texttable(max_width=0) table.set_deco(texttable.Texttable.HEADER) rows = [] rows.append(['Resource', 'Count']) all_collections = self.pod.list_collections() rows.append(['Collections', len(all_collections)]) documents = [] for collection in all_collections: documents += collection.list_docs() rows.append(['Documents', len(documents)]) locales = self.pod.list_locales() rows.append(['Locales', len(locales)]) if self.pod.use_reroute: self.pod.router.use_simple() self.pod.router.add_all() rows.append(['Routes', len(self.pod.router.routes)]) else: routes = self.pod.routes rows.append(['Routes', len(routes.list_concrete_paths())]) template = self.pod.catalogs.get_template() rows.append(['Messages', len(template)]) table.add_rows(rows) content = table.draw() results.append(content) if self.full: table = texttable.Texttable(max_width=0) table.set_deco(texttable.Texttable.HEADER) rows = [] rows.append(['File type', 'Count']) exts_and_counts = self.get_num_files_per_type() exts_and_counts = sorted(exts_and_counts, key=lambda message: -message.count) for message in exts_and_counts: ext = message.ext or '.html' rows.append([ext, message.count]) table.add_rows(rows) content = table.draw() results.append(content) table = texttable.Texttable(max_width=0) table.set_deco(texttable.Texttable.HEADER) catalogs = sorted(self.pod.catalogs, key=str) rows = [] rows.append(['Translations ({})'.format(len(catalogs)), 'Messages']) for catalog in catalogs: num_messages = len(catalog) untranslated_messages = catalog.list_untranslated() translated_messages = num_messages - len(untranslated_messages) label = '{} / {}'.format(translated_messages, num_messages) rows.append([str(catalog.locale), label]) table.add_rows(rows) if catalogs: content = table.draw() results.append(content) return results
import json import sys import texttable import operator input_file = sys.argv[1] output_file = sys.argv[2] with open(input_file) as input: full_dict = json.load(input) #DOMAIN TABLE------------------------------------------------------------------------- domain_table = texttable.Texttable(375) domain_table.set_cols_align( ["l", "c", "c", "c", "c", "c", "c", "c", "c", "c", "c", "c", "c"]) domain_table.set_cols_valign( ["t", "t", "t", "t", "t", "t", "t", "t", "t", "t", "t", "t", "t"]) domain_table.add_row([ "Domain", "Scan Time", "IPv4\nAddresses", "IPv6\nAddresses", "HTTP\nServer", "Insecure\nHTTP", "Redirects", "Hosts", "TLS\nVersions", "Root CA", "RDNS Names", "RTT Range", "Geo\nLocations" ]) for domain in full_dict: row = [] row.append(domain) for i in full_dict[domain]:
timings = {} # Receive results bss = BSSeval([ 'mir_eval_images', 'mir_eval_images_noperm', 'bss_eval_matlab_images', 'mir_eval_sources', 'bss_eval_matlab_sources', ]) results = bss.evaluate(est_audio, src_audio, src_rate) # Try for a readable output try: import texttable as txtbl table = txtbl.Texttable() table.set_cols_dtype([ 't', # method used 'f', # SDR 'f', # SIR 'f', # ISR 'f' ]) # SAR table.set_cols_align(["c", "c", "c", "c", "c"]) table.header(["Method", "SDR", "SIR", "ISR", "SAR"]) for result in results: method = result['Method'] try: sources = len(result['SDR']) except: sources = 1
print('ROI crop selected') (input_dir,output_dir)=in_out() #ROI class call Crop(input_dir, output_dir).ROI() #Spot crop selection elif select_def=='S': print('Spot crop selected') (input_dir,output_dir)=in_out() #Spot class call Crop(input_dir, output_dir).Spot() #Help selection elif select_def=='H': print('Help selected') table = tt.Texttable() table.header(["Instructions"]) table.add_row(["The present code allows to manage MIRAX files of histological samples. Two possible solutions were implemented:\n\n 1) Select a specific region of interest(ROI) from the whole slide image (WSI) and then cut it in sub-images\n 2) Find the spots in a TMA sample and cut them in sigle images"]) table.add_row(["Prerequisites:\n1) Matlab\n - MATLAB Engine API for Python\n2) Python3 \n - openslide,sys, os, time, csv, pandas, numpy, texttable"]) table.add_row(["How to run:\n - Open the terminal in the code folder and write:\n\n\tPython CropCode.py\n\nThen follow simply answer to the question!"]) table.add_row(["ROI: The ROI option allows to select a specif area and then cut it in sub-images. This can be useful in case it is necessary to select only a portion of the WSI. The following cut is implemented in order to work with the image that problably is in the order of ~GB.\n"\ "\nThe program will ask you:\n - Input directory\n - Output directory\n - If you want only the files with the dimension of the ROI or if you want to cut it directly with this program\n - Dimesion of the square used for cut the ROI, in case your answer was 'y' (yes) to the previous question\n\nA GUI is used to select the ROI.\n\nSuggestion: check the size in pixel of the WSI and then choose the dimension of the square according to it."\ "It is also better select the ROI considering the total width and white contours over and under of the WSI."]) table.add_row(["Spot: The Spot option allows to reconize the spots in a TMA and cut them in order to obtain sigle image of spot per TMAs. If a document with the ID patients is attached it is possible to save each spot with the proper ID and in the corresponding folder per patient."\ " if this is not the casey, the code will name the spots with a number from the bottom left to the upper right.\nThe program will ask you:\n\n - Input directory\n - Output directory\n - Number of row in the TMA (remember to choose the maximum value considering all the TMAs)\n - Number of column in the TMA (remember to choose the maximum value considering all the TMAs)\n - Dimension of the radius in case the detected matrix of the spots is not correct"\ "\n\nA GUI is used to check if the program have found all the spots and, in case, allows to add or remove incorrect elements.\n\nSuggestion: Considering the IDpatient file, take a look to the example given with this code!"]) table.set_cols_width([100]) print(table.draw()) #In case of unknown character else: print('ERROR!Command not reconized!')
def start(): parser = argparse.ArgumentParser( description= 'theHarvester is used to gather open source intelligence on a company or domain.' ) parser.add_argument('-d', '--domain', help='Company or domain to search', required=True) parser.add_argument('-l', '--limit', help='Limit the number of search results, default=500', default=500, type=int) parser.add_argument('-S', '--start', help='Start with result number X, default=0', default=0, type=int) parser.add_argument('-g', '--google-dork', help='Use Google Dorks for Google search', default=False, action='store_true') parser.add_argument( '-p', '--port-scan', help= 'Scan discovered hosts and check for Takeovers (21,22,80,443,8080) default=False, params=True', default=False) parser.add_argument('-s', '--shodan', help='Use Shodan to query discovered hosts', default=False, action='store_true') parser.add_argument( '-v', '--virtual-host', help= 'Verify host name via DNS resolution and search for virtual hosts params=basic, default=False', default=False) parser.add_argument('-e', '--dns-server', help='Specify a DNS server to use for lookup') parser.add_argument('-c', '--dns-brute', help='Perform a DNS brute force on a domain', default=False, action='store_true') parser.add_argument( '-n', '--dns-lookup', help='Enable a DNS server lookup, default=False, params=True', default=False) parser.add_argument( '-t', '--dns-tld', help='Perform a DNS TLD expansion discovery, default False', default=False) parser.add_argument('-f', '--filename', help='Save the results to an HTML and/or XML file', default='', type=str) parser.add_argument( '-b', '--source', help='''Source: baidu, bing, bingapi, censys, crtsh, cymon, dogpile, duckduckgo, google, googleCSE, google-certificates, hunter, linkedin, netcraft, pgp, securityTrails, threatcrowd, trello, twitter, vhost, virustotal, yahoo, all''') args = parser.parse_args() try: db = stash.stash_manager() db.do_init() except Exception: pass all_emails = [] all_hosts = [] all_ip = [] dnsbrute = args.dns_brute dnslookup = args.dns_lookup dnsserver = args.dns_server dnstld = args.dns_tld filename = args.filename full = [] google_dorking = args.google_dork host_ip = [] limit = args.limit ports_scanning = args.port_scan shodan = args.shodan start = args.start takeover_check = False trello_info = ([], False) vhost = [] virtual = args.virtual_host word = args.domain if args.source is not None: engines = set(args.source.split(',')) if set(engines).issubset(Core.get_supportedengines()): print(f'\033[94m[*] Target: {word} \n \033[0m') for engineitem in engines: if engineitem == 'baidu': print('\033[94m[*] Searching Baidu. \033[0m') try: search = baidusearch.SearchBaidu(word, limit) search.process() all_emails = filter(search.get_emails()) hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'baidu') db.store_all(word, all_emails, 'email', 'baidu') except Exception: pass elif engineitem == 'bing' or engineitem == 'bingapi': print('\033[94m[*] Searching Bing. \033[0m') try: search = bingsearch.SearchBing(word, limit, start) bingapi = '' if engineitem == 'bingapi': bingapi += 'yes' else: bingapi += 'no' search.process(bingapi) all_emails = filter(search.get_emails()) hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'email', 'bing') db.store_all(word, all_hosts, 'host', 'bing') except Exception as e: if isinstance(e, MissingKey): print(e) else: pass elif engineitem == 'censys': print('\033[94m[*] Searching Censys. \033[0m') from discovery import censys # Import locally or won't work search = censys.SearchCensys(word, limit) search.process() all_ip = search.get_ipaddresses() hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'censys') db.store_all(word, all_ip, 'ip', 'censys') elif engineitem == 'crtsh': print('\033[94m[*] Searching CRT.sh. \033[0m') search = crtsh.search_crtsh(word) search.process() hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'CRTsh') elif engineitem == 'cymon': print('\033[94m[*] Searching Cymon. \033[0m') from discovery import cymon # Import locally or won't work. search = cymon.search_cymon(word) search.process() all_ip = search.get_ipaddresses() db = stash.stash_manager() db.store_all(word, all_ip, 'ip', 'cymon') elif engineitem == 'dogpile': try: print('\033[94m[*] Searching Dogpile. \033[0m') search = dogpilesearch.SearchDogpile(word, limit) search.process() emails = filter(search.get_emails()) hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) all_emails.extend(emails) db = stash.stash_manager() db.store_all(word, all_hosts, 'email', 'dogpile') db.store_all(word, all_hosts, 'host', 'dogpile') except Exception as e: print( f'\033[93m[!] A error occurred in Dogpile: {e} \033[0m' ) elif engineitem == 'duckduckgo': print('\033[94m[*] Searching DuckDuckGo. \033[0m') from discovery import duckduckgosearch search = duckduckgosearch.SearchDuckDuckGo(word, limit) search.process() emails = filter(search.get_emails()) hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) all_emails.extend(emails) db = stash.stash_manager() db.store_all(word, all_hosts, 'email', 'duckduckgo') db.store_all(word, all_hosts, 'host', 'duckduckgo') elif engineitem == 'google': print('\033[94m[*] Searching Google. \033[0m') search = googlesearch.search_google(word, limit, start) search.process(google_dorking) emails = filter(search.get_emails()) all_emails.extend(emails) hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'google') db.store_all(word, all_emails, 'email', 'google') elif engineitem == 'googleCSE': print( '\033[94m[*] Searching Google Custom Search. \033[0m') try: search = googleCSE.SearchGoogleCSE(word, limit, start) search.process() search.store_results() all_emails = filter(search.get_emails()) db = stash.stash_manager() hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) db.store_all(word, all_hosts, 'email', 'googleCSE') db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'googleCSE') except Exception as e: if isinstance(e, MissingKey): print(e) else: pass elif engineitem == 'google-certificates': print( '\033[94m[*] Searching Google Certificate transparency report. \033[0m' ) search = googlecertificates.SearchGoogleCertificates( word, limit, start) search.process() hosts = filter(search.get_domains()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'google-certificates') elif engineitem == 'hunter': print('\033[94m[*] Searching Hunter. \033[0m') from discovery import huntersearch # Import locally or won't work. try: search = huntersearch.SearchHunter(word, limit, start) search.process() emails = filter(search.get_emails()) all_emails.extend(emails) hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'hunter') db.store_all(word, all_emails, 'email', 'hunter') except Exception as e: if isinstance(e, MissingKey): print(e) else: pass elif engineitem == 'linkedin': print('\033[94m[*] Searching Linkedin. \033[0m') search = linkedinsearch.SearchLinkedin(word, limit) search.process() people = search.get_people() db = stash.stash_manager() db.store_all(word, people, 'name', 'linkedin') if len(people) == 0: print('\n[*] No users found Linkedin.\n\n') else: print(f'\n[*] Users found: {len(people)}') print('---------------------') for user in sorted(list(set(people))): print(user) sys.exit(0) elif engineitem == 'netcraft': print('\033[94m[*] Searching Netcraft. \033[0m') search = netcraft.SearchNetcraft(word) search.process() hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'netcraft') elif engineitem == 'pgp': print('\033[94m[*] Searching PGP key server. \033[0m') try: search = pgpsearch.SearchPgp(word) search.process() all_emails = filter(search.get_emails()) hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'pgp') db.store_all(word, all_emails, 'email', 'pgp') except Exception: pass elif engineitem == 'securityTrails': print('\033[94m[*] Searching SecurityTrails. \033[0m') from discovery import securitytrailssearch try: search = securitytrailssearch.search_securitytrail( word) search.process() hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, hosts, 'host', 'securityTrails') ips = search.get_ips() all_ip.extend(ips) db = stash.stash_manager() db.store_all(word, ips, 'ip', 'securityTrails') except Exception as e: if isinstance(e, MissingKey): print(e) else: pass elif engineitem == 'threatcrowd': print('\033[94m[*] Searching Threatcrowd. \033[0m') try: search = threatcrowd.search_threatcrowd(word) search.process() hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'threatcrowd') except Exception: pass elif engineitem == 'trello': print('\033[94m[*] Searching Trello. \033[0m') from discovery import trello # Import locally or won't work. search = trello.search_trello(word, limit) search.process() emails = filter(search.get_emails()) all_emails.extend(emails) info = search.get_urls() hosts = filter(info[0]) trello_info = (info[1], True) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, hosts, 'host', 'trello') db.store_all(word, emails, 'email', 'trello') elif engineitem == 'twitter': print('\033[94m[*] Searching Twitter. \033[0m') search = twittersearch.search_twitter(word, limit) search.process() people = search.get_people() db = stash.stash_manager() db.store_all(word, people, 'name', 'twitter') if len(people) == 0: print('\n[*] No users found.\n\n') else: print('\n[*] Users found: ' + str(len(people))) print('---------------------') for user in sorted(list(set(people))): print(user) elif engineitem == 'virustotal': print('\033[94m[*] Searching VirusTotal. \033[0m') search = virustotal.search_virustotal(word) search.process() hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'virustotal') elif engineitem == 'yahoo': print('\033[94m[*] Searching Yahoo. \033[0m') search = yahoosearch.search_yahoo(word, limit) search.process() hosts = search.get_hostnames() emails = search.get_emails() all_hosts.extend(filter(hosts)) all_emails.extend(filter(emails)) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'yahoo') db.store_all(word, all_emails, 'email', 'yahoo') elif engineitem == 'all': print('\033[94m[*] Running a full harvest. \033[0m') all_emails = [] all_hosts = [] try: print('\033[94m\n[*] Searching Baidu. \033[0m') search = baidusearch.SearchBaidu(word, limit) search.process() all_emails = filter(search.get_emails()) hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'baidu') db.store_all(word, all_emails, 'email', 'baidu') except Exception: pass print('\033[94m\n[*] Searching Bing. \033[0m') bingapi = 'no' search = bingsearch.SearchBing(word, limit, start) search.process(bingapi) emails = filter(search.get_emails()) hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'bing') all_emails.extend(emails) all_emails = sorted(set(all_emails)) db.store_all(word, all_emails, 'email', 'bing') print('\033[94m\n[*] Searching Censys. \033[0m') from discovery import censys search = censys.SearchCensys(word, limit) search.process() ips = search.get_ipaddresses() setips = set(ips) uniqueips = list(setips) # Remove duplicates. all_ip.extend(uniqueips) hosts = filter(search.get_hostnames()) sethosts = set(hosts) uniquehosts = list(sethosts) # Remove duplicates. all_hosts.extend(uniquehosts) db = stash.stash_manager() db.store_all(word, uniquehosts, 'host', 'censys') db.store_all(word, uniqueips, 'ip', 'censys') print('\033[94m\n[*] Searching CRT.sh. \033[0m') search = crtsh.search_crtsh(word) search.process() hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'CRTsh') # cymon print('\033[94m\n[*] Searching Cymon. \033[0m') from discovery import cymon # Import locally or won't work. search = cymon.search_cymon(word) search.process() all_ip = search.get_ipaddresses() db = stash.stash_manager() db.store_all(word, all_ip, 'ip', 'cymon') print('\033[94m\n[*] Searching Dogpile. \033[0m') search = dogpilesearch.SearchDogpile(word, limit) search.process() emails = filter(search.get_emails()) hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) all_emails.extend(emails) db = stash.stash_manager() db.store_all(word, all_hosts, 'email', 'dogpile') db.store_all(word, all_hosts, 'host', 'dogpile') print('\033[94m\n[*] Searching DuckDuckGo. \033[0m') from discovery import duckduckgosearch search = duckduckgosearch.SearchDuckDuckGo(word, limit) search.process() emails = filter(search.get_emails()) hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) all_emails.extend(emails) db = stash.stash_manager() db.store_all(word, all_hosts, 'email', 'duckduckgo') db.store_all(word, all_hosts, 'host', 'duckduckgo') print('\033[94m\n[*] Searching Google. \033[0m') search = googlesearch.search_google(word, limit, start) search.process(google_dorking) emails = filter(search.get_emails()) hosts = filter(search.get_hostnames()) all_emails.extend(emails) db = stash.stash_manager() db.store_all(word, all_emails, 'email', 'google') all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'google') print( '\033[94m\n[*] Searching Google Certificate transparency report. \033[0m' ) search = googlecertificates.SearchGoogleCertificates( word, limit, start) search.process() domains = filter(search.get_domains()) all_hosts.extend(domains) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'google-certificates') print('\033[94m\n[*] Searching Hunter. \033[0m') from discovery import huntersearch # Import locally. try: search = huntersearch.SearchHunter(word, limit, start) search.process() emails = filter(search.get_emails()) hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, hosts, 'host', 'hunter') all_emails.extend(emails) all_emails = sorted(set(all_emails)) db.store_all(word, all_emails, 'email', 'hunter') except Exception as e: if isinstance(e, MissingKey): print(e) else: pass print('\033[94m\n[*] Searching Linkedin. \033[0m') search = linkedinsearch.SearchLinkedin(word, limit) search.process() people = search.get_people() db = stash.stash_manager() db.store_all(word, people, 'name', 'linkedin') if len(people) == 0: print('\n[*] No users found.\n\n') else: print('\n[*] Users found: ' + str(len(people))) print('---------------------') for user in sorted(list(set(people))): print(user) print('\033[94m\n[*] Searching Netcraft. \033[0m') search = netcraft.SearchNetcraft(word) search.process() hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'netcraft') print('\033[94m\n[*] Searching PGP key server. \033[0m') try: search = pgpsearch.SearchPgp(word) search.process() emails = filter(search.get_emails()) hosts = filter(search.get_hostnames()) sethosts = set(hosts) uniquehosts = list(sethosts) # Remove duplicates. all_hosts.extend(uniquehosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'PGP') all_emails.extend(emails) db = stash.stash_manager() db.store_all(word, all_emails, 'email', 'PGP') except Exception: pass print('\033[94m\n[*] Searching Threatcrowd. \033[0m') try: search = threatcrowd.search_threatcrowd(word) search.process() hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'threatcrowd') except Exception: pass print('\033[94m\n[*] Searching Trello. \033[0m') from discovery import trello # Import locally or won't work. search = trello.search_trello(word, limit) search.process() emails = filter(search.get_emails()) all_emails.extend(emails) info = search.get_urls() hosts = filter(info[0]) trello_info = (info[1], True) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, hosts, 'host', 'trello') db.store_all(word, emails, 'email', 'trello') try: print('\033[94m\n[*] Searching Twitter. \033[0m') search = twittersearch.search_twitter(word, limit) search.process() people = search.get_people() db = stash.stash_manager() db.store_all(word, people, 'name', 'twitter') print('\nUsers from Twitter:') print('-------------------') for user in people: print(user) except Exception: pass print('\n[*] Virtual hosts:') print('------------------') for l in host_ip: search = bingsearch.SearchBing(l, limit, start) search.process_vhost() res = search.get_allhostnames() for x in res: x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x) x = re.sub('<', '', x) x = re.sub('>', '', x) print((l + '\t' + x)) vhost.append(l + ':' + x) full.append(l + ':' + x) vhost = sorted(set(vhost)) print('\033[94m\n[*] Searching VirusTotal. \033[0m') search = virustotal.search_virustotal(word) search.process() hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'virustotal') print('\033[94m\n[*] Searching Yahoo. \033[0m') search = yahoosearch.search_yahoo(word, limit) search.process() hosts = search.get_hostnames() emails = search.get_emails() all_hosts.extend(filter(hosts)) all_emails.extend(filter(emails)) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'yahoo') db.store_all(word, all_emails, 'email', 'yahoo') else: print('\033[93m[!] Invalid source.\n\n \033[0m') sys.exit(1) # Sanity check to see if all_emails and all_hosts are defined. try: all_emails except NameError: print( '\n\n\033[93m[!] No emails found because all_emails is not defined.\n\n \033[0m' ) sys.exit(1) try: all_hosts except NameError: print( '\n\n\033[93m[!] No hosts found because all_hosts is not defined.\n\n \033[0m' ) sys.exit(1) # Results if len(all_ip) == 0: print('\n[*] No IPs found.') else: print('\n[*] IPs found: ' + str(len(all_ip))) print('-------------------') ips = sorted( ipaddress.ip_address(line.strip()) for line in set(all_ip)) print('\n'.join(map(str, ips))) if len(all_emails) == 0: print('\n[*] No emails found.') else: print('\n[*] Emails found: ' + str(len(all_emails))) print('----------------------') print(('\n'.join(sorted(list(set(all_emails)))))) if len(all_hosts) == 0: print('\n[*] No hosts found.\n\n') else: print('\n[*] Hosts found: ' + str(len(all_hosts))) print('---------------------') all_hosts = sorted(list(set(all_hosts))) full_host = hostchecker.Checker(all_hosts) full = full_host.check() for host in full: ip = host.split(':')[1] print(host) if ip != 'empty': if host_ip.count(ip.lower()): pass else: host_ip.append(ip.lower()) db = stash.stash_manager() db.store_all(word, host_ip, 'ip', 'DNS-resolver') if trello_info[1] is True: trello_urls = trello_info[0] if trello_urls == []: print('\n[*] No URLs found.') else: total = len(trello_urls) print('\n[*] URLs found: ' + str(total)) print('--------------------') for url in sorted(list(set(trello_urls))): print(url) # DNS brute force dnsres = [] if dnsbrute is True: print('\n[*] Starting DNS brute force.') a = dnssearch.dns_force(word, dnsserver, verbose=True) res = a.process() print('\n[*] Hosts found after DNS brute force:') print('-------------------------------------') for y in res: print(y) dnsres.append(y.split(':')[0]) if y not in full: full.append(y) db = stash.stash_manager() db.store_all(word, dnsres, 'host', 'dns_bruteforce') # Port scanning if ports_scanning is True: print('\n\n[*] Scanning ports (active).\n') for x in full: host = x.split(':')[1] domain = x.split(':')[0] if host != 'empty': print(('[*] Scanning ' + host)) ports = [21, 22, 80, 443, 8080] try: scan = port_scanner.PortScan(host, ports) openports = scan.process() if len(openports) > 1: print(('\t[*] Detected open ports: ' + ','.join(str(e) for e in openports))) takeover_check = 'True' if takeover_check == 'True': if len(openports) > 0: search_take = takeover.take_over(domain) search_take.process() except Exception as e: print(e) # DNS reverse lookup dnsrev = [] if dnslookup is True: print('\n[*] Starting active queries.') analyzed_ranges = [] for x in host_ip: print(x) ip = x.split(':')[0] range = ip.split('.') range[3] = '0/24' s = '.' range = s.join(range) if not analyzed_ranges.count(range): print('[*] Performing reverse lookup in ' + range) a = dnssearch.dns_reverse(range, True) a.list() res = a.process() analyzed_ranges.append(range) else: continue for x in res: if x.count(word): dnsrev.append(x) if x not in full: full.append(x) print('[*] Hosts found after reverse lookup (in target domain):') print('--------------------------------------------------------') for xh in dnsrev: print(xh) # DNS TLD expansion dnstldres = [] if dnstld is True: print('[*] Starting DNS TLD expansion.') a = dnssearch.dns_tld(word, dnsserver, verbose=True) res = a.process() print('\n[*] Hosts found after DNS TLD expansion:') print('----------------------------------------') for y in res: print(y) dnstldres.append(y) if y not in full: full.append(y) # Virtual hosts search if virtual == 'basic': print('\n[*] Virtual hosts:') print('------------------') for l in host_ip: search = bingsearch.SearchBing(l, limit, start) search.process_vhost() res = search.get_allhostnames() for x in res: x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x) x = re.sub('<', '', x) x = re.sub('>', '', x) print((l + '\t' + x)) vhost.append(l + ':' + x) full.append(l + ':' + x) vhost = sorted(set(vhost)) else: pass # Shodan shodanres = [] if shodan is True: import texttable tab = texttable.Texttable() header = [ 'IP address', 'Hostname', 'Org', 'Services:Ports', 'Technologies' ] tab.header(header) tab.set_cols_align(['c', 'c', 'c', 'c', 'c']) tab.set_cols_valign(['m', 'm', 'm', 'm', 'm']) tab.set_chars(['-', '|', '+', '#']) tab.set_cols_width([15, 20, 15, 15, 18]) host_ip = list(set(host_ip)) print('\033[94m[*] Searching Shodan. \033[0m') try: for ip in host_ip: print(('\tSearching for ' + ip)) shodan = shodansearch.search_shodan() rowdata = shodan.search_ip(ip) time.sleep(2) tab.add_row(rowdata) printedtable = tab.draw() print(printedtable) except Exception as e: print( f'\033[93m[!] Error occurred in the Shodan search module: {e} \033[0m' ) else: pass # Here we need to add explosion mode. # We have to take out the TLDs to do this. recursion = None if recursion: start = 0 for word in vhost: search = googlesearch.search_google(word, limit, start) search.process(google_dorking) emails = search.get_emails() hosts = search.get_hostnames() print(emails) print(hosts) else: pass # Reporting if filename != "": try: print('\nNEW REPORTING BEGINS.') db = stash.stash_manager() scanboarddata = db.getscanboarddata() latestscanresults = db.getlatestscanresults(word) previousscanresults = db.getlatestscanresults(word, previousday=True) latestscanchartdata = db.latestscanchartdata(word) scanhistorydomain = db.getscanhistorydomain(word) pluginscanstatistics = db.getpluginscanstatistics() generator = statichtmlgenerator.htmlgenerator(word) HTMLcode = generator.beginhtml() HTMLcode += generator.generatelatestscanresults(latestscanresults) HTMLcode += generator.generatepreviousscanresults( previousscanresults) graph = reportgraph.GraphGenerator(word) HTMLcode += graph.drawlatestscangraph(word, latestscanchartdata) HTMLcode += graph.drawscattergraphscanhistory( word, scanhistorydomain) HTMLcode += generator.generatepluginscanstatistics( pluginscanstatistics) HTMLcode += generator.generatedashboardcode(scanboarddata) HTMLcode += '<p><span style="color: #000000;">Report generated on ' + str( datetime.datetime.now()) + '</span></p>' HTMLcode += ''' </body> </html> ''' Html_file = open('report.html', 'w') Html_file.write(HTMLcode) Html_file.close() print('NEW REPORTING FINISHED!') print('[*] Saving files.') html = htmlExport.htmlExport(all_emails, full, vhost, dnsres, dnsrev, filename, word, shodanres, dnstldres) save = html.writehtml() except Exception as e: print(e) print('\n\033[93m[!] An error occurred creating the file.\033[0m') try: filename = filename.split('.')[0] + '.xml' file = open(filename, 'w') file.write('<?xml version="1.0" encoding="UTF-8"?><theHarvester>') for x in all_emails: file.write('<email>' + x + '</email>') for x in full: x = x.split(':') if len(x) == 2: file.write('<host>' + '<ip>' + x[1] + '</ip><hostname>' + x[0] + '</hostname>' + '</host>') else: file.write('<host>' + x + '</host>') for x in vhost: x = x.split(':') if len(x) == 2: file.write('<vhost>' + '<ip>' + x[1] + '</ip><hostname>' + x[0] + '</hostname>' + '</vhost>') else: file.write('<vhost>' + x + '</vhost>') if shodanres != []: shodanalysis = [] for x in shodanres: res = x.split('SAPO') file.write('<shodan>') file.write('<host>' + res[0] + '</host>') file.write('<port>' + res[2] + '</port>') file.write('<banner><!--' + res[1] + '--></banner>') reg_server = re.compile('Server:.*') temp = reg_server.findall(res[1]) if temp != []: shodanalysis.append(res[0] + ':' + temp[0]) file.write('</shodan>') if shodanalysis != []: shodanalysis = sorted(set(shodanalysis)) file.write('<servers>') for x in shodanalysis: file.write('<server>' + x + '</server>') file.write('</servers>') file.write('</theHarvester>') file.flush() file.close() print('[*] Files saved.') except Exception as er: print( f'\033[93m[!] An error occurred saving XML file: {er} \033[0m') print('\n\n') sys.exit(0)
def list_all(self, jails): """List all jails.""" self.full = True if self.plugin else self.full jail_list = [] for jail in jails: mountpoint = jail.properties["mountpoint"].value try: conf = iocage_lib.ioc_json.IOCJson(mountpoint).json_get_value( 'all') state = '' except (Exception, SystemExit): # Jail is corrupt, we want all the keys to exist. # So we will take the defaults and let the user # know that they are not correct. def_props = iocage_lib.ioc_json.IOCJson().json_get_value( 'all', default=True) conf = {x: 'N/A' for x in def_props} conf['host_hostuuid'] = \ f'{jail.name.split("/")[-1]}' conf['release'] = 'N/A' state = 'CORRUPT' jid = '-' if self.basejail_only and conf.get('basejail', 'no') != 'yes': continue uuid_full = conf["host_hostuuid"] uuid = uuid_full if not self.full: # We only want to show the first 8 characters of a UUID, # if it's not a UUID, we will show the whole name no matter # what. try: uuid = str(_uuid.UUID(uuid, version=4))[:8] except ValueError: # We leave the "uuid" untouched, as it's not a valid # UUID, but instead a named jail. pass full_ip4 = conf["ip4_addr"] ip6 = conf["ip6_addr"] try: short_ip4 = ",".join([ item.split("|")[1].split("/")[0] for item in full_ip4.split(",") ]) except IndexError: short_ip4 = full_ip4 if full_ip4 != "none" else "-" boot = conf["boot"] jail_type = conf["type"] full_release = conf["release"] basejail = conf.get('basejail', 'no') if "HBSD" in full_release: full_release = re.sub(r"\W\w.", "-", full_release) full_release = full_release.replace("--SD", "-STABLE-HBSD") short_release = full_release.rstrip("-HBSD") else: short_release = "-".join(full_release.rsplit("-")[:2]) if full_ip4 == "none": full_ip4 = "-" if ip6 == "none": ip6 = "-" # Will be set already by a corrupt jail status = False if state != 'CORRUPT': status, jid = self.list_get_jid(uuid_full) if status: state = "up" else: state = "down" if conf["type"] == "template": template = "-" else: jail_root = self.zfs.get_dataset(f"{jail.name}/root") _origin_property = jail_root.properties["origin"] if _origin_property and _origin_property.value != "": template = jail_root.properties["origin"].value template = template.rsplit("/root@", 1)[0].rsplit("/", 1)[-1] else: template = "-" if "release" in template.lower() or "stable" in template.lower(): template = "-" if conf["dhcp"] == "on" and status and os.geteuid() == 0: interface = conf["interfaces"].split(",")[0].split(":")[0] if interface == "vnet0": # Inside jails they are epairNb interface = f"{interface.replace('vnet', 'epair')}b" short_ip4 = "DHCP" full_ip4_cmd = [ "jexec", f"ioc-{uuid_full.replace('.', '_')}", "ifconfig", interface, "inet" ] try: out = su.check_output(full_ip4_cmd) full_ip4 = f'{interface}|' \ f'{out.splitlines()[2].split()[1].decode()}' except (su.CalledProcessError, IndexError) as e: short_ip4 += '(Network Issue)' if isinstance(e, su.CalledProcessError): full_ip4 = f'DHCP - Network Issue: {e}' else: full_ip4 = f'DHCP - Failed Parsing: {e}' elif conf["dhcp"] == "on" and not status: short_ip4 = "DHCP" full_ip4 = "DHCP (not running)" elif conf["dhcp"] == "on" and os.geteuid() != 0: short_ip4 = "DHCP" full_ip4 = "DHCP (running -- address requires root)" # Append the JID and the NAME to the table if self.full and self.plugin: if jail_type != "plugin" and jail_type != "pluginv2": # We only want plugin type jails to be apart of the # list continue try: with open(f"{mountpoint}/plugin/ui.json", "r") as u: ip = full_ip4.split("|", 1) ip = ip[1] if len(ip) != 1 else ip[0] ip = ip.split("/", 1)[0] if "DHCP" not in full_ip4 \ else "DHCP" ui_data = json.load(u) admin_portal = ui_data["adminportal"] admin_portal = admin_portal.replace("%%IP%%", ip) try: ph = ui_data["adminportal_placeholders"].items() for placeholder, prop in ph: admin_portal = admin_portal.replace( placeholder, iocage_lib.ioc_json.IOCJson( mountpoint).json_plugin_get_value( prop.split("."))) except KeyError: pass except iocage_lib.ioc_exceptions.CommandNeedsRoot: admin_portal = "Admin Portal requires root" except iocage_lib.ioc_exceptions.CommandFailed as e: admin_portal = b' '.join(e.message).decode() except FileNotFoundError: # They just didn't set a admin portal. admin_portal = "-" jail_list.append([ jid, uuid, boot, state, jail_type, full_release, full_ip4, ip6, template, admin_portal ]) elif self.full: jail_list.append([ jid, uuid, boot, state, jail_type, full_release, full_ip4, ip6, template, basejail ]) else: jail_list.append([jid, uuid, state, short_release, short_ip4]) list_type = "list_full" if self.full else "list_short" sort = iocage_lib.ioc_common.ioc_sort(list_type, self.sort, data=jail_list) jail_list.sort(key=sort) # return the list... if not self.header: flat_jail = [j for j in jail_list] return flat_jail # Prints the table table = texttable.Texttable(max_width=0) if self.full: if self.plugin: table.set_cols_dtype( ["t", "t", "t", "t", "t", "t", "t", "t", "t", "t"]) jail_list.insert(0, [ "JID", "NAME", "BOOT", "STATE", "TYPE", "RELEASE", "IP4", "IP6", "TEMPLATE", "PORTAL" ]) else: # We get an infinite float otherwise. table.set_cols_dtype( ["t", "t", "t", "t", "t", "t", "t", "t", "t", "t"]) jail_list.insert(0, [ "JID", "NAME", "BOOT", "STATE", "TYPE", "RELEASE", "IP4", "IP6", "TEMPLATE", 'BASEJAIL' ]) else: # We get an infinite float otherwise. table.set_cols_dtype(["t", "t", "t", "t", "t"]) jail_list.insert(0, ["JID", "NAME", "STATE", "RELEASE", "IP4"]) table.add_rows(jail_list) return table.draw()
def cli(prop, _type, _pool, jail, recursive, header, plugin, force): """Get a list of jails and print the property.""" table = texttable.Texttable(max_width=0) if _type: # Confusing I know. jail = prop prop = _type elif _pool: pool = ioc.IOCage(skip_jails=True).get('', pool=True) ioc_common.logit({'level': 'INFO', 'message': pool}) exit() else: if not jail and not recursive: ioc_common.logit({ 'level': 'EXCEPTION', 'message': 'You must specify a jail!' }) if _type == 'all' and recursive: # TODO: Port this back ioc_common.logit({ 'level': 'EXCEPTION', 'message': 'You cannot use --all (-a) and --recursive (-r) ' 'together. ' }) if _type == 'all' and not jail: ioc_common.logit({ 'level': 'EXCEPTION', 'message': 'Please specify a jail name when using -a flag.' }) if not recursive: if prop == 'state' or _type == 'state': state = ioc.IOCage(jail=jail).get(prop) ioc_common.logit({'level': 'INFO', 'message': state}) elif prop == 'jid' or _type == 'jid': jid = ioc.IOCage(jail=jail).list('jid', uuid=jail)[1] ioc_common.logit({'level': 'INFO', 'message': jid}) elif plugin: _plugin = ioc.IOCage(jail=jail, skip_jails=True).get(prop, plugin=True, start_jail=force) ioc_common.logit({'level': 'INFO', 'message': _plugin}) elif prop == 'all': props = ioc.IOCage(jail=jail, skip_jails=True).get(prop) for p, v in props.items(): ioc_common.logit({'level': 'INFO', 'message': f'{p}:{v}'}) else: p = ioc.IOCage(jail=jail, skip_jails=True).get(prop) ioc_common.logit({'level': 'INFO', 'message': p}) else: jails = ioc.IOCage().get(prop, recursive=True) table.header(['NAME', f'PROP - {prop}']) for jail_dict in jails: for jail, prop in jail_dict.items(): if header: table.add_row([jail, prop]) else: ioc_common.logit({ 'level': 'INFO', 'message': f'{jail}\t{prop}' }) if header: # Prints the table ioc_common.logit({'level': 'INFO', 'message': table.draw()})
def _print_scores(y_true, y_pred, metrics): tab = tt.Texttable() print('\nTable: Evaluation Scores') headings = ['Metric', 'Score'] tab.header(headings) for m in metrics: score = None try: if m == 'accuracy': y_pred = optimize_threshold(y_true, y_pred, accuracy_score) score = accuracy_score(y_true, y_pred) elif m == 'precision': y_pred = optimize_threshold(y_true, y_pred, precision_score) score = precision_score(y_true, y_pred) elif m == 'precision.micro': y_pred = optimize_threshold(y_true, y_pred, precision_score, average='micro') score = precision_score(y_true, y_pred, average='micro') elif m == 'precision.macro': y_pred = optimize_threshold(y_true, y_pred, precision_score, average='macro') score = precision_score(y_true, y_pred, average='macro') elif m == 'recall': score = recall_score(y_true, y_pred) elif m == 'recall.micro': y_pred = optimize_threshold(y_true, y_pred, recall_score, average='micro') score = recall_score(y_true, y_pred, average='micro') elif m == 'recall.macro': y_pred = optimize_threshold(y_true, y_pred, recall_score, average='macro') score = recall_score(y_true, y_pred, average='macro') elif m == 'f1': y_pred = optimize_threshold(y_true, y_pred, f1_score) score = f1_score(y_true, y_pred) elif m == 'f1.micro': y_pred = optimize_threshold(y_true, y_pred, f1_score, average='micro') score = f1_score(y_true, y_pred, average='micro') elif m == 'f1.macro': y_pred = optimize_threshold(y_true, y_pred, f1_score, average='macro') score = f1_score(y_true, y_pred, average='macro') elif m == 'pearson_corr': score = pearson_corr(y_true, y_pred) elif m == 'spearman_corr': score = spearman_corr(y_true, y_pred) tab.add_row([m, score]) except Exception as e: tab.add_row([m, 'Invalid']) s = tab.draw() print(s)
def status_report(jobs, width, height=None, tmp_prefix='', dst_prefix=''): '''height, if provided, will limit the number of rows in the table, showing first and last rows, row numbers and an elipsis in the middle.''' abbreviate_jobs_list = False n_begin_rows = 0 n_end_rows = 0 if height and height < len(jobs) + 1: # One row for header abbreviate_jobs_list = True if abbreviate_jobs_list: n_rows = height - 2 # One for header, one for elipsis n_begin_rows = int(n_rows / 2) n_end_rows = n_rows - n_begin_rows tab = tt.Texttable() headings = [ 'plot id', 'k', 'tmp', 'dst', 'wall', 'phase', 'tmp', 'pid', 'stat', 'mem', 'user', 'sys', 'io' ] if height: headings.insert(0, '#') tab.header(headings) tab.set_cols_dtype('t' * len(headings)) tab.set_cols_align('r' * len(headings)) tab.set_header_align('r' * len(headings)) for i, j in enumerate(sorted(jobs, key=job.Job.get_time_wall)): # Elipsis row if abbreviate_jobs_list and i == n_begin_rows: row = ['...'] + ([''] * 13) # Omitted row elif abbreviate_jobs_list and i > n_begin_rows and i < (len(jobs) - n_end_rows): continue # Regular row else: try: row = [ j.plot_id[:8] + '...', j.k, abbr_path(j.tmpdir, tmp_prefix), abbr_path(j.dstdir, dst_prefix), plot_util.time_format(j.get_time_wall()), '%d:%d' % j.progress(), plot_util.human_format(j.get_tmp_usage(), 0), j.proc.pid, j.get_run_status(), plot_util.human_format(j.get_mem_usage(), 1), plot_util.time_format(j.get_time_user()), plot_util.time_format(j.get_time_sys()), plot_util.time_format(j.get_time_iowait()) ] except psutil.NoSuchProcess: # In case the job has disappeared row = [j.plot_id[:8] + '...'] + (['--'] * 12) if height: row.insert(0, '%3d' % i) tab.add_row(row) tab.set_max_width(width) tab.set_deco(0) # No borders # return ('tmp dir prefix: %s ; dst dir prefix: %s\n' % (tmp_prefix, dst_prefix) return tab.draw()
def start(): parser = argparse.ArgumentParser( description= 'theHarvester is used to gather open source intelligence (OSINT) on a\n' 'company or domain.') parser.add_argument('-d', '--domain', help='company name or domain to search', required=True) parser.add_argument('-l', '--limit', help='limit the number of search results, default=500', default=500, type=int) parser.add_argument('-S', '--start', help='start with result number X, default=0', default=0, type=int) parser.add_argument('-g', '--google-dork', help='use Google Dorks for Google search', default=False, action='store_true') parser.add_argument( '-p', '--port-scan', help= 'scan the detected hosts and check for Takeovers (21,22,80,443,8080)', default=False, action='store_true') parser.add_argument('-s', '--shodan', help='use Shodan to query discovered hosts', default=False, action='store_true') parser.add_argument( '-v', '--virtual-host', help='verify host name via DNS resolution and search for virtual hosts', action='store_const', const='basic', default=False) parser.add_argument('-e', '--dns-server', help='DNS server to use for lookup') parser.add_argument( '-t', '--dns-tld', help='perform a DNS TLD expansion discovery, default False', default=False) parser.add_argument('-n', '--dns-lookup', help='enable DNS server lookup, default False', default=False, action='store_true') parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain', default=False, action='store_true') parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file', default='', type=str) parser.add_argument( '-b', '--source', help='''baidu, bing, bingapi, censys, crtsh, dnsdumpster, dogpile, duckduckgo, github-code, google, hunter, intelx, linkedin, linkedin_links, netcraft, securityTrails, threatcrowd, trello, twitter, vhost, virustotal, yahoo''') args = parser.parse_args() try: db = stash.stash_manager() db.do_init() except Exception: pass all_emails = [] all_hosts = [] all_ip = [] dnsbrute = args.dns_brute dnslookup = args.dns_lookup dnsserver = args.dns_server dnstld = args.dns_tld filename = args.filename # type: str full = [] google_dorking = args.google_dork host_ip = [] limit = args.limit # type: int ports_scanning = args.port_scan shodan = args.shodan start = args.start # type: int takeover_check = False trello_info = ([], False) vhost = [] virtual = args.virtual_host word = args.domain # type: str if args.source is not None: engines = set(map(str.strip, args.source.split(','))) if set(engines).issubset(Core.get_supportedengines()): print(f'\033[94m[*] Target: {word} \n \033[0m') for engineitem in engines: if engineitem == 'baidu': print('\033[94m[*] Searching Baidu. \033[0m') from theHarvester.discovery import baidusearch try: baidu_search = baidusearch.SearchBaidu(word, limit) baidu_search.process() all_emails = filter(baidu_search.get_emails()) hosts = filter(baidu_search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'baidu') db.store_all(word, all_emails, 'email', 'baidu') except Exception: pass elif engineitem == 'bing' or engineitem == 'bingapi': print('\033[94m[*] Searching Bing. \033[0m') from theHarvester.discovery import bingsearch try: bing_search = bingsearch.SearchBing(word, limit, start) bingapi = '' if engineitem == 'bingapi': bingapi += 'yes' else: bingapi += 'no' bing_search.process(bingapi) all_emails = filter(bing_search.get_emails()) hosts = filter(bing_search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'email', 'bing') db.store_all(word, all_hosts, 'host', 'bing') except Exception as e: if isinstance(e, MissingKey): print(e) else: pass elif engineitem == 'censys': print('\033[94m[*] Searching Censys. \033[0m') from theHarvester.discovery import censys # Import locally or won't work censys_search = censys.SearchCensys(word, limit) censys_search.process() all_ip = censys_search.get_ipaddresses() hosts = filter(censys_search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'censys') db.store_all(word, all_ip, 'ip', 'censys') elif engineitem == 'crtsh': try: print('\033[94m[*] Searching CRT.sh. \033[0m') from theHarvester.discovery import crtsh crtsh_search = crtsh.SearchCrtsh(word) crtsh_search.process() hosts = filter(crtsh_search.get_data()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'CRTsh') except Exception: print( f'\033[93m[!] An timeout occurred with crtsh, cannot find {args.domain}\033[0m' ) elif engineitem == 'dnsdumpster': try: print('\033[94m[*] Searching DNSdumpster. \033[0m') from theHarvester.discovery import dnsdumpster dns_dumpster_search = dnsdumpster.SearchDnsDumpster( word) dns_dumpster_search.process() hosts = filter(dns_dumpster_search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'dnsdumpster') except Exception as e: print( f'\033[93m[!] An error occurred with dnsdumpster: {e} \033[0m' ) elif engineitem == 'dogpile': try: print('\033[94m[*] Searching Dogpile. \033[0m') from theHarvester.discovery import dogpilesearch dogpile_search = dogpilesearch.SearchDogpile( word, limit) dogpile_search.process() emails = filter(dogpile_search.get_emails()) hosts = filter(dogpile_search.get_hostnames()) all_hosts.extend(hosts) all_emails.extend(emails) db = stash.stash_manager() db.store_all(word, all_hosts, 'email', 'dogpile') db.store_all(word, all_hosts, 'host', 'dogpile') except Exception as e: print( f'\033[93m[!] An error occurred with Dogpile: {e} \033[0m' ) elif engineitem == 'duckduckgo': print('\033[94m[*] Searching DuckDuckGo. \033[0m') from theHarvester.discovery import duckduckgosearch duckduckgo_search = duckduckgosearch.SearchDuckDuckGo( word, limit) duckduckgo_search.process() emails = filter(duckduckgo_search.get_emails()) hosts = filter(duckduckgo_search.get_hostnames()) all_hosts.extend(hosts) all_emails.extend(emails) db = stash.stash_manager() db.store_all(word, all_hosts, 'email', 'duckduckgo') db.store_all(word, all_hosts, 'host', 'duckduckgo') elif engineitem == 'github-code': print('\033[94m[*] Searching Github (code). \033[0m') try: from theHarvester.discovery import githubcode github_search = githubcode.SearchGithubCode( word, limit) github_search.process() emails = filter(github_search.get_emails()) all_emails.extend(emails) hosts = filter(github_search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'github-code') db.store_all(word, all_emails, 'email', 'github-code') except MissingKey as ex: print(ex) else: pass elif engineitem == 'exalead': print('\033[94m[*] Searching Exalead \033[0m') from theHarvester.discovery import exaleadsearch exalead_search = exaleadsearch.SearchExalead( word, limit, start) exalead_search.process() emails = filter(exalead_search.get_emails()) all_emails.extend(emails) hosts = filter(exalead_search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'exalead') db.store_all(word, all_emails, 'email', 'exalead') elif engineitem == 'google': print('\033[94m[*] Searching Google. \033[0m') from theHarvester.discovery import googlesearch google_search = googlesearch.SearchGoogle( word, limit, start) google_search.process(google_dorking) emails = filter(google_search.get_emails()) all_emails.extend(emails) hosts = filter(google_search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'google') db.store_all(word, all_emails, 'email', 'google') elif engineitem == 'hunter': print('\033[94m[*] Searching Hunter. \033[0m') from theHarvester.discovery import huntersearch # Import locally or won't work. try: hunter_search = huntersearch.SearchHunter( word, limit, start) hunter_search.process() emails = filter(hunter_search.get_emails()) all_emails.extend(emails) hosts = filter(hunter_search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'hunter') db.store_all(word, all_emails, 'email', 'hunter') except Exception as e: if isinstance(e, MissingKey): print(e) else: pass elif engineitem == 'intelx': print('\033[94m[*] Searching Intelx. \033[0m') from theHarvester.discovery import intelxsearch # Import locally or won't work. try: intelx_search = intelxsearch.SearchIntelx(word, limit) intelx_search.process() emails = filter(intelx_search.get_emails()) all_emails.extend(emails) hosts = filter(intelx_search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'intelx') db.store_all(word, all_emails, 'email', 'intelx') except Exception as e: if isinstance(e, MissingKey): print(e) else: print(e) elif engineitem == 'linkedin': print('\033[94m[*] Searching Linkedin. \033[0m') from theHarvester.discovery import linkedinsearch linkedin_search = linkedinsearch.SearchLinkedin( word, limit) linkedin_search.process() people = linkedin_search.get_people() db = stash.stash_manager() db.store_all(word, people, 'name', 'linkedin') if len(people) == 0: print('\n[*] No users found Linkedin.\n\n') else: print(f'\n[*] Users found: {len(people)}') print('---------------------') for user in sorted(list(set(people))): print(user) elif engineitem == 'linkedin_links': print('\033[94m[*] Searching Linkedin. \033[0m') from theHarvester.discovery import linkedinsearch linkedin_links_search = linkedinsearch.SearchLinkedin( word, limit) linkedin_links_search.process() people = linkedin_links_search.get_links() db = stash.stash_manager() db.store_all(word, people, 'name', 'linkedin') if len(people) == 0: print('\n[*] No links found Linkedin.\n\n') else: print(f'\n[*] Links found: {len(people)}') print('---------------------') for user in sorted(list(set(people))): print(user) elif engineitem == 'netcraft': print('\033[94m[*] Searching Netcraft. \033[0m') from theHarvester.discovery import netcraft netcraft_search = netcraft.SearchNetcraft(word) netcraft_search.process() hosts = filter(netcraft_search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'netcraft') elif engineitem == 'securityTrails': print('\033[94m[*] Searching SecurityTrails. \033[0m') from theHarvester.discovery import securitytrailssearch try: securitytrails_search = securitytrailssearch.SearchSecuritytrail( word) securitytrails_search.process() hosts = filter(securitytrails_search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, hosts, 'host', 'securityTrails') ips = securitytrails_search.get_ips() all_ip.extend(ips) db = stash.stash_manager() db.store_all(word, ips, 'ip', 'securityTrails') except Exception as e: if isinstance(e, MissingKey): print(e) else: pass elif engineitem == 'threatcrowd': print('\033[94m[*] Searching Threatcrowd. \033[0m') from theHarvester.discovery import threatcrowd try: threatcrowd_search = threatcrowd.SearchThreatcrowd( word) threatcrowd_search.process() hosts = filter(threatcrowd_search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'threatcrowd') except Exception as e: print(e) elif engineitem == 'trello': print('\033[94m[*] Searching Trello. \033[0m') from theHarvester.discovery import trello # Import locally or won't work. trello_search = trello.SearchTrello(word, limit) trello_search.process() emails = filter(trello_search.get_emails()) all_emails.extend(emails) info = trello_search.get_urls() hosts = filter(info[0]) trello_info = (info[1], True) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, hosts, 'host', 'trello') db.store_all(word, emails, 'email', 'trello') elif engineitem == 'twitter': print( '\033[94m[*] Searching Twitter usernames using Google. \033[0m' ) from theHarvester.discovery import twittersearch twitter_search = twittersearch.SearchTwitter(word, limit) twitter_search.process() people = twitter_search.get_people() db = stash.stash_manager() db.store_all(word, people, 'name', 'twitter') if len(people) == 0: print('\n[*] No users found.\n\n') else: print('\n[*] Users found: ' + str(len(people))) print('---------------------') for user in sorted(list(set(people))): print(user) elif engineitem == 'virustotal': print('\033[94m[*] Searching VirusTotal. \033[0m') from theHarvester.discovery import virustotal virustotal_search = virustotal.SearchVirustotal(word) virustotal_search.process() hosts = filter(virustotal_search.get_hostnames()) all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'virustotal') elif engineitem == 'yahoo': print('\033[94m[*] Searching Yahoo. \033[0m') from theHarvester.discovery import yahoosearch yahoo_search = yahoosearch.SearchYahoo(word, limit) yahoo_search.process() hosts = yahoo_search.get_hostnames() emails = yahoo_search.get_emails() all_hosts.extend(filter(hosts)) all_emails.extend(filter(emails)) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'yahoo') db.store_all(word, all_emails, 'email', 'yahoo') else: print('\033[93m[!] Invalid source.\n\n \033[0m') sys.exit(1) # Sanity check to see if all_emails and all_hosts are defined. try: all_emails except NameError: print( '\n\n\033[93m[!] No emails found because all_emails is not defined.\n\n \033[0m' ) sys.exit(1) try: all_hosts except NameError: print( '\n\n\033[93m[!] No hosts found because all_hosts is not defined.\n\n \033[0m' ) sys.exit(1) # Results if len(all_ip) == 0: print('\n[*] No IPs found.') else: print('\n[*] IPs found: ' + str(len(all_ip))) print('-------------------') ips = sorted( ipaddress.ip_address(line.strip()) for line in set(all_ip)) print('\n'.join(map(str, ips))) if len(all_emails) == 0: print('\n[*] No emails found.') else: print('\n[*] Emails found: ' + str(len(all_emails))) print('----------------------') print(('\n'.join(sorted(list(set(all_emails)))))) if len(all_hosts) == 0: print('\n[*] No hosts found.\n\n') else: print('\n[*] Hosts found: ' + str(len(all_hosts))) print('---------------------') all_hosts = sorted(list(set(all_hosts))) full_host = hostchecker.Checker(all_hosts) full = full_host.check() for host in full: ip = host.split(':')[1] print(host) if ip != 'empty': if host_ip.count(ip.lower()): pass else: host_ip.append(ip.lower()) db = stash.stash_manager() db.store_all(word, host_ip, 'ip', 'DNS-resolver') if trello_info[1] is True: trello_urls = trello_info[0] if trello_urls is []: print('\n[*] No URLs found.') else: total = len(trello_urls) print('\n[*] URLs found: ' + str(total)) print('--------------------') for url in sorted(list(set(trello_urls))): print(url) # DNS brute force # dnsres = [] if dnsbrute is True: print('\n[*] Starting DNS brute force.') a = dnssearch.DnsForce(word, dnsserver, verbose=True) res = a.process() # print('\n[*] Hosts found after DNS brute force:') # for y in res: # print('-------------------------------------') # print(y) # dnsres.append(y.split(':')[0]) # if y not in full: # full.append(y) # db = stash.stash_manager() # db.store_all(word, dnsres, 'host', 'dns_bruteforce') # Port scanning if ports_scanning is True: print('\n\n[*] Scanning ports (active).\n') for x in full: host = x.split(':')[1] domain = x.split(':')[0] if host != 'empty': print(('[*] Scanning ' + host)) ports = [21, 22, 80, 443, 8080] try: scan = port_scanner.PortScan(host, ports) openports = scan.process() if len(openports) > 1: print(('\t[*] Detected open ports: ' + ','.join(str(e) for e in openports))) takeover_check = 'True' if takeover_check == 'True' and len(openports) > 0: search_take = takeover.TakeOver(domain) search_take.process() except Exception as e: print(e) # DNS reverse lookup dnsrev = [] if dnslookup is True: print('\n[*] Starting active queries.') analyzed_ranges = [] for entry in host_ip: print(entry) ip = entry.split(':')[0] ip_range = ip.split('.') ip_range[3] = '0/24' s = '.' ip_range = s.join(ip_range) if not analyzed_ranges.count(ip_range): print('[*] Performing reverse lookup in ' + ip_range) a = dnssearch.DnsReverse(ip_range, True) a.list() res = a.process() analyzed_ranges.append(ip_range) else: continue for entries in res: if entries.count(word): dnsrev.append(entries) if entries not in full: full.append(entries) print('[*] Hosts found after reverse lookup (in target domain):') print('--------------------------------------------------------') for xh in dnsrev: print(xh) # DNS TLD expansion dnstldres = [] if dnstld is True: print('[*] Starting DNS TLD expansion.') a = dnssearch.DnsTld(word, dnsserver, verbose=True) res = a.process() print('\n[*] Hosts found after DNS TLD expansion:') print('----------------------------------------') for y in res: print(y) dnstldres.append(y) if y not in full: full.append(y) # Virtual hosts search if virtual == 'basic': print('\n[*] Virtual hosts:') print('------------------') for l in host_ip: basic_search = bingsearch.SearchBing(l, limit, start) basic_search.process_vhost() results = basic_search.get_allhostnames() for result in results: result = re.sub(r'[[\<\/?]*[\w]*>]*', '', result) result = re.sub('<', '', result) result = re.sub('>', '', result) print((l + '\t' + result)) vhost.append(l + ':' + result) full.append(l + ':' + result) vhost = sorted(set(vhost)) else: pass # Shodan shodanres = [] if shodan is True: import texttable tab = texttable.Texttable() header = [ 'IP address', 'Hostname', 'Org', 'Services:Ports', 'Technologies' ] tab.header(header) tab.set_cols_align(['c', 'c', 'c', 'c', 'c']) tab.set_cols_valign(['m', 'm', 'm', 'm', 'm']) tab.set_chars(['-', '|', '+', '#']) tab.set_cols_width([15, 20, 15, 15, 18]) host_ip = list(set(host_ip)) print('\033[94m[*] Searching Shodan. \033[0m') try: for ip in host_ip: print(('\tSearching for ' + ip)) shodan = shodansearch.SearchShodan() rowdata = shodan.search_ip(ip) time.sleep(2) tab.add_row(rowdata) printedtable = tab.draw() print(printedtable) except Exception as e: print(f'\033[93m[!] An error occurred with Shodan: {e} \033[0m') else: pass # Here we need to add explosion mode. # We have to take out the TLDs to do this. recursion = None if recursion: counter = 0 for word in vhost: search = googlesearch.SearchGoogle(word, limit, counter) search.process(google_dorking) emails = search.get_emails() hosts = search.get_hostnames() print(emails) print(hosts) else: pass # Reporting if filename != "": try: print('\n[*] Reporting started.') db = stash.stash_manager() scanboarddata = db.getscanboarddata() latestscanresults = db.getlatestscanresults(word) previousscanresults = db.getlatestscanresults(word, previousday=True) latestscanchartdata = db.latestscanchartdata(word) scanhistorydomain = db.getscanhistorydomain(word) pluginscanstatistics = db.getpluginscanstatistics() generator = statichtmlgenerator.HtmlGenerator(word) HTMLcode = generator.beginhtml() HTMLcode += generator.generatelatestscanresults(latestscanresults) HTMLcode += generator.generatepreviousscanresults( previousscanresults) graph = reportgraph.GraphGenerator(word) HTMLcode += graph.drawlatestscangraph(word, latestscanchartdata) HTMLcode += graph.drawscattergraphscanhistory( word, scanhistorydomain) HTMLcode += generator.generatepluginscanstatistics( pluginscanstatistics) HTMLcode += generator.generatedashboardcode(scanboarddata) HTMLcode += '<p><span style="color: #000000;">Report generated on ' + str( datetime.datetime.now()) + '</span></p>' HTMLcode += ''' </body> </html> ''' Html_file = open(filename, 'w') Html_file.write(HTMLcode) Html_file.close() print('[*] Reporting finished.') print('[*] Saving files.') except Exception as e: print(e) print( '\n\033[93m[!] An error occurred while creating the output file.\n\n \033[0m' ) sys.exit(1) try: filename = filename.split('.')[0] + '.xml' file = open(filename, 'w') file.write('<?xml version="1.0" encoding="UTF-8"?><theHarvester>') for x in all_emails: file.write('<email>' + x + '</email>') for x in full: x = x.split(':') if len(x) == 2: file.write('<host>' + '<ip>' + x[1] + '</ip><hostname>' + x[0] + '</hostname>' + '</host>') else: file.write('<host>' + x + '</host>') for x in vhost: x = x.split(':') if len(x) == 2: file.write('<vhost>' + '<ip>' + x[1] + '</ip><hostname>' + x[0] + '</hostname>' + '</vhost>') else: file.write('<vhost>' + x + '</vhost>') if shodanres != []: shodanalysis = [] for x in shodanres: res = x.split('SAPO') file.write('<shodan>') file.write('<host>' + res[0] + '</host>') file.write('<port>' + res[2] + '</port>') file.write('<banner><!--' + res[1] + '--></banner>') reg_server = re.compile('Server:.*') temp = reg_server.findall(res[1]) if temp != []: shodanalysis.append(res[0] + ':' + temp[0]) file.write('</shodan>') if shodanalysis != []: shodanalysis = sorted(set(shodanalysis)) file.write('<servers>') for x in shodanalysis: file.write('<server>' + x + '</server>') file.write('</servers>') file.write('</theHarvester>') file.flush() file.close() print('[*] Files saved.') except Exception as er: print( f'\033[93m[!] An error occurred while saving the XML file: {er} \033[0m' ) print('\n\n') sys.exit(0)
def main(): # The first, and only argument needs to be a directory parser = argparse.ArgumentParser( description= "Checks for duplicated images in a directory tree. Compares just image data, metadata is ignored, so physically different files may be reported as duplicates if they have different metadata (tags, titles, JPEG rotation, EXIF info...)." ) parser.add_argument('directory', help="Base directory to check") parser.add_argument('-1', '--sameline', help="List each set of matches in a single line", action='store_true', required=False) parser.add_argument( '-d', '--delete', help="Prompt user for files to preserve, deleting all others", action='store_true', required=False) parser.add_argument( '-a', '--auto', help= "If -d or --delete flag is enabled, always select the 'auto' action to select the file to preserve.", action='store_true', required=False) parser.add_argument('-c', '--clean', help="Don't write to disk signatures cache", action='store_true', required=False) parser.add_argument( '-m', '--method', help= "Hash method to use. Default is MD5, but CRC might be faster on slower CPUs where process is not I/O bound", default="MD5", choices=["MD5", "CRC"], required=False) parser.add_argument('--version', action='version', version='%(prog)s ' + VERSION) args = parser.parse_args() if args.auto and not args.delete: sys.stderr.write( "'-a' or '--auto' only makes sense when deleting files with '-d' or '--delete'\n" ) exit(1) pwd = os.getcwd() # Check if jpeginfo is installed havejpeginfo = True try: devnull = open(os.devnull, 'w') check_call(["jpeginfo", "--version"], stdout=devnull, stderr=devnull) sys.stderr.write( "jpeginfo found in system, will be used to check JPEG file integrity\n" ) except (sub.CalledProcessError, FileNotFoundError): sys.stderr.write( "jpeginfo not found in system, please install it for smarter JPEG file integrity detection\n" ) havejpeginfo = False try: os.chdir(args.directory) except: sys.stderr.write("Directory %s doesn't exist\n" % args.directory) exit(1) # Extensiones admitidas (case insensitive) extensiones = ('jpg', 'jpeg') # Diccionario con información sobre los ficheros jpegs = {} # Flag para saber si hay que volver a escribir la caché por cambios modif = False # Recuperar información de los ficheros generada previamente, si existe rootDir = '.' fsigs = '.signatures' if os.path.isfile(fsigs): cache = open(fsigs, 'rb') try: sys.stderr.write("Signatures cache detected, loading...\n") jpegs = pickle.load(file=cache) cache.close() # Clean up non-existing entries sys.stderr.write("Cleaning up deleted files from cache...\n") jpegs = dict( [x for x in iter(jpegs.items()) if os.path.exists(x[0])]) except (pickle.UnpicklingError, KeyError, EOFError, ImportError, IndexError): # Si el fichero no es válido, ignorarlo y marcar que hay que escribir cambios jpegs = {} modif = True cache.close() # Create process pool for parallel hash calculation pool = Pool() count = 1 for dirName, subdirList, fileList in os.walk(rootDir): sys.stderr.write('Exploring %s\n' % dirName) for fname in fileList: # Update signatures cache every 100 files if modif and ((count % 100) == 0): writecache(jpegs, args, fsigs) modif = False if fname.lower().endswith(extensiones): ruta = os.path.join(dirName, fname) # Si el fichero no está en la caché, o está pero con tamaño diferente, añadirlo if (ruta not in jpegs) or ( (ruta in jpegs) and (jpegs[ruta]['size'] != os.path.getsize(ruta))): sys.stderr.write(" Calculating hash of %s...\n" % ruta) jpegs[ruta] = { 'name': fname, 'dir': dirName, 'hash': hashcalc(ruta, pool, args.method, havejpeginfo), 'size': os.path.getsize(ruta) } modif = True count += 1 # Write hash cache to disk if modif: writecache(jpegs, args, fsigs) # Check for duplicates # Create a new dict indexed by hash # Initialize dict with an empty list for every possible hash hashes = {} for f in jpegs: for h in jpegs[f]['hash']: hashes[h] = [] # Group files with the same hash together for f in jpegs: for h in jpegs[f]['hash']: hashes[h].append(jpegs[f]) # Discard hashes without duplicates dupes = {} for h in hashes: if len(hashes[h]) > 1: dupes[h] = hashes[h] # Delete entries whose hash couldn't be generated, so they're not reported as duplicates if 'ERR' in dupes: del dupes['ERR'] # Discard duplicated sets (probably a lot if --rotations is activated) nodupes = [] for elem in list(dupes.values()): if not elem in nodupes: nodupes.append(elem) # Cleanup. Not strictly necessary, but if there're a lot of files these can get quite big del hashes, dupes nset = 1 tmpdirs = [] for dupset in nodupes: # Add path field (for convenience) and sort by file path for d in dupset: d.update({"path": os.path.join(d["dir"], d["name"])}) dupset.sort(key=lambda k: k['path']) print() if args.delete: # Calculate best guess for auto mode dupaux = [d['path'] for d in dupset] # Sort by path length (probably not needed as dupset is already sorted, but just in case) dupaux.sort(key=len) # Best guess is the entry with most tags, or the one with shorter path if tags are equal (due to previous sort) bestguess = dupaux.index( max(dupaux, key=lambda k: len(metadata_summary(k)["tags"]))) optselected = False while not optselected: # Prompt for which duplicated file to keep, delete the others t = tt.Texttable() t.set_cols_align(["c", "r", "l", "l", "l", "l", "l", "l"]) t.set_chars(['-', '|', '+', '-']) t.set_deco(t.HEADER) # Get terminal width in order to set column sizes. colsize = int(os.popen('stty size', 'r').read().split()[1]) t.set_cols_width( [1, 1, 50, 20, 11, 10, 10, colsize - 103 - 30]) rws = [[ "", "#", "file", "date", "orientation", "title", "software", "tags" ]] for i in range(len(dupset)): aux = dupset[i] md = metadata_summary(aux["path"]) rws.append([ "*" if i == bestguess else " ", i + 1, dupset[i]["path"], md["date"], md["orientation"], md["title"], md["software"], ", ".join(md["tags"]) ]) t.add_rows(rws) print(("\n" + t.draw())) if args.auto: answer = 'auto' else: answer = input( "\nSet %d of %d, preserve files [%d - %d, all, auto, show, detail, help, quit] (default: auto): " % (nset, len(nodupes), 1, len(dupset))) if answer in ["detail", "d"]: # Show detailed differences in EXIF tags filelist = [ os.path.join(x['dir'], x['name']) for x in dupset ] metadata_comp_table(filelist) elif answer in ["help", "h"]: print() print("[0-9]: Keep the selected file, delete the rest") print("(a)ll: Keep all files, don't delete anything") print( "auto: Keep picture with most tags, or shorter path. If equal, don't delete anything" ) print( "(s)how: Copy duplicated files to a temporary directory and open in a file manager window (desktop default)" ) print( "(d)etail: Show a detailed table with metadata differences between files" ) print("(h)elp: Show this screen") print("(q)uit: Exit program") print() elif answer in ["quit", "q"]: # If asked, write changes, delete temps and quit if modif: writecache(jpegs, args, fsigs) rmtemps(tmpdirs) exit(0) elif answer in ["show", "s"]: # Create a temporary directory, copy duplicated files and open a file manager tmpdir = tempfile.mkdtemp() tmpdirs.append(tmpdir) for i in range(len(dupset)): p = dupset[i]['path'] ntemp = "%d_%s" % (i, dupset[i]['name']) shutil.copyfile(p, os.path.join(tmpdir, ntemp)) sub.Popen(["xdg-open", tmpdir], stdout=None, stderr=None) elif answer in ["all", "a"]: # Don't delete anything sys.stderr.write( "Skipping deletion, all duplicates remain\n") optselected = True elif answer in ["auto", ""]: answer = bestguess else: # If it's no option, assume it's a number and convert it to an array index answer = int(answer) - 1 # If we have a valid number as an answer, delete all but the selected file try: answer = int(answer) for i in range(len(dupset)): if i != answer: p = dupset[i]['path'] os.remove(p) del jpegs[p] modif = True sys.stderr.write("Kept %s, deleted others\n" % (dupset[answer]["name"])) optselected = True except ValueError: pass nset += 1 else: # Just show duplicates for f in dupset: print(f['path'], end=' ') if not args.sameline: print("\n", end=' ') # Final update of the cache in order to remove signatures of deleted files if modif: writecache(jpegs, args, fsigs) # Delete temps rmtemps(tmpdirs) # Restore directory os.chdir(pwd)
def run(self): citationMap = { 'Mike': { 'Mike': 0, 'Jim': 0, 'Mary': 0, 'Bob': 0, 'Ann': 0, 'Joe': 0, 'Nancy': 0 }, 'Jim': { 'Mike': 20, 'Jim': 0, 'Mary': 20, 'Bob': 20, 'Ann': 0, 'Joe': 20, 'Nancy': 0 }, 'Mary': { 'Mike': 1, 'Jim': 10, 'Mary': 0, 'Bob': 1, 'Ann': 0, 'Joe': 1, 'Nancy': 0 }, 'Bob': { 'Mike': 1, 'Jim': 10, 'Mary': 1, 'Bob': 0, 'Ann': 0, 'Joe': 1, 'Nancy': 0 }, 'Ann': { 'Mike': 0, 'Jim': 0, 'Mary': 0, 'Bob': 0, 'Ann': 0, 'Joe': 0, 'Nancy': 0 }, 'Joe': { 'Mike': 0, 'Jim': 0, 'Mary': 0, 'Bob': 0, 'Ann': 0, 'Joe': 0, 'Nancy': 0 }, 'Nancy': { 'Mike': 1, 'Jim': 10, 'Mary': 1, 'Bob': 1, 'Ann': 0, 'Joe': 1, 'Nancy': 0 } } self.graph, authorMap, conferenceMap =\ SampleGraphUtility.constructPathSimExampleThree(extraAuthorsAndCitations=True, citationMap = citationMap) # Get the nodes we care about conferences = [ conferenceMap['SIGMOD'], conferenceMap['VLDB'], conferenceMap['ICDE'], conferenceMap['KDD'] ] authors = [ authorMap['Mike'], authorMap['Jim'], authorMap['Mary'], authorMap['Bob'], authorMap['Ann'], authorMap['Joe'], authorMap['Nancy'], ] metaPathUtility = EdgeBasedMetaPathUtility() # Project a 2-typed heterogeneous graph over adapted PathSim example publicationProjectedGraph = metaPathUtility.createHeterogeneousProjection( self.graph, [Author, Paper, Conference], symmetric=True) self.output('\nAdjacency Matrix (Projected):') adjMatrixTable = texttable.Texttable() rows = [['Author'] + [conference.name for conference in conferences]] rows += [[author.name] + [ publicationProjectedGraph.getNumberOfEdges(author, conference) for conference in conferences ] for author in authors] adjMatrixTable.add_rows(rows) self.output(adjMatrixTable.draw()) # Project a homogeneous citation graph over adapted PathSim example citationProjectedGraph = metaPathUtility.createHomogeneousProjection( self.graph, [Author, Paper, Paper, Author]) self.output('\nCitation Matrix:') adjMatrixTable = texttable.Texttable() rows = [['Author'] + [author.name for author in authors]] rows += [[author.name] + [ citationProjectedGraph.getNumberOfEdges(author, otherAuthor) for otherAuthor in authors ] for author in authors] adjMatrixTable.add_rows(rows) self.output(adjMatrixTable.draw()) # Output total out/in citations self.output('\nCitations Total:') totalCitationsTable = texttable.Texttable() rows = [['Author', 'In', 'Out']] for author in authors: inCount = sum( citationProjectedGraph.getNumberOfEdges(otherAuthor, author) for otherAuthor in authors) outCount = sum( citationProjectedGraph.getNumberOfEdges(author, otherAuthor) for otherAuthor in authors) rows += [[author.name, inCount, outCount]] totalCitationsTable.add_rows(rows) self.output(totalCitationsTable.draw()) # Get PathSim similarity scores pathSimStrategy = PathSimStrategy( self.graph, [Author, Paper, Conference, Paper, Author], True) self.outputSimilarityScores(authorMap, authors, pathSimStrategy, 'APCPA PathSim') # Output SimRank-related scores strategy = SimRankStrategy(self.graph, [Author, Paper, Paper, Author], symmetric=True) self.outputSimilarityScores(authorMap, authors, strategy, "SimRank") # Output the projected PageRank/HITS similarity scores for name, algorithm in zip( ['PageRank', 'HITS'], [PageRankDistanceStrategy, HITSDistanceStrategy]): strategy = algorithm(self.graph, [Author, Paper, Paper, Author], symmetric=True) self.outputSimilarityScores(authorMap, authors, strategy, "%s-Distance" % name) # Get NeighborSim similarity scores inNeighborSimStrategy = NeighborSimStrategy( self.graph, [Author, Paper, Paper, Author]) self.outputSimilarityScores(authorMap, authors, inNeighborSimStrategy, 'APPA NeighborSim-In') outNeighborSimStrategy = NeighborSimStrategy( self.graph, [Author, Paper, Paper, Author], reversed=True, smoothed=True) self.outputSimilarityScores(authorMap, authors, outNeighborSimStrategy, 'APPA NeighborSim-Out') # Combined best PR-distance algorithm simRankStrategy = SimRankStrategy(self.graph, [Author, Paper, Paper, Author], symmetric=True) simRank = AggregateSimilarityStrategy( self.graph, [pathSimStrategy, simRankStrategy], [0.5, 0.5]) self.outputSimilarityScores(authorMap, authors, simRank, 'APCPA Pathsim, APPA SimRank') # Combined best neighborsim score combinedNeighborSim = AggregateSimilarityStrategy( self.graph, [pathSimStrategy, inNeighborSimStrategy, outNeighborSimStrategy], [0.6, 0.2, 0.2]) self.outputSimilarityScores( authorMap, authors, combinedNeighborSim, 'APCPA Pathsim, APPA NeighborSim-Combined')
def analyze(logfilenames: typing.List[str], clipterminals: bool, bytmp: bool, bybitfield: bool) -> None: data: typing.Dict[str, typing.Dict[str, typing.List[float]]] = {} for logfilename in logfilenames: with open(logfilename, 'r') as f: # Record of slicing and data associated with the slice sl = 'x' # Slice key phase_time: typing.Dict[str, float] = {} # Map from phase index to time n_sorts = 0 n_uniform = 0 is_first_last = False # Read the logfile, triggering various behaviors on various # regex matches. for line in f: # Beginning of plot job. We may encounter this multiple # times, if a job was run with -n > 1. Sample log line: # 2021-04-08T13:33:43.542 chia.plotting.create_plots : INFO Starting plot 1/5 m = re.search(r'Starting plot (\d*)/(\d*)', line) if m: # (re)-initialize data structures sl = 'x' # Slice key phase_time = {} # Map from phase index to time n_sorts = 0 n_uniform = 0 seq_num = int(m.group(1)) seq_total = int(m.group(2)) is_first_last = seq_num == 1 or seq_num == seq_total # Temp dirs. Sample log line: # Starting plotting progress into temporary dirs: /mnt/tmp/01 and /mnt/tmp/a m = re.search(r'^Starting plotting.*dirs: (.*) and (.*)', line) if m: # Record tmpdir, if slicing by it if bytmp: tmpdir = m.group(1) sl += '-' + tmpdir # Bitfield marker. Sample log line(s): # Starting phase 2/4: Backpropagation without bitfield into tmp files... Mon Mar 1 03:56:11 2021 # or # Starting phase 2/4: Backpropagation into tmp files... Fri Apr 2 03:17:32 2021 m = re.search(r'^Starting phase 2/4: Backpropagation', line) if bybitfield and m: if 'without bitfield' in line: sl += '-nobitfield' else: sl += '-bitfield' # CHIA: Phase timing. Sample log line: # Time for phase 1 = 22796.7 seconds. CPU (98%) Tue Sep 29 17:57:19 2020 for phase in ['1', '2', '3', '4']: m = re.search( r'^Time for phase ' + phase + ' = (\d+.\d+) seconds..*', line) if m: phase_time[phase] = float(m.group(1)) # MADMAX: Phase timing. Sample log line: "Phase 2 took 2193.37 sec" for phase in ['1', '2', '3', '4']: m = re.search(r'^Phase ' + phase + ' took (\d+.\d+) sec.*', line) if m: phase_time[phase] = float(m.group(1)) # Uniform sort. Sample log line: # Bucket 267 uniform sort. Ram: 0.920GiB, u_sort min: 0.688GiB, qs min: 0.172GiB. # or # ....?.... # or # Bucket 511 QS. Ram: 0.920GiB, u_sort min: 0.375GiB, qs min: 0.094GiB. force_qs: 1 m = re.search(r'Bucket \d+ ([^\.]+)\..*', line) if m and not 'force_qs' in line: sorter = m.group(1) n_sorts += 1 if sorter == 'uniform sort': n_uniform += 1 elif sorter == 'QS': pass else: print('Warning: unrecognized sort ' + sorter) # CHIA: Job completion. Record total time in sliced data store. # Sample log line: # Total time = 49487.1 seconds. CPU (97.26%) Wed Sep 30 01:22:10 2020 m = re.search(r'^Total time = (\d+.\d+) seconds.*', line) if m: if clipterminals and is_first_last: pass # Drop this data; omit from statistics. else: data.setdefault(sl, {}).setdefault( 'total time', []).append(float(m.group(1))) for phase in ['1', '2', '3', '4']: data.setdefault(sl, {}).setdefault( 'phase ' + phase, []).append(phase_time[phase]) data.setdefault(sl, {}).setdefault( '%usort', []).append(100 * n_uniform // n_sorts) # MADMAX: Job completion. Record total time in sliced data store. # Sample log line: "Total plot creation time was 2530.76 sec" m = re.search(r'^Total plot creation time was (\d+.\d+) sec.*', line) if m: data.setdefault(sl, {}).setdefault('total time', []).append( float(m.group(1))) for phase in ['1', '2', '3', '4']: data.setdefault(sl, {}).setdefault( 'phase ' + phase, []).append(phase_time[phase]) data.setdefault(sl, {}).setdefault('%usort', []).append( 0) # Not available for MADMAX # Prepare report tab = tt.Texttable() all_measures = [ '%usort', 'phase 1', 'phase 2', 'phase 3', 'phase 4', 'total time' ] headings = ['Slice', 'n'] + all_measures tab.header(headings) for sl in data.keys(): row = [sl] # Sample size sample_sizes = [] for measure in all_measures: values = data.get(sl, {}).get(measure, []) sample_sizes.append(len(values)) sample_size_lower_bound = min(sample_sizes) sample_size_upper_bound = max(sample_sizes) if sample_size_lower_bound == sample_size_upper_bound: row.append('%d' % sample_size_lower_bound) else: row.append('%d-%d' % (sample_size_lower_bound, sample_size_upper_bound)) # Phase timings for measure in all_measures: values = data.get(sl, {}).get(measure, []) if (len(values) > 1): row.append( 'μ=%s σ=%s' % (plot_util.human_format(statistics.mean(values), 1), plot_util.human_format(statistics.stdev(values), 0))) elif (len(values) == 1): row.append(plot_util.human_format(values[0], 1)) else: row.append('N/A') tab.add_row(row) (rows, columns) = os.popen('stty size', 'r').read().split() tab.set_max_width(int(columns)) s = tab.draw() print(s)
def format_table(rows): table = texttable.Texttable(max_width=0) table.header(('dBV', 'dBu', 'Volts RMS', 'Volts peak-to-peak')) table.add_rows(rows, header=False) table.set_cols_align(['r', 'r', 'r', 'r']) return table.draw()
def fetch_plugin_index(self, props, _list=False, list_header=False, list_long=False, accept_license=False, icon=False, official=False): if self.server == "download.freebsd.org": git_server = "https://github.com/freenas/iocage-ix-plugins.git" else: git_server = self.server git_working_dir = f"{self.iocroot}/.plugin_index" # list --plugins won't often be root. if os.geteuid() == 0: try: self.__clone_repo(git_server, git_working_dir) except Exception as err: iocage_lib.ioc_common.logit( { "level": "EXCEPTION", "message": err }, _callback=self.callback) with open(f"{self.iocroot}/.plugin_index/INDEX", "r") as plugins: plugins = json.load(plugins) _plugins = self.__fetch_sort_plugin__(plugins, official=official) if self.plugin is None and not _list: for p in _plugins: iocage_lib.ioc_common.logit( { "level": "INFO", "message": f"[{_plugins.index(p)}] {p}" }, _callback=self.callback, silent=self.silent) if _list: plugin_list = [] for p in _plugins: p = p.split("-", 1) name = p[0] desc, pkg = re.sub(r'[()]', '', p[1]).rsplit(" ", 1) license = plugins[pkg].get("license", "") _official = str(plugins[pkg].get("official", False)) icon_path = plugins[pkg].get("icon", None) p = [name, desc, pkg] if not list_header: p += [license, _official] if icon: p += [icon_path] if official and _official == "False": continue plugin_list.append(p) if not list_header: return plugin_list else: if list_long: table = texttable.Texttable(max_width=0) else: table = texttable.Texttable(max_width=80) list_header = ["NAME", "DESCRIPTION", "PKG"] if icon: list_header += ["ICON"] plugin_list.insert(0, list_header) table.add_rows(plugin_list) return table.draw() if self.plugin is None: self.plugin = input("\nType the number of the desired" " plugin\nPress [Enter] or type EXIT to" " quit: ") self.plugin = self.__fetch_validate_plugin__(self.plugin.lower(), _plugins) self.fetch_plugin(f"{self.iocroot}/.plugin_index/{self.plugin}.json", props, 0, accept_license)
if isinstance(self.spec[k],list): for v in self.spec[k]: if hasattr(v, 'prettyprint'): v = v.prettyprint(max_width=max_width-20) else: v = str(v) summary += v + "\n" else: v = self.spec[k] if hasattr(v, 'prettyprint'): summary = v.prettyprint(max_width=max_width-20) else: summary = str(v) rows.append((k, summary)) table = texttable.Texttable(max_width=max_width) table.add_rows(rows) return table.draw() <<<<<<< HEAD ======= class SpecManagerBase(): """ Manage a specification directory """ def __init__(self, *args, **kwargs): self.specs = [] """ Accept path and read list of spec files available in path. required prefix: spec_
import texttable as tt carlist = [['DT-LT-87', 'Citroen', 'XM', '1999-09-23', 34500], ['GF-NX-07', 'Volkswagen', 'Polo', "1999-07-12", 78000], ['GF-PD-34', 'Volkswagen', 'Polo', "1999-07-22", 57500], ['KR-RT-65', 'Volkswagen', 'Golf', "1999-08-08", 42000], ['PT-ER-45', 'Ford', 'Fiesta', "1999-03-02", 25000], ['TT-PR-73', 'Citroen', 'XM', '1999-03-02', 1200], ['TT-RW-01', 'Volkswagen', 'Polo', "1999-11-14", 4500]] tab = tt.Texttable() header = ['Kenteken', 'Merk', 'Type', 'DatumAPK', 'Kilometerstand'] tab.header(header) for c in carlist: tab.add_row(c) draw = tab.draw() print(draw)
def list_all(self, jails): """List all jails.""" self.full = True if self.plugin else self.full jail_list = [] plugin_index_data = {} for jail in jails: try: mountpoint = jail.properties['mountpoint'] except KeyError: iocage_lib.ioc_common.logit( { 'level': 'ERROR', 'message': f'{jail.name} mountpoint is misconfigured. ' 'Please correct this.' }, _callback=self.callback, silent=self.silent) continue try: conf = iocage_lib.ioc_json.IOCJson(mountpoint).json_get_value( 'all') state = '' except (Exception, SystemExit): # Jail is corrupt, we want all the keys to exist. # So we will take the defaults and let the user # know that they are not correct. def_props = iocage_lib.ioc_json.IOCJson().json_get_value( 'all', default=True) conf = {x: 'N/A' for x in def_props} conf['host_hostuuid'] = \ f'{jail.name.split("/")[-1]}' conf['release'] = 'N/A' state = 'CORRUPT' jid = '-' if self.basejail_only and not iocage_lib.ioc_common.check_truthy( conf.get('basejail', 0)): continue uuid_full = conf["host_hostuuid"] uuid = uuid_full if not self.full: # We only want to show the first 8 characters of a UUID, # if it's not a UUID, we will show the whole name no matter # what. try: uuid = str(_uuid.UUID(uuid, version=4))[:8] except ValueError: # We leave the "uuid" untouched, as it's not a valid # UUID, but instead a named jail. pass full_ip4 = conf["ip4_addr"] ip6 = conf["ip6_addr"] try: short_ip4 = ",".join([ item.split("|")[1].split("/")[0] for item in full_ip4.split(",") ]) except IndexError: short_ip4 = full_ip4 if full_ip4 != "none" else "-" boot = 'on' if iocage_lib.ioc_common.check_truthy( conf.get('boot', 0)) else 'off' jail_type = conf["type"] full_release = conf["release"] basejail = 'yes' if iocage_lib.ioc_common.check_truthy( conf.get('basejail', 0)) else 'no' if "HBSD" in full_release: full_release = re.sub(r"\W\w.", "-", full_release) full_release = full_release.replace("--SD", "-STABLE-HBSD") short_release = full_release.rstrip("-HBSD") else: short_release = "-".join(full_release.rsplit("-")[:2]) if full_ip4 == "none": full_ip4 = "-" if ip6 == "none": ip6 = "-" # Will be set already by a corrupt jail status = False if state != 'CORRUPT': status, jid = self.list_get_jid(uuid_full) if status: state = "up" else: state = "down" if conf["type"] == "template": template = "-" else: jail_root = Dataset(f'{jail.name}/root') if jail_root.exists: _origin_property = jail_root.properties.get('origin') else: _origin_property = None if _origin_property: template = _origin_property template = template.rsplit("/root@", 1)[0].rsplit("/", 1)[-1] else: template = "-" if "release" in template.lower() or "stable" in template.lower(): template = "-" if iocage_lib.ioc_common.check_truthy( conf['dhcp']) and status and os.geteuid() == 0: interface = conf["interfaces"].split(",")[0].split(":")[0] if interface == "vnet0": # Inside jails they are epairNb interface = f"{interface.replace('vnet', 'epair')}b" short_ip4 = "DHCP" full_ip4_cmd = [ "jexec", f"ioc-{uuid_full.replace('.', '_')}", "ifconfig", interface, "inet" ] try: out = su.check_output(full_ip4_cmd) full_ip4 = f'{interface}|' \ f'{out.splitlines()[2].split()[1].decode()}' except (su.CalledProcessError, IndexError) as e: short_ip4 += '(Network Issue)' if isinstance(e, su.CalledProcessError): full_ip4 = f'DHCP - Network Issue: {e}' else: full_ip4 = f'DHCP - Failed Parsing: {e}' elif iocage_lib.ioc_common.check_truthy( conf['dhcp']) and not status: short_ip4 = "DHCP" full_ip4 = "DHCP (not running)" elif iocage_lib.ioc_common.check_truthy( conf['dhcp']) and os.geteuid() != 0: short_ip4 = "DHCP" full_ip4 = "DHCP (running -- address requires root)" # Append the JID and the NAME to the table if self.full and self.plugin: if jail_type != "plugin" and jail_type != "pluginv2": # We only want plugin type jails to be apart of the # list continue try: with open(f"{mountpoint}/plugin/ui.json", "r") as u: # We want to ensure that we show the correct NAT # ports for nat based plugins and when NAT isn't # desired, we don't show them at all. In all these # variable values, what persists across NAT/DHCP/Static # ip based plugins is that the internal ports of the # jail don't change. For example if a plugin jail has # nginx running on port 4000, it will still want to # have it running on 4000 regardless of the fact # how user configures to start the plugin jail. We take # this fact, and search for an explicit specified port # number in the admin portal, if none is found, that # means that it is ( 80 - default for http ). nat_forwards_dict = {} nat_forwards = conf.get('nat_forwards', 'none') for rule in nat_forwards.split( ',') if nat_forwards != 'none' else (): # Rule can be proto(port), proto(in/out), port if rule.isdigit(): jail = host = rule else: rule = rule.split('(')[-1].strip(')') if ':' in rule: jail, host = rule.split(':') else: # only one port provided jail = host = rule nat_forwards_dict[int(jail)] = int(host) if not conf.get('nat'): all_ips = map( lambda v: 'DHCP' if 'dhcp' in v.lower() else v, [ i.split('|')[-1].split('/')[0].strip() for i in full_ip4.split(',') ]) else: default_gateways = \ iocage_lib.ioc_common.get_host_gateways() all_ips = [ f['addr'] for k in default_gateways if default_gateways[k]['interface'] for f in netifaces.ifaddresses( default_gateways[k]['interface']) [netifaces.AF_INET if k == 'ipv4' else netifaces.AF_INET6] ] ui_data = json.load(u) admin_portal = ui_data["adminportal"] admin_portals = [] for portal in admin_portal.split(','): if conf.get('nat'): portal_uri = urllib.parse.urlparse(portal) portal_port = portal_uri.port or 80 # We do this safely as it's possible # dev hasn't added it to plugin's json yet nat_port = nat_forwards_dict.get(portal_port) if nat_port: uri = portal_uri._replace( netloc=f'{portal_uri._hostinfo[0]}:' f'{nat_port}').geturl() else: uri = portal else: uri = portal admin_portals.append(','.join( map(lambda v: uri.replace('%%IP%%', v), all_ips))) admin_portal = ','.join(admin_portals) try: ph = ui_data["adminportal_placeholders"].items() if ph and not status: admin_portal = f"{uuid} is not running!" else: for placeholder, prop in ph: admin_portal = admin_portal.replace( placeholder, iocage_lib.ioc_json.IOCJson( mountpoint).json_plugin_get_value( prop.split("."))) except KeyError: pass except iocage_lib.ioc_exceptions.CommandNeedsRoot: admin_portal = "Admin Portal requires root" except iocage_lib.ioc_exceptions.CommandFailed as e: admin_portal = b' '.join(e.message).decode() doc_url = ui_data.get('docurl', '-') except FileNotFoundError: # They just didn't set a admin portal. admin_portal = doc_url = '-' jail_list.append([ jid, uuid, boot, state, jail_type, full_release, full_ip4, ip6, template, admin_portal, doc_url ]) if self.plugin_data: if conf['plugin_repository'] not in plugin_index_data: repo_obj = iocage_lib.ioc_plugin.IOCPlugin( git_repository=conf['plugin_repository']) if not os.path.exists(repo_obj.git_destination): repo_obj.pull_clone_git_repo() with open( os.path.join(repo_obj.git_destination, 'INDEX')) as f: plugin_index_data[conf['plugin_repository']] = \ json.loads(f.read()) index_plugin_conf = plugin_index_data[ conf['plugin_repository']].get(conf['plugin_name'], {}) jail_list[-1].extend([ conf['plugin_name'], conf['plugin_repository'], index_plugin_conf.get('primary_pkg'), index_plugin_conf.get('category'), index_plugin_conf.get('maintainer'), ]) elif self.full: jail_list.append([ jid, uuid, boot, state, jail_type, full_release, full_ip4, ip6, template, basejail ]) else: jail_list.append([jid, uuid, state, short_release, short_ip4]) list_type = "list_full" if self.full else "list_short" sort = iocage_lib.ioc_common.ioc_sort(list_type, self.sort, data=jail_list) jail_list.sort(key=sort) # return the list... if not self.header: flat_jail = [j for j in jail_list] return flat_jail # Prints the table table = texttable.Texttable(max_width=0) if self.full: if self.plugin: table.set_cols_dtype( ["t", "t", "t", "t", "t", "t", "t", "t", "t", "t", "t"]) jail_list.insert(0, [ "JID", "NAME", "BOOT", "STATE", "TYPE", "RELEASE", "IP4", "IP6", "TEMPLATE", "PORTAL", "DOC_URL" ]) else: # We get an infinite float otherwise. table.set_cols_dtype( ["t", "t", "t", "t", "t", "t", "t", "t", "t", "t"]) jail_list.insert(0, [ "JID", "NAME", "BOOT", "STATE", "TYPE", "RELEASE", "IP4", "IP6", "TEMPLATE", 'BASEJAIL' ]) else: # We get an infinite float otherwise. table.set_cols_dtype(["t", "t", "t", "t", "t"]) jail_list.insert(0, ["JID", "NAME", "STATE", "RELEASE", "IP4"]) table.add_rows(jail_list) return table.draw()