def pretty_print(self, show_all=False):
"""Outputs the translation stats to a table formatted view."""
if not self.untranslated and not self.untagged:
self.log('\nNo untranslated strings found.\n')
return
# Most frequent untranslated and untagged messages.
if self.untagged:
table = texttable.Texttable(max_width=120)
table.set_deco(texttable.Texttable.HEADER)
table.set_cols_dtype(['t', 'i', 't'])
table.set_cols_align(['l', 'r', 'l'])
rows = []
missing = self.missing
for locale in missing:
for message in missing[locale]:
rows.append(
[str(locale), missing[locale][message], message])
rows = sorted(rows, key=lambda x: -x[1])
if not show_all:
num_rows = len(rows)
rows = rows[:self.ROW_COUNT]
if num_rows > self.ROW_COUNT:
rows.append([
'', num_rows - self.ROW_COUNT,
'+ Additional untranslated strings...'
])
table.add_rows(
[['Locale', '#', 'Untagged and Untranslated Message']] + rows)
self.log('\n' + table.draw() + '\n')
# Most frequent untranslated messages.
if self.untranslated:
table = texttable.Texttable(max_width=120)
table.set_deco(texttable.Texttable.HEADER)
table.set_cols_dtype(['t', 'i', 't'])
table.set_cols_align(['l', 'r', 'l'])
rows = []
for locale in self.untranslated:
for message in self.untranslated[locale]:
rows.append([
str(locale), self.untranslated[locale][message],
message
])
rows = sorted(rows, key=lambda x: -x[1])
if not show_all:
num_rows = len(rows)
rows = rows[:self.ROW_COUNT]
if num_rows > self.ROW_COUNT:
rows.append([
'', num_rows - self.ROW_COUNT,
'+ Additional untranslated strings...'
])
table.add_rows([['Locale', '#', 'Untranslated Message']] + rows)
self.log('\n' + table.draw() + '\n')
# Untranslated messages per locale.
table = texttable.Texttable(max_width=120)
table.set_deco(texttable.Texttable.HEADER)
table.set_cols_dtype(['t', 'i'])
table.set_cols_align(['l', 'r'])
rows = []
for locale in self.untranslated:
rows.append([str(locale), len(self.untranslated[locale])])
rows = sorted(rows, key=lambda x: -x[1])
table.add_rows([['Locale', 'Untranslated']] + rows)
self.log('\n' + table.draw() + '\n')
def get_table():
table = texttable.Texttable(max_width=140)
table.set_deco(0)
return table
In this project, I was able to build two different tables with prefilled user data. I was able to store all of the information
into the appropriate variables as lists of information. I then was able to gain user input for the person's name. I then performed
the logic of earnings or losses and built a new list with those given values by using the Python append() function.
I imported a library known as texttable to build the first formatted table of information and have also built a custom table beneath it.
I used a proper header with the name value and the format() functions for appending all of these values to a string of text.
Attached to this submission as well is a table that is built completely off of user input. I wanted to give it a try with functions in Python.
"""
# Import Text Table Module/Library
import texttable
# Set
table = texttable.Texttable()
# Pre-defined variables of stock information
name = input("What is your full name? ")
titles = ["Stock Symbol", "NO Shares", "Purchase Price", "Current Value", "Earnings/Losses"]
stock_symbols = ["GOOGL", "MSFT", "RDS-A", "AIG", "FB"]
number_shares = [125, 85, 400, 235, 150]
purchase_prices = [772.88, 56.60, 49.58, 54.21, 124.31]
current_prices = [941.53, 73.04, 55.74, 65.27, 172.45]
earns_losses = []
full_list = []
# The formula is (Shares own * current price) - (shares own * bought price)
def list_all(self, jails):
"""List all jails."""
table = texttable.Texttable(max_width=0)
jail_list = []
for jail in jails:
jail = jail.properties["mountpoint"].value
conf = iocage.lib.ioc_json.IOCJson(jail).json_load()
uuid = conf["host_hostuuid"]
full_ip4 = conf["ip4_addr"]
ip6 = conf["ip6_addr"]
jail_root = f"{self.pool}/iocage/jails/{uuid}/root"
try:
short_ip4 = full_ip4.split("|")[1].split("/")[0]
except IndexError:
short_ip4 = "-"
tag = conf["tag"]
boot = conf["boot"]
jail_type = conf["type"]
full_release = conf["release"]
if "HBSD" in full_release:
full_release = re.sub(r"\W\w.", "-", full_release)
full_release = full_release.replace("--SD", "-STABLE-HBSD")
short_release = full_release.rstrip("-HBSD")
else:
short_release = "-".join(full_release.rsplit("-")[:2])
if full_ip4 == "none":
full_ip4 = "-"
status, jid = self.list_get_jid(uuid)
if status:
state = "up"
else:
state = "down"
if conf["type"] == "template":
template = "-"
else:
try:
template = iocage.lib.ioc_common.checkoutput([
"zfs", "get", "-H", "-o", "value", "origin", jail_root
]).split("/")[3]
except IndexError:
template = "-"
if "release" in template.lower() or "stable" in template.lower():
template = "-"
# Append the JID and the UUID to the table
if self.full:
jail_list.append([
jid, uuid, boot, state, tag, jail_type, full_release,
full_ip4, ip6, template
])
else:
jail_list.append(
[jid, uuid[:8], state, tag, short_release, short_ip4])
list_type = "list_full" if self.full else "list_short"
sort = iocage.lib.ioc_common.ioc_sort(list_type,
self.sort,
data=jail_list)
jail_list.sort(key=sort)
# Prints the table
if self.header:
if self.full:
# We get an infinite float otherwise.
table.set_cols_dtype(
["t", "t", "t", "t", "t", "t", "t", "t", "t", "t"])
jail_list.insert(0, [
"JID", "UUID", "BOOT", "STATE", "TAG", "TYPE", "RELEASE",
"IP4", "IP6", "TEMPLATE"
])
else:
# We get an infinite float otherwise.
table.set_cols_dtype(["t", "t", "t", "t", "t", "t"])
jail_list.insert(
0, ["JID", "UUID", "STATE", "TAG", "RELEASE", "IP4"])
table.add_rows(jail_list)
return table.draw()
else:
flat_jail = [j for j in jail_list]
return flat_jail
def run(argv):
fixtures = requests.get(URL, headers=headers).json()['fixtures']
fixturesR16_json = list(filter(lambda x: x['matchday'] == 4, fixtures))
fixturesR8_json_QF = list(filter(lambda x: x['matchday'] == 5, fixtures))
fixturesR4_json_SF = list(filter(lambda x: x['matchday'] == 6, fixtures))
fixturesR2_json_F = list(filter(lambda x: x['matchday'] == 7, fixtures))
fixturesR2_json_TP = list(filter(lambda x: x['matchday'] == 8, fixtures))
fixturesR16_knockOutMatch = matchResults(fixturesR16_json)
fixturesR8_knockOutMatch_QF = matchResults(fixturesR8_json_QF)
fixturesR4_knockOutMatch_SF = matchResults(fixturesR4_json_SF)
fixturesR2_knockOutMatch_F = matchResults(fixturesR2_json_F)
fixturesR2_knockOutMatch_TP = matchResults(fixturesR2_json_TP)
# fixturesR16_knockOutMatch = list(map(
# lambda x: knockOutMatch(x['homeTeamName'], x['awayTeamName'], str(x['result']['goalsHomeTeam']) + ' ('+ str(x['result'].get('penaltyShootout', {0:0}).get('goalsHomeTeam', '') ) + ')',
# x['result']['goalsAwayTeam'], x['date']).__str__(), fixturesR16_json))
# fixturesR8_knockOutMatch_QF = list(map(
# lambda x: knockOutMatch(x['homeTeamName'], x['awayTeamName'], x['result']['goalsHomeTeam'],
# x['result']['goalsAwayTeam'], x['date']).__str__(), fixturesR8_json_QF))
# fixturesR4_knockOutMatch_SF = list(map(
# lambda x: knockOutMatch(x['homeTeamName'], x['awayTeamName'], x['result']['goalsHomeTeam'],
# x['result']['goalsAwayTeam'], x['date']).__str__(), fixturesR4_json_SF))
# fixturesR2_knockOutMatch_F = list(map(
# lambda x: knockOutMatch(x['homeTeamName'], x['awayTeamName'], x['result']['goalsHomeTeam'],
# x['result']['goalsAwayTeam'], x['date']).__str__(), fixturesR2_json_F))
# fixturesR2_knockOutMatch_TP = list(map(
# lambda x: knockOutMatch(x['homeTeamName'], x['awayTeamName'], x['result']['goalsHomeTeam'],
# x['result']['goalsAwayTeam'], x['date']).__str__(), fixturesR2_json_TP))
rightDownArrow = "\n\n_ _ _ _ _ _ .\n|\n|\n|\n|\n\u2193"
rightUpArrow = "\u2191\n|\n|\n|\n|\n_ _ _ _ _ _ .\n\n"
downDownArrow = "\n|\n|\n|\n|\n|\n|\n|\n\u2193"
upUpArrow = "\u2191\n|\n|\n|\n|\n|\n|\n|"
#
r16 = [fixturesR16_knockOutMatch[0], '', fixturesR16_knockOutMatch[1], '', fixturesR16_knockOutMatch[2], '',
fixturesR16_knockOutMatch[3], '', fixturesR16_knockOutMatch[4], '', fixturesR16_knockOutMatch[5], '',
fixturesR16_knockOutMatch[6], '', fixturesR16_knockOutMatch[7]
]
qf = [rightDownArrow, fixturesR8_knockOutMatch_QF[0], rightUpArrow,'',rightDownArrow, fixturesR8_knockOutMatch_QF[1], rightUpArrow,'',rightDownArrow,
fixturesR8_knockOutMatch_QF[2], rightUpArrow,'',rightDownArrow, fixturesR8_knockOutMatch_QF[3], rightUpArrow
]
sf = ['',rightDownArrow, downDownArrow, fixturesR4_knockOutMatch_SF[0],
upUpArrow, rightUpArrow, '','','',rightDownArrow, downDownArrow, fixturesR4_knockOutMatch_SF[1],
upUpArrow, rightUpArrow, ''
]
f = ['', '', '',rightDownArrow, downDownArrow, downDownArrow, downDownArrow,
fixturesR2_knockOutMatch_F[0],
upUpArrow, upUpArrow, upUpArrow, rightUpArrow, '', '', ''
]
l_all_matches_transpose = [r16, qf, sf, f]
l_all_matches = list(zip(*l_all_matches_transpose))
print('+---------------------------------------------------------------------------------------------------+')
print('---------------------------------FIFA WORLDCUP 2018 KNOCK-OUT STAGE----------------------------------')
table = texttable.Texttable()
table.set_cols_align(["c", "c", "c", "c"])
table.set_cols_width([23, 23, 23, 23])
table.set_deco(table.BORDER)
table.add_rows(l_all_matches, header=False)
print(table.draw())
print('---------------------------------created by harleen mann on 30-6-18----------------------------------')
async def start():
parser = argparse.ArgumentParser(
description=
'theHarvester is used to gather open source intelligence (OSINT) on a company or domain.'
)
parser.add_argument('-d',
'--domain',
help='Company name or domain to search.',
required=True)
parser.add_argument(
'-l',
'--limit',
help='Limit the number of search results, default=500.',
default=500,
type=int)
parser.add_argument('-S',
'--start',
help='Start with result number X, default=0.',
default=0,
type=int)
parser.add_argument('-g',
'--google-dork',
help='Use Google Dorks for Google search.',
default=False,
action='store_true')
parser.add_argument(
'-p',
'--proxies',
help='Use proxies for requests, enter proxies in proxies.yaml.',
default=False,
action='store_true')
parser.add_argument('-s',
'--shodan',
help='Use Shodan to query discovered hosts.',
default=False,
action='store_true')
parser.add_argument(
'--screenshot',
help=
'Take screenshots of resolved domains specify output directory: --screenshot output_directory',
default="",
type=str)
parser.add_argument(
'-v',
'--virtual-host',
help=
'Verify host name via DNS resolution and search for virtual hosts.',
action='store_const',
const='basic',
default=False)
parser.add_argument('-e',
'--dns-server',
help='DNS server to use for lookup.')
parser.add_argument(
'-t',
'--dns-tld',
help='Perform a DNS TLD expansion discovery, default False.',
default=False)
parser.add_argument('-r',
'--take-over',
help='Check for takeovers.',
default=False,
action='store_true')
parser.add_argument('-n',
'--dns-lookup',
help='Enable DNS server lookup, default False.',
default=False,
action='store_true')
parser.add_argument('-c',
'--dns-brute',
help='Perform a DNS brute force on the domain.',
default=False,
action='store_true')
parser.add_argument('-f',
'--filename',
help='Save the results to an HTML and/or XML file.',
default='',
type=str)
parser.add_argument(
'-b',
'--source',
help=
'''baidu, bing, bingapi, bufferoverun, certspotter, crtsh, dnsdumpster,
duckduckgo, exalead, github-code, google,
hackertarget, hunter, intelx, linkedin, linkedin_links, netcraft, otx, pentesttools,
qwant, rapiddns, securityTrails, spyse, sublist3r, threatcrowd, threatminer,
trello, twitter, urlscan, virustotal, yahoo''')
args = parser.parse_args()
filename: str = args.filename
dnsbrute = (args.dns_brute, False)
try:
db = stash.StashManager()
await db.do_init()
except Exception:
pass
all_emails: list = []
all_hosts: list = []
all_ip: list = []
dnslookup = args.dns_lookup
dnsserver = args.dns_server
dnstld = args.dns_tld
engines = []
# If the user specifies
full: list = []
ips: list = []
google_dorking = args.google_dork
host_ip: list = []
limit: int = args.limit
shodan = args.shodan
start: int = args.start
all_urls: list = []
vhost: list = []
virtual = args.virtual_host
word: str = args.domain
takeover_status = args.take_over
use_proxy = args.proxies
async def store(search_engine: Any,
source: str,
process_param: Any = None,
store_host: bool = False,
store_emails: bool = False,
store_ip: bool = False,
store_people: bool = False,
store_links: bool = False,
store_results: bool = False) -> None:
"""
Persist details into the database.
The details to be stored is controlled by the parameters passed to the method.
:param search_engine: search engine to fetch details from
:param source: source against which the details (corresponding to the search engine) need to be persisted
:param process_param: any parameters to be passed to the search engine eg: Google needs google_dorking
:param store_host: whether to store hosts
:param store_emails: whether to store emails
:param store_ip: whether to store IP address
:param store_people: whether to store user details
:param store_links: whether to store links
:param store_results: whether to fetch details from get_results() and persist
"""
await search_engine.process(use_proxy) if process_param is None else await \
search_engine.process(process_param, use_proxy)
db_stash = stash.StashManager()
if source == 'suip':
print(
f'\033[94m[*] Searching {source[0].upper() + source[1:]} this module can take 10+ min but is worth '
f'it. \033[0m')
else:
print(
f'\033[94m[*] Searching {source[0].upper() + source[1:]}. \033[0m'
)
if store_host:
host_names = [
host for host in filter(await search_engine.get_hostnames())
if f'.{word}' in host
]
if source != 'hackertarget' and source != 'pentesttools' and source != 'rapiddns':
# If source is inside this conditional it means the hosts returned must be resolved to obtain ip
full_hosts_checker = hostchecker.Checker(host_names)
temp_hosts, temp_ips = await full_hosts_checker.check()
ips.extend(temp_ips)
full.extend(temp_hosts)
else:
full.extend(host_names)
all_hosts.extend(host_names)
await db_stash.store_all(word, all_hosts, 'host', source)
if store_emails:
email_list = filter(await search_engine.get_emails())
all_emails.extend(email_list)
await db_stash.store_all(word, email_list, 'email', source)
if store_ip:
ips_list = await search_engine.get_ips()
all_ip.extend(ips_list)
await db_stash.store_all(word, all_ip, 'ip', source)
if store_results:
email_list, host_names, urls = await search_engine.get_results()
all_emails.extend(email_list)
host_names = [
host for host in filter(host_names) if f'.{word}' in host
]
all_urls.extend(filter(urls))
all_hosts.extend(host_names)
await db.store_all(word, all_hosts, 'host', source)
await db.store_all(word, all_emails, 'email', source)
if store_people:
people_list = await search_engine.get_people()
await db_stash.store_all(word, people_list, 'people', source)
if len(people_list) == 0:
print('\n[*] No users found.\n\n')
else:
print('\n[*] Users found: ' + str(len(people_list)))
print('---------------------')
for usr in sorted(list(set(people_list))):
print(usr)
if store_links:
links = await search_engine.get_links()
await db.store_all(word, links, 'name', engineitem)
if len(links) == 0:
print('\n[*] No links found.\n\n')
else:
print(f'\n[*] Links found: {len(links)}')
print('---------------------')
for link in sorted(list(set(links))):
print(link)
stor_lst = []
if args.source is not None:
if args.source.lower() != 'all':
engines = sorted(set(map(str.strip, args.source.split(','))))
else:
engines = Core.get_supportedengines()
# Iterate through search engines in order
if set(engines).issubset(Core.get_supportedengines()):
print(f'\033[94m[*] Target: {word} \n \033[0m')
for engineitem in engines:
if engineitem == 'baidu':
from theHarvester.discovery import baidusearch
try:
baidu_search = baidusearch.SearchBaidu(word, limit)
stor_lst.append(
store(baidu_search,
engineitem,
store_host=True,
store_emails=True))
except Exception:
pass
elif engineitem == 'bing' or engineitem == 'bingapi':
from theHarvester.discovery import bingsearch
try:
bing_search = bingsearch.SearchBing(word, limit, start)
bingapi = ''
if engineitem == 'bingapi':
bingapi += 'yes'
else:
bingapi += 'no'
stor_lst.append(
store(bing_search,
'bing',
process_param=bingapi,
store_host=True,
store_emails=True))
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
print(e)
elif engineitem == 'bufferoverun':
from theHarvester.discovery import bufferoverun
try:
bufferoverun_search = bufferoverun.SearchBufferover(
word)
stor_lst.append(
store(bufferoverun_search,
engineitem,
store_host=True,
store_ip=True))
except Exception as e:
print(e)
elif engineitem == 'certspotter':
from theHarvester.discovery import certspottersearch
try:
certspotter_search = certspottersearch.SearchCertspoter(
word)
stor_lst.append(
store(certspotter_search,
engineitem,
None,
store_host=True))
except Exception as e:
print(e)
elif engineitem == 'crtsh':
try:
from theHarvester.discovery import crtsh
crtsh_search = crtsh.SearchCrtsh(word)
stor_lst.append(
store(crtsh_search, 'CRTsh', store_host=True))
except Exception as e:
print(
f'\033[93m[!] A timeout occurred with crtsh, cannot find {args.domain}\n {e}\033[0m'
)
elif engineitem == 'dnsdumpster':
try:
from theHarvester.discovery import dnsdumpster
dns_dumpster_search = dnsdumpster.SearchDnsDumpster(
word)
stor_lst.append(
store(dns_dumpster_search,
engineitem,
store_host=True))
except Exception as e:
print(
f'\033[93m[!] An error occurred with dnsdumpster: {e} \033[0m'
)
elif engineitem == 'duckduckgo':
from theHarvester.discovery import duckduckgosearch
duckduckgo_search = duckduckgosearch.SearchDuckDuckGo(
word, limit)
stor_lst.append(
store(duckduckgo_search,
engineitem,
store_host=True,
store_emails=True))
elif engineitem == 'exalead':
from theHarvester.discovery import exaleadsearch
exalead_search = exaleadsearch.SearchExalead(
word, limit, start)
stor_lst.append(
store(exalead_search,
engineitem,
store_host=True,
store_emails=True))
elif engineitem == 'github-code':
try:
from theHarvester.discovery import githubcode
github_search = githubcode.SearchGithubCode(
word, limit)
stor_lst.append(
store(github_search,
engineitem,
store_host=True,
store_emails=True))
except MissingKey as ex:
print(ex)
else:
pass
elif engineitem == 'google':
from theHarvester.discovery import googlesearch
google_search = googlesearch.SearchGoogle(
word, limit, start)
stor_lst.append(
store(google_search,
engineitem,
process_param=google_dorking,
store_host=True,
store_emails=True))
elif engineitem == 'hackertarget':
from theHarvester.discovery import hackertarget
hackertarget_search = hackertarget.SearchHackerTarget(word)
stor_lst.append(
store(hackertarget_search, engineitem,
store_host=True))
elif engineitem == 'hunter':
from theHarvester.discovery import huntersearch
# Import locally or won't work.
try:
hunter_search = huntersearch.SearchHunter(
word, limit, start)
stor_lst.append(
store(hunter_search,
engineitem,
store_host=True,
store_emails=True))
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
elif engineitem == 'intelx':
from theHarvester.discovery import intelxsearch
# Import locally or won't work.
try:
intelx_search = intelxsearch.SearchIntelx(word, limit)
stor_lst.append(
store(intelx_search,
engineitem,
store_host=True,
store_emails=True))
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
print(
f'An exception has occurred in Intelx search: {e}'
)
elif engineitem == 'linkedin':
from theHarvester.discovery import linkedinsearch
linkedin_search = linkedinsearch.SearchLinkedin(
word, limit)
stor_lst.append(
store(linkedin_search, engineitem, store_people=True))
elif engineitem == 'linkedin_links':
from theHarvester.discovery import linkedinsearch
linkedin_links_search = linkedinsearch.SearchLinkedin(
word, limit)
stor_lst.append(
store(linkedin_links_search,
'linkedin',
store_links=True))
elif engineitem == 'netcraft':
from theHarvester.discovery import netcraft
netcraft_search = netcraft.SearchNetcraft(word)
stor_lst.append(
store(netcraft_search, engineitem, store_host=True))
elif engineitem == 'otx':
from theHarvester.discovery import otxsearch
try:
otxsearch_search = otxsearch.SearchOtx(word)
stor_lst.append(
store(otxsearch_search,
engineitem,
store_host=True,
store_ip=True))
except Exception as e:
print(e)
elif engineitem == 'pentesttools':
from theHarvester.discovery import pentesttools
try:
pentesttools_search = pentesttools.SearchPentestTools(
word)
stor_lst.append(
store(pentesttools_search,
engineitem,
store_host=True))
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
print(
f'An exception has occurred in PentestTools search: {e}'
)
elif engineitem == 'qwant':
from theHarvester.discovery import qwantsearch
qwant_search = qwantsearch.SearchQwant(word, start, limit)
stor_lst.append(
store(qwant_search,
engineitem,
store_host=True,
store_emails=True))
elif engineitem == 'rapiddns':
from theHarvester.discovery import rapiddns
try:
rapiddns_search = rapiddns.SearchRapidDns(word)
stor_lst.append(
store(rapiddns_search, engineitem,
store_host=True))
except Exception as e:
print(e)
elif engineitem == 'securityTrails':
from theHarvester.discovery import securitytrailssearch
try:
securitytrails_search = securitytrailssearch.SearchSecuritytrail(
word)
stor_lst.append(
store(securitytrails_search,
engineitem,
store_host=True,
store_ip=True))
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
elif engineitem == 'sublist3r':
from theHarvester.discovery import sublist3r
try:
sublist3r_search = sublist3r.SearchSublist3r(word)
stor_lst.append(
store(sublist3r_search,
engineitem,
store_host=True))
except Exception as e:
print(e)
elif engineitem == 'spyse':
from theHarvester.discovery import spyse
try:
spyse_search = spyse.SearchSpyse(word)
stor_lst.append(
store(spyse_search,
engineitem,
store_host=True,
store_ip=True))
except Exception as e:
print(e)
elif engineitem == 'threatcrowd':
from theHarvester.discovery import threatcrowd
try:
threatcrowd_search = threatcrowd.SearchThreatcrowd(
word)
stor_lst.append(
store(threatcrowd_search,
engineitem,
store_host=True))
except Exception as e:
print(e)
elif engineitem == 'threatminer':
from theHarvester.discovery import threatminer
try:
threatminer_search = threatminer.SearchThreatminer(
word)
stor_lst.append(
store(threatminer_search,
engineitem,
store_host=True))
except Exception as e:
print(e)
elif engineitem == 'trello':
from theHarvester.discovery import trello
# Import locally or won't work.
trello_search = trello.SearchTrello(word)
stor_lst.append(
store(trello_search, engineitem, store_results=True))
elif engineitem == 'twitter':
from theHarvester.discovery import twittersearch
twitter_search = twittersearch.SearchTwitter(word, limit)
stor_lst.append(
store(twitter_search, engineitem, store_people=True))
elif engineitem == 'urlscan':
from theHarvester.discovery import urlscan
try:
urlscan_search = urlscan.SearchUrlscan(word)
stor_lst.append(
store(urlscan_search,
engineitem,
store_host=True,
store_ip=True))
except Exception as e:
print(e)
elif engineitem == 'virustotal':
from theHarvester.discovery import virustotal
virustotal_search = virustotal.SearchVirustotal(word)
stor_lst.append(
store(virustotal_search, engineitem, store_host=True))
elif engineitem == 'yahoo':
from theHarvester.discovery import yahoosearch
yahoo_search = yahoosearch.SearchYahoo(word, limit)
stor_lst.append(
store(yahoo_search,
engineitem,
store_host=True,
store_emails=True))
else:
print('\033[93m[!] Invalid source.\n\n \033[0m')
sys.exit(1)
async def worker(queue):
while True:
# Get a "work item" out of the queue.
stor = await queue.get()
try:
await stor
queue.task_done()
# Notify the queue that the "work item" has been processed.
except Exception:
queue.task_done()
async def handler(lst):
queue = asyncio.Queue()
for stor_method in lst:
# enqueue the coroutines
queue.put_nowait(stor_method)
# Create five worker tasks to process the queue concurrently.
tasks = []
for i in range(5):
task = asyncio.create_task(worker(queue))
tasks.append(task)
# Wait until the queue is fully processed.
await queue.join()
# Cancel our worker tasks.
for task in tasks:
task.cancel()
# Wait until all worker tasks are cancelled.
await asyncio.gather(*tasks, return_exceptions=True)
await handler(lst=stor_lst)
# Sanity check to see if all_emails and all_hosts are defined.
try:
all_emails
except NameError:
print(
'\n\n\033[93m[!] No emails found because all_emails is not defined.\n\n \033[0m'
)
sys.exit(1)
try:
all_hosts
except NameError:
print(
'\n\n\033[93m[!] No hosts found because all_hosts is not defined.\n\n \033[0m'
)
sys.exit(1)
# Results
if len(all_ip) == 0:
print('\n[*] No IPs found.')
else:
print('\n[*] IPs found: ' + str(len(all_ip)))
print('-------------------')
# use netaddr as the list may contain ipv4 and ipv6 addresses
ip_list = sorted([netaddr.IPAddress(ip.strip()) for ip in set(all_ip)])
print('\n'.join(map(str, ip_list)))
if len(all_emails) == 0:
print('\n[*] No emails found.')
else:
print('\n[*] Emails found: ' + str(len(all_emails)))
print('----------------------')
print(('\n'.join(sorted(list(set(all_emails))))))
if len(all_hosts) == 0:
print('\n[*] No hosts found.\n\n')
else:
print('\n[*] Hosts found: ' + str(len(all_hosts)))
print('---------------------')
all_hosts = sorted(list(set(all_hosts)))
db = stash.StashManager()
full = [
host if ':' in host and word in host else
word in host.split(':')[0] and host for host in full
]
full = list({host for host in full if host})
full.sort(key=lambda el: el.split(':')[0])
for host in full:
print(host)
host_ip = [
netaddr_ip.format()
for netaddr_ip in sorted([netaddr.IPAddress(ip) for ip in ips])
]
await db.store_all(word, host_ip, 'ip', 'DNS-resolver')
length_urls = len(all_urls)
if length_urls == 0:
if len(engines) >= 1 and 'trello' in engines:
print('\n[*] No Trello URLs found.')
else:
total = length_urls
print('\n[*] Trello URLs found: ' + str(total))
print('--------------------')
for url in sorted(all_urls):
print(url)
# DNS brute force
if dnsbrute and dnsbrute[0] is True:
print('\n[*] Starting DNS brute force.')
dns_force = dnssearch.DnsForce(word, dnsserver, verbose=True)
hosts, ips = await dns_force.run()
hosts = list({host for host in hosts if ':' in host})
hosts.sort(key=lambda el: el.split(':')[0])
print('\n[*] Hosts found after DNS brute force:')
db = stash.StashManager()
for host in hosts:
print(host)
full.append(host)
await db.store_all(word, hosts, 'host', 'dns_bruteforce')
# TakeOver Checking
if takeover_status:
print('\n[*] Performing subdomain takeover check')
print('\n[*] Subdomain Takeover checking IS ACTIVE RECON')
search_take = takeover.TakeOver(all_hosts)
await search_take.process(proxy=use_proxy)
# DNS reverse lookup
dnsrev = []
if dnslookup is True:
print('\n[*] Starting active queries.')
# load the reverse dns tools
from theHarvester.discovery.dnssearch import (
generate_postprocessing_callback, reverse_all_ips_in_range,
serialize_ip_range)
# reverse each iprange in a separate task
__reverse_dns_tasks = {}
for entry in host_ip:
__ip_range = serialize_ip_range(ip=entry, netmask='24')
if __ip_range and __ip_range not in set(
__reverse_dns_tasks.keys()):
print('\n[*] Performing reverse lookup on ' + __ip_range)
__reverse_dns_tasks[__ip_range] = asyncio.create_task(
reverse_all_ips_in_range(
iprange=__ip_range,
callback=generate_postprocessing_callback(
target=word,
local_results=dnsrev,
overall_results=full),
nameservers=list(map(str, dnsserver.split(',')))
if dnsserver else None))
# run all the reversing tasks concurrently
await asyncio.gather(*__reverse_dns_tasks.values())
# Display the newly found hosts
print('\n[*] Hosts found after reverse lookup (in target domain):')
print('--------------------------------------------------------')
for xh in dnsrev:
print(xh)
# DNS TLD expansion
dnstldres = []
if dnstld is True:
print('[*] Starting DNS TLD expansion.')
a = dnssearch.DnsTld(word, dnsserver, verbose=True)
res = a.process()
print('\n[*] Hosts found after DNS TLD expansion:')
print('----------------------------------------')
for y in res:
print(y)
dnstldres.append(y)
if y not in full:
full.append(y)
# Virtual hosts search
if virtual == 'basic':
print('\n[*] Virtual hosts:')
print('------------------')
for data in host_ip:
basic_search = bingsearch.SearchBing(data, limit, start)
await basic_search.process_vhost()
results = await basic_search.get_allhostnames()
for result in results:
result = re.sub(r'[[</?]*[\w]*>]*', '', result)
result = re.sub('<', '', result)
result = re.sub('>', '', result)
print((data + '\t' + result))
vhost.append(data + ':' + result)
full.append(data + ':' + result)
vhost = sorted(set(vhost))
else:
pass
# Screenshots
screenshot_tups = []
if len(args.screenshot) > 0:
import time
from aiomultiprocess import Pool
from theHarvester.screenshot.screenshot import ScreenShotter
screen_shotter = ScreenShotter(args.screenshot)
path_exists = screen_shotter.verify_path()
# Verify path exists if not create it or if user does not create it skip screenshot
if path_exists:
await screen_shotter.verify_installation()
print(
f'\nScreenshots can be found in: {screen_shotter.output}{screen_shotter.slash}'
)
start_time = time.perf_counter()
print('Filtering domains for ones we can reach')
unique_resolved_domains = {
url.split(':')[0]
for url in full if ':' in url and 'www.' not in url
}
if len(unique_resolved_domains) > 0:
# First filter out ones that didn't resolve
print(
'Attempting to visit unique resolved domains, this is ACTIVE RECON'
)
async with Pool(15) as pool:
results = await pool.map(screen_shotter.visit,
list(unique_resolved_domains))
# Filter out domains that we couldn't connect to
unique_resolved_domains = list(
sorted({tup[0]
for tup in results if len(tup[1]) > 0}))
async with Pool(3) as pool:
print(
f'Length of unique resolved domains: {len(unique_resolved_domains)} chunking now!\n'
)
# If you have the resources you could make the function faster by increasing the chunk number
chunk_number = 25
for chunk in screen_shotter.chunk_list(
unique_resolved_domains, chunk_number):
try:
screenshot_tups.extend(await pool.map(
screen_shotter.take_screenshot, chunk))
except Exception as ee:
print(
f'An exception has occurred while mapping: {ee}'
)
end = time.perf_counter()
# There is probably an easier way to do this
total = end - start_time
mon, sec = divmod(total, 60)
hr, mon = divmod(mon, 60)
total_time = "%02d:%02d" % (mon, sec)
print(f'Finished taking screenshots in {total_time} seconds')
print(
'[+] Note there may be leftover chrome processes you may have to kill manually\n'
)
# Shodan
shodanres = []
if shodan is True:
import texttable
tab = texttable.Texttable()
header = [
'IP address', 'Hostname', 'Org', 'Services:Ports', 'Technologies'
]
tab.header(header)
tab.set_cols_align(['c', 'c', 'c', 'c', 'c'])
tab.set_cols_valign(['m', 'm', 'm', 'm', 'm'])
tab.set_chars(['-', '|', '+', '#'])
tab.set_cols_width([15, 20, 15, 15, 18])
print('\033[94m[*] Searching Shodan. \033[0m')
try:
for ip in host_ip:
print(('\tSearching for ' + ip))
shodan = shodansearch.SearchShodan()
rowdata = await shodan.search_ip(ip)
await asyncio.sleep(2)
tab.add_row(rowdata)
printedtable = tab.draw()
print(printedtable)
except Exception as e:
print(f'\033[93m[!] An error occurred with Shodan: {e} \033[0m')
else:
pass
# Here we need to add explosion mode.
# We have to take out the TLDs to do this.
if args.dns_tld is not False:
counter = 0
for word in vhost:
search = googlesearch.SearchGoogle(word, limit, counter)
await search.process(google_dorking)
emails = await search.get_emails()
hosts = await search.get_hostnames()
print(emails)
print(hosts)
else:
pass
# Reporting
if filename != "":
try:
print('\n[*] Reporting started.')
db = stash.StashManager()
scanboarddata = await db.getscanboarddata()
latestscanresults = await db.getlatestscanresults(word)
previousscanresults = await db.getlatestscanresults(
word, previousday=True)
latestscanchartdata = await db.latestscanchartdata(word)
scanhistorydomain = await db.getscanhistorydomain(word)
pluginscanstatistics = await db.getpluginscanstatistics()
generator = statichtmlgenerator.HtmlGenerator(word)
html_code = await generator.beginhtml()
html_code += await generator.generatedashboardcode(scanboarddata)
html_code += await generator.generatelatestscanresults(
latestscanresults)
if len(screenshot_tups) > 0:
html_code += await generator.generatescreenshots(
screenshot_tups)
html_code += await generator.generatepreviousscanresults(
previousscanresults)
graph = reportgraph.GraphGenerator(word)
await graph.init_db()
html_code += await graph.drawlatestscangraph(
word, latestscanchartdata)
html_code += await graph.drawscattergraphscanhistory(
word, scanhistorydomain)
html_code += await generator.generatepluginscanstatistics(
pluginscanstatistics)
html_code += '<p><span style="color: #000000;">Report generated on ' + str(
datetime.datetime.now()) + '</span></p>'
html_code += '''
</body>
</html>
'''
except Exception as e:
print(e)
print(
'\n\033[93m[!] An error occurred while creating the output file.\n\n \033[0m'
)
sys.exit(1)
html_file = open(
f'{filename}.html' if '.html' not in filename else filename, 'w')
html_file.write(html_code)
html_file.close()
print('[*] Reporting finished.')
print('[*] Saving files.')
try:
filename = filename.rsplit('.', 1)[0] + '.xml'
with open(filename, 'w+') as file:
file.write(
'<?xml version="1.0" encoding="UTF-8"?><theHarvester>')
for x in all_emails:
file.write('<email>' + x + '</email>')
for x in full:
host, ip = x.split(':', 1) if ':' in x else (x, '')
if ip and len(ip) > 3:
file.write(
f'<host><ip>{ip}</ip><hostname>{host}</hostname></host>'
)
else:
file.write(f'<host>{host}</host>')
for x in vhost:
host, ip = x.split(':', 1) if ':' in x else (x, '')
if ip and len(ip) > 3:
file.write(
f'<vhost><ip>{ip} </ip><hostname>{host}</hostname></vhost>'
)
else:
file.write(f'<vhost>{host}</vhost>')
if shodanres != []:
shodanalysis = []
for x in shodanres:
res = x.split('SAPO')
file.write('<shodan>')
file.write('<host>' + res[0] + '</host>')
file.write('<port>' + res[2] + '</port>')
file.write('<banner><!--' + res[1] + '--></banner>')
reg_server = re.compile('Server:.*')
temp = reg_server.findall(res[1])
if temp:
shodanalysis.append(res[0] + ':' + temp[0])
file.write('</shodan>')
if shodanalysis:
shodanalysis = sorted(set(shodanalysis))
file.write('<servers>')
for x in shodanalysis:
file.write('<server>' + x + '</server>')
file.write('</servers>')
file.write('</theHarvester>')
print('[*] Files saved.')
except Exception as er:
print(
f'\033[93m[!] An error occurred while saving the XML file: {er} \033[0m'
)
print('\n\n')
sys.exit(0)
],
[
Embedding(max_dict_words + skip_common,
500,
input_length=max_sent_words),
LSTM(64)
],
[
Embedding(max_dict_words + skip_common,
1000,
input_length=max_sent_words),
LSTM(64)
],
]
batch_sizes = [256]
optimizers = [keras.optimizers.Adam()]
epochs = [15]
model = RNNGridSearch(archs, epochs, batch_sizes, optimizers, 'sentiment1.csv')
histories = model.run(x_train, y_train, x_test, y_test)
result = model.best_model()
hist = result.pop('history_loss')
result.pop('history_score')
print('Best model is: ')
tbl = texttable.Texttable()
tbl.set_cols_align(["c", "c"])
tbl.set_cols_valign(["c", "c"])
tbl.add_rows([['Hyperparameter', 'Best value'], *list(result.items())])
print(tbl.draw())
def report_end(bids, pyps):
bidders = sorted({b['bidder_name']: True for b in bids}.keys())
for bidder in bidders:
address = ""
item_counts = {}
won_message = []
shipping = {}
bb = [b for b in bids if b['bidder_name'] == bidder]
userid = bb[0]['bidder_url'].split('=')[-1]
contact_url = "https://truedungeon.com/component/uddeim/?task=new&recip=%s" % (
userid)
for bid in sorted(bb, key=operator.itemgetter('item')):
won_message.append("Auction %s: %s" %
(bid['auction'], bid['message']))
if bid['item'] in item_counts:
item_counts[bid['item']] += bid['won_quantity']
else:
item_counts[bid['item']] = bid['won_quantity']
if bid['address']:
address = bid['address']
pyp_choices = []
pyp_won = 0
pyp_text = ""
for pyp in pyps:
try:
if pyp['bidder_url'] == bb[0]['bidder_url']:
pyp_won += pyp['won_quantity']
for k, v in pyp.items():
if k.startswith('ur'):
if v is not None and v != '':
pyp_choices.append(v)
except Exception as e:
raise e
#import pdb
#pdb.set_trace()
#import pdb
#pdb.set_trace()
#1+1
if pyp_won != 0 or pyp_choices != []:
pyp_text = "%d PyP selections which were:\n%s" % (
pyp_won, "\n".join(sorted(pyp_choices)))
dt = texttable.Texttable()
dt.set_cols_align(['r', 'l'])
dt.set_cols_dtype(['t', 't'])
dt.set_deco(texttable.Texttable.HEADER)
line_items = [['Qty', 'Item']]
for ic in sorted(item_counts.keys()):
line_items.append([item_counts[ic], ic])
dt.add_rows(line_items)
# DEBUG - this is a nightmare - skip it - it will be faster to just print the ~50 labels one by one.
'''
shipping['Order ID (required)'] = bidder
shipping['Order Date'] = datetime.datetime.today().strftime( '%m/%d/%Y' )
# Dictates insurance?
shipping['Order Value'] = 0
shipping['Requested Service'] = 'Standard Shipping'
# I need to ensure my addresses are formatted as:
# full name\nstreet\ncity, state zip
address_parts = address.split( '\n' )
shipping['Ship To - Name'] = address_parts[0]
shipping['Ship To - Address 1'] = address_parts[1]
city, rest = address_parts[2].split( ', ' )
state = rest[:2]
zipcode = rest[3:]
shipping['Ship To - State/Province'] = state
shipping['Ship To - City'] = city
shipping['Ship To - Postal Code'] = zipcode
'''
details = "\n".join(sorted(won_message))
print "-" * 80, "\n", contact_url, "\n", "Auction Items for: %s\n\nPlease verify your address and won items below, if everything is correct no need to respond. If not, please let me know!\n\n%s\n\n%s\n\n%s\n\nAuction Breakdown:\n%s\n\n" % (
bidder, address, pyp_text, dt.draw(), details)
def main():
"""Main function."""
args = parse_arguments()
start = time.time()
print("Expected runtime: 20 s on ATIS dataset")
grammar, sents = data_load(args.grammar_f, args.sents_f)
test_sents = nltk.parse.util.extract_test_sentences(sents)
# Display sentences and counts in a tabulated manner using texttable
# References: https://pypi.python.org/pypi/texttable
t = texttable.Texttable()
length = [5, 70, 5, 10, 15]
t.set_cols_width(length)
rows = [[
"S.No.", "test sentence", "CFG", "parse tree counts",
"simple parse tree counts (doesn't compute parse trees)"
]]
cyk_runtime = 0
bp_runtime = 0
for index, sent in enumerate(test_sents):
tokens = sent[0]
sentence = " ".join(tokens)
if args.show_chart or args.show_tree:
print("(", index, ")", " ".join(tokens), "\n")
root = grammar.start()
cyk_start = time.time()
chart, table = cky_parser(tokens, grammar)
cyk_end = time.time()
cyk_runtime += cyk_end - cyk_start
tree_counts = 0
tree_counts_simple = 0
next_row = [
str(index + 1),
str(sentence),
str("False"),
str(tree_counts),
str(tree_counts_simple)
]
if cky_recognizer(chart, root, args.show_chart, args.show_tree):
if args.show_chart:
print_chart(chart, tokens)
bp_start = time.time()
tree_counts, tree_counts_simple = print_parsed_tree(
table, tokens, root, args.show_tree)
bp_end = time.time()
bp_runtime = bp_end - bp_start
next_row = [
str(index + 1),
str(sentence),
str("True"),
str(tree_counts),
str(tree_counts_simple)
]
if args.show_chart or args.show_tree:
dash = "-=" * 40
print(dash, '\n')
rows.append(next_row)
if args.show_summary:
t.add_rows(rows)
print(t.draw())
end = time.time()
print("Total runtime: %.3f s" % (end - start))
print("CYK parser runtime: %.3f s" % cyk_runtime)
print("Backpointer runtime: %.6f s" % bp_runtime)
def format_diff_for_console(cls, list_of_results):
"""
create a table with io performance report
for console
"""
tab = texttable.Texttable(max_width=200)
tab.set_deco(tab.HEADER | tab.VLINES | tab.BORDER)
header = [
cls.fiels_and_header_dct["name"].header,
cls.fiels_and_header_dct["summ"].header,
]
allign = ["l", "l"]
header.append("IOPS ~ Cnf% ~ Dev%")
allign.extend(["r"] * len(list_of_results))
header.extend("IOPS_{0} %".format(i + 2)
for i in range(len(list_of_results[1:])))
header.append("BW")
allign.extend(["r"] * len(list_of_results))
header.extend("BW_{0} %".format(i + 2)
for i in range(len(list_of_results[1:])))
header.append("LAT")
allign.extend(["r"] * len(list_of_results))
header.extend("LAT_{0}".format(i + 2)
for i in range(len(list_of_results[1:])))
tab.header(header)
sep = ["-" * 3] * len(header)
processed_results = map(cls.prepare_data, list_of_results)
key2results = []
for res in processed_results:
key2results.append(
dict(((item["name"], item["summ"]), item) for item in res))
prev_k = None
iops_frmt = "{0[iops]} ~ {0[conf]:>2} ~ {0[dev]:>2}"
for item in processed_results[0]:
if prev_k is not None:
if prev_k != item["key"]:
tab.add_row(sep)
prev_k = item["key"]
key = (item['name'], item['summ'])
line = list(key)
base = key2results[0][key]
line.append(iops_frmt.format(base))
for test_results in key2results[1:]:
val = test_results.get(key)
if val is None:
line.append("-")
elif base['iops'] == 0:
line.append("Nan")
else:
prc_val = {'dev': val['dev'], 'conf': val['conf']}
prc_val['iops'] = int(100 * val['iops'] / base['iops'])
line.append(iops_frmt.format(prc_val))
line.append(base['bw'])
for test_results in key2results[1:]:
val = test_results.get(key)
if val is None:
line.append("-")
elif base['bw'] == 0:
line.append("Nan")
else:
line.append(int(100 * val['bw'] / base['bw']))
for test_results in key2results:
val = test_results.get(key)
if val is None:
line.append("-")
else:
line.append("{0[lat_50]} - {0[lat_95]}".format(val))
tab.add_row(line)
tab.set_cols_align(allign)
return tab.draw()
async def stats(self, ctx):
try:
sql = "SELECT `messages_id` FROM `messages` WHERE action=0 ORDER BY `messages_id` DESC LIMIT 1"
message_count = self.cursor.execute(
sql).fetchall()[0]['messages_id']
sql = "SELECT COUNT(`action`) FROM `messages` WHERE action=1"
message_delete_count = self.cursor.execute(
sql).fetchall()[0]['COUNT(`action`)']
sql = "SELECT COUNT(`action`) FROM `messages` WHERE action=2"
message_edit_count = self.cursor.execute(
sql).fetchall()[0]['COUNT(`action`)']
sql = "SELECT COUNT(`command`) FROM `command_logs`"
command_count = self.cursor.execute(
sql).fetchall()[0]['COUNT(`command`)']
sql = 'SELECT COUNT(`command`) FROM `command_logs` WHERE server={0}'.format(
ctx.message.server.id)
server_command_count = self.cursor.execute(
sql).fetchall()[0]['COUNT(`command`)']
sql = "SELECT `command`, COUNT(`command`) AS magnitude FROM `command_logs` GROUP BY `command` ORDER BY magnitude DESC LIMIT 6"
command_magnitude = self.cursor.execute(sql).fetchall()
sql = 'SELECT `server`, `server_name`, COUNT(`command`) AS magnitude FROM `command_logs` GROUP BY `server` ORDER BY magnitude DESC LIMIT 5'
server_magnitude = self.cursor.execute(sql).fetchall()
sql = 'SELECT * FROM `stats`'
results = self.cursor.execute(sql).fetchall()
server_list = []
for shard in results:
server_list.append([
shard['largest_member_server_name'],
shard['largest_member_server']
])
count = 0
counts = []
for x in server_list:
count = int(x[1])
counts.append(count)
max_ = int(max(counts))
max_index = int(counts.index(max_))
biggest_server_name = server_list[max_index][0]
biggest_server_count = server_list[max_index][1]
magnitude_table = texttable.Texttable(max_width=90)
# magnitude_table.set_cols_width([90, 90])
for x in server_magnitude:
magnitude_table.add_rows(
[["Server", "Commands"],
[x['server_name'][:25], x['magnitude']]])
magnitude_msg = magnitude_table.draw()
command_table = texttable.Texttable(max_width=90)
# command_table.set_cols_width([90, 90])
for x in command_magnitude:
command_table.add_rows([["Command", "Count"],
[x['command'], x['magnitude']]])
command_msg = command_table.draw()
command_stats_msg = magnitude_msg + '\n\n' + command_msg
text_channels = 0
voice_channels = 0
for shard in results:
text_channels += int(shard['text_channels'])
voice_channels += int(shard['voice_channels'])
user_count = 0
unique_users = 0
server_count = 0
for shard in results:
server_count += int(shard['servers'])
user_count += int(shard['users'])
unique_users += int(shard['unique_users'])
seconds = time.time() - start_time
m, s = divmod(seconds, 60)
h, m = divmod(m, 60)
d, h = divmod(h, 24)
w, d = divmod(d, 7)
if s != 0:
uptime = '**{0}** seconds.'.format(int(s))
if m != 0:
uptime = ' : **{0}** minutes : '.format(int(m)) + uptime
if h != 0:
uptime = ' : **{0}** hours'.format(int(h)) + uptime
if d != 0:
uptime = ' : **{0}** days'.format(int(d)) + uptime
if w != 0:
uptime = ' : **{0}** weeks {1}'.format(int(w)) + uptime
if m == 0:
uptime = ' ' + uptime
else:
uptime = uptime[2:]
msg = ":bar_chart: **User/Bot Statistics**\n"
msg += "> Uptime: " + uptime + "\n"
msg += "> On **{0}** Servers\n".format(server_count)
msg += "> **{0}** Text channels | **{1}** Voice\n".format(
text_channels, voice_channels)
msg += "> Serving **{0}** Users\n".format(user_count)
msg += "> Unique Users: **{0}**\n".format(unique_users)
msg += "> Who've messaged **{0}** times ".format(message_count)
msg += "where **{0}** of them have been edited ".format(
message_edit_count)
msg += "and **{0}** deleted.\n".format(message_delete_count)
msg += "> In total **{0}** commands have been called, __{1}__ from this server.\n".format(
command_count, server_command_count)
msg += ':keyboard: **Command Statistics**\n'
msg += "```\n{0}```".format(command_stats_msg)
msg += ':desktop: **Server Statistics**\n'
msg += '> Largest Server: **{0}** (Users: **{1}**)\n'.format(
biggest_server_name, biggest_server_count)
msg += '> Most used on: **{0}** (Commands: **{1}**/{2})\n'.format(
server_magnitude[0]['server_name'],
server_magnitude[0]['magnitude'], command_count)
# msg += '> Server with most messages: *{0}* (Messages: **{1}/{2}**)'
await self.bot.say(msg)
except Exception as e:
print(e)
def pretty_print_table(group_name, table):
print('\n\n*** {} ***'.format(group_name))
tt = texttable.Texttable()
tt.add_rows(table)
print(tt.draw())
def to_tables(self):
results = []
table = texttable.Texttable(max_width=0)
table.set_deco(texttable.Texttable.HEADER)
rows = []
rows.append(['Resource', 'Count'])
all_collections = self.pod.list_collections()
rows.append(['Collections', len(all_collections)])
documents = []
for collection in all_collections:
documents += collection.list_docs()
rows.append(['Documents', len(documents)])
locales = self.pod.list_locales()
rows.append(['Locales', len(locales)])
if self.pod.use_reroute:
self.pod.router.use_simple()
self.pod.router.add_all()
rows.append(['Routes', len(self.pod.router.routes)])
else:
routes = self.pod.routes
rows.append(['Routes', len(routes.list_concrete_paths())])
template = self.pod.catalogs.get_template()
rows.append(['Messages', len(template)])
table.add_rows(rows)
content = table.draw()
results.append(content)
if self.full:
table = texttable.Texttable(max_width=0)
table.set_deco(texttable.Texttable.HEADER)
rows = []
rows.append(['File type', 'Count'])
exts_and_counts = self.get_num_files_per_type()
exts_and_counts = sorted(exts_and_counts,
key=lambda message: -message.count)
for message in exts_and_counts:
ext = message.ext or '.html'
rows.append([ext, message.count])
table.add_rows(rows)
content = table.draw()
results.append(content)
table = texttable.Texttable(max_width=0)
table.set_deco(texttable.Texttable.HEADER)
catalogs = sorted(self.pod.catalogs, key=str)
rows = []
rows.append(['Translations ({})'.format(len(catalogs)), 'Messages'])
for catalog in catalogs:
num_messages = len(catalog)
untranslated_messages = catalog.list_untranslated()
translated_messages = num_messages - len(untranslated_messages)
label = '{} / {}'.format(translated_messages, num_messages)
rows.append([str(catalog.locale), label])
table.add_rows(rows)
if catalogs:
content = table.draw()
results.append(content)
return results
import json
import sys
import texttable
import operator
input_file = sys.argv[1]
output_file = sys.argv[2]
with open(input_file) as input:
full_dict = json.load(input)
#DOMAIN TABLE-------------------------------------------------------------------------
domain_table = texttable.Texttable(375)
domain_table.set_cols_align(
["l", "c", "c", "c", "c", "c", "c", "c", "c", "c", "c", "c", "c"])
domain_table.set_cols_valign(
["t", "t", "t", "t", "t", "t", "t", "t", "t", "t", "t", "t", "t"])
domain_table.add_row([
"Domain", "Scan Time", "IPv4\nAddresses", "IPv6\nAddresses",
"HTTP\nServer", "Insecure\nHTTP", "Redirects", "Hosts",
"TLS\nVersions", "Root CA", "RDNS Names", "RTT Range", "Geo\nLocations"
])
for domain in full_dict:
row = []
row.append(domain)
for i in full_dict[domain]:
timings = {}
# Receive results
bss = BSSeval([
'mir_eval_images',
'mir_eval_images_noperm',
'bss_eval_matlab_images',
'mir_eval_sources',
'bss_eval_matlab_sources',
])
results = bss.evaluate(est_audio, src_audio, src_rate)
# Try for a readable output
try:
import texttable as txtbl
table = txtbl.Texttable()
table.set_cols_dtype([
't', # method used
'f', # SDR
'f', # SIR
'f', # ISR
'f'
]) # SAR
table.set_cols_align(["c", "c", "c", "c", "c"])
table.header(["Method", "SDR", "SIR", "ISR", "SAR"])
for result in results:
method = result['Method']
try:
sources = len(result['SDR'])
except:
sources = 1
print('ROI crop selected')
(input_dir,output_dir)=in_out()
#ROI class call
Crop(input_dir, output_dir).ROI()
#Spot crop selection
elif select_def=='S':
print('Spot crop selected')
(input_dir,output_dir)=in_out()
#Spot class call
Crop(input_dir, output_dir).Spot()
#Help selection
elif select_def=='H':
print('Help selected')
table = tt.Texttable()
table.header(["Instructions"])
table.add_row(["The present code allows to manage MIRAX files of histological samples. Two possible solutions were implemented:\n\n 1) Select a specific region of interest(ROI) from the whole slide image (WSI) and then cut it in sub-images\n 2) Find the spots in a TMA sample and cut them in sigle images"])
table.add_row(["Prerequisites:\n1) Matlab\n - MATLAB Engine API for Python\n2) Python3 \n - openslide,sys, os, time, csv, pandas, numpy, texttable"])
table.add_row(["How to run:\n - Open the terminal in the code folder and write:\n\n\tPython CropCode.py\n\nThen follow simply answer to the question!"])
table.add_row(["ROI: The ROI option allows to select a specif area and then cut it in sub-images. This can be useful in case it is necessary to select only a portion of the WSI. The following cut is implemented in order to work with the image that problably is in the order of ~GB.\n"\
"\nThe program will ask you:\n - Input directory\n - Output directory\n - If you want only the files with the dimension of the ROI or if you want to cut it directly with this program\n - Dimesion of the square used for cut the ROI, in case your answer was 'y' (yes) to the previous question\n\nA GUI is used to select the ROI.\n\nSuggestion: check the size in pixel of the WSI and then choose the dimension of the square according to it."\
"It is also better select the ROI considering the total width and white contours over and under of the WSI."])
table.add_row(["Spot: The Spot option allows to reconize the spots in a TMA and cut them in order to obtain sigle image of spot per TMAs. If a document with the ID patients is attached it is possible to save each spot with the proper ID and in the corresponding folder per patient."\
" if this is not the casey, the code will name the spots with a number from the bottom left to the upper right.\nThe program will ask you:\n\n - Input directory\n - Output directory\n - Number of row in the TMA (remember to choose the maximum value considering all the TMAs)\n - Number of column in the TMA (remember to choose the maximum value considering all the TMAs)\n - Dimension of the radius in case the detected matrix of the spots is not correct"\
"\n\nA GUI is used to check if the program have found all the spots and, in case, allows to add or remove incorrect elements.\n\nSuggestion: Considering the IDpatient file, take a look to the example given with this code!"])
table.set_cols_width([100])
print(table.draw())
#In case of unknown character
else:
print('ERROR!Command not reconized!')
def start():
parser = argparse.ArgumentParser(
description=
'theHarvester is used to gather open source intelligence on a company or domain.'
)
parser.add_argument('-d',
'--domain',
help='Company or domain to search',
required=True)
parser.add_argument('-l',
'--limit',
help='Limit the number of search results, default=500',
default=500,
type=int)
parser.add_argument('-S',
'--start',
help='Start with result number X, default=0',
default=0,
type=int)
parser.add_argument('-g',
'--google-dork',
help='Use Google Dorks for Google search',
default=False,
action='store_true')
parser.add_argument(
'-p',
'--port-scan',
help=
'Scan discovered hosts and check for Takeovers (21,22,80,443,8080) default=False, params=True',
default=False)
parser.add_argument('-s',
'--shodan',
help='Use Shodan to query discovered hosts',
default=False,
action='store_true')
parser.add_argument(
'-v',
'--virtual-host',
help=
'Verify host name via DNS resolution and search for virtual hosts params=basic, default=False',
default=False)
parser.add_argument('-e',
'--dns-server',
help='Specify a DNS server to use for lookup')
parser.add_argument('-c',
'--dns-brute',
help='Perform a DNS brute force on a domain',
default=False,
action='store_true')
parser.add_argument(
'-n',
'--dns-lookup',
help='Enable a DNS server lookup, default=False, params=True',
default=False)
parser.add_argument(
'-t',
'--dns-tld',
help='Perform a DNS TLD expansion discovery, default False',
default=False)
parser.add_argument('-f',
'--filename',
help='Save the results to an HTML and/or XML file',
default='',
type=str)
parser.add_argument(
'-b',
'--source',
help='''Source: baidu, bing, bingapi, censys, crtsh, cymon,
dogpile, duckduckgo, google, googleCSE,
google-certificates, hunter, linkedin,
netcraft, pgp, securityTrails, threatcrowd,
trello, twitter, vhost, virustotal, yahoo, all''')
args = parser.parse_args()
try:
db = stash.stash_manager()
db.do_init()
except Exception:
pass
all_emails = []
all_hosts = []
all_ip = []
dnsbrute = args.dns_brute
dnslookup = args.dns_lookup
dnsserver = args.dns_server
dnstld = args.dns_tld
filename = args.filename
full = []
google_dorking = args.google_dork
host_ip = []
limit = args.limit
ports_scanning = args.port_scan
shodan = args.shodan
start = args.start
takeover_check = False
trello_info = ([], False)
vhost = []
virtual = args.virtual_host
word = args.domain
if args.source is not None:
engines = set(args.source.split(','))
if set(engines).issubset(Core.get_supportedengines()):
print(f'\033[94m[*] Target: {word} \n \033[0m')
for engineitem in engines:
if engineitem == 'baidu':
print('\033[94m[*] Searching Baidu. \033[0m')
try:
search = baidusearch.SearchBaidu(word, limit)
search.process()
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'baidu')
db.store_all(word, all_emails, 'email', 'baidu')
except Exception:
pass
elif engineitem == 'bing' or engineitem == 'bingapi':
print('\033[94m[*] Searching Bing. \033[0m')
try:
search = bingsearch.SearchBing(word, limit, start)
bingapi = ''
if engineitem == 'bingapi':
bingapi += 'yes'
else:
bingapi += 'no'
search.process(bingapi)
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'bing')
db.store_all(word, all_hosts, 'host', 'bing')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
elif engineitem == 'censys':
print('\033[94m[*] Searching Censys. \033[0m')
from discovery import censys
# Import locally or won't work
search = censys.SearchCensys(word, limit)
search.process()
all_ip = search.get_ipaddresses()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'censys')
db.store_all(word, all_ip, 'ip', 'censys')
elif engineitem == 'crtsh':
print('\033[94m[*] Searching CRT.sh. \033[0m')
search = crtsh.search_crtsh(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'CRTsh')
elif engineitem == 'cymon':
print('\033[94m[*] Searching Cymon. \033[0m')
from discovery import cymon
# Import locally or won't work.
search = cymon.search_cymon(word)
search.process()
all_ip = search.get_ipaddresses()
db = stash.stash_manager()
db.store_all(word, all_ip, 'ip', 'cymon')
elif engineitem == 'dogpile':
try:
print('\033[94m[*] Searching Dogpile. \033[0m')
search = dogpilesearch.SearchDogpile(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'dogpile')
db.store_all(word, all_hosts, 'host', 'dogpile')
except Exception as e:
print(
f'\033[93m[!] A error occurred in Dogpile: {e} \033[0m'
)
elif engineitem == 'duckduckgo':
print('\033[94m[*] Searching DuckDuckGo. \033[0m')
from discovery import duckduckgosearch
search = duckduckgosearch.SearchDuckDuckGo(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'duckduckgo')
db.store_all(word, all_hosts, 'host', 'duckduckgo')
elif engineitem == 'google':
print('\033[94m[*] Searching Google. \033[0m')
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = filter(search.get_emails())
all_emails.extend(emails)
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'google')
db.store_all(word, all_emails, 'email', 'google')
elif engineitem == 'googleCSE':
print(
'\033[94m[*] Searching Google Custom Search. \033[0m')
try:
search = googleCSE.SearchGoogleCSE(word, limit, start)
search.process()
search.store_results()
all_emails = filter(search.get_emails())
db = stash.stash_manager()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db.store_all(word, all_hosts, 'email', 'googleCSE')
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'googleCSE')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
elif engineitem == 'google-certificates':
print(
'\033[94m[*] Searching Google Certificate transparency report. \033[0m'
)
search = googlecertificates.SearchGoogleCertificates(
word, limit, start)
search.process()
hosts = filter(search.get_domains())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'google-certificates')
elif engineitem == 'hunter':
print('\033[94m[*] Searching Hunter. \033[0m')
from discovery import huntersearch
# Import locally or won't work.
try:
search = huntersearch.SearchHunter(word, limit, start)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'hunter')
db.store_all(word, all_emails, 'email', 'hunter')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
elif engineitem == 'linkedin':
print('\033[94m[*] Searching Linkedin. \033[0m')
search = linkedinsearch.SearchLinkedin(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'linkedin')
if len(people) == 0:
print('\n[*] No users found Linkedin.\n\n')
else:
print(f'\n[*] Users found: {len(people)}')
print('---------------------')
for user in sorted(list(set(people))):
print(user)
sys.exit(0)
elif engineitem == 'netcraft':
print('\033[94m[*] Searching Netcraft. \033[0m')
search = netcraft.SearchNetcraft(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'netcraft')
elif engineitem == 'pgp':
print('\033[94m[*] Searching PGP key server. \033[0m')
try:
search = pgpsearch.SearchPgp(word)
search.process()
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'pgp')
db.store_all(word, all_emails, 'email', 'pgp')
except Exception:
pass
elif engineitem == 'securityTrails':
print('\033[94m[*] Searching SecurityTrails. \033[0m')
from discovery import securitytrailssearch
try:
search = securitytrailssearch.search_securitytrail(
word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'securityTrails')
ips = search.get_ips()
all_ip.extend(ips)
db = stash.stash_manager()
db.store_all(word, ips, 'ip', 'securityTrails')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
elif engineitem == 'threatcrowd':
print('\033[94m[*] Searching Threatcrowd. \033[0m')
try:
search = threatcrowd.search_threatcrowd(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'threatcrowd')
except Exception:
pass
elif engineitem == 'trello':
print('\033[94m[*] Searching Trello. \033[0m')
from discovery import trello
# Import locally or won't work.
search = trello.search_trello(word, limit)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
info = search.get_urls()
hosts = filter(info[0])
trello_info = (info[1], True)
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'trello')
db.store_all(word, emails, 'email', 'trello')
elif engineitem == 'twitter':
print('\033[94m[*] Searching Twitter. \033[0m')
search = twittersearch.search_twitter(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'twitter')
if len(people) == 0:
print('\n[*] No users found.\n\n')
else:
print('\n[*] Users found: ' + str(len(people)))
print('---------------------')
for user in sorted(list(set(people))):
print(user)
elif engineitem == 'virustotal':
print('\033[94m[*] Searching VirusTotal. \033[0m')
search = virustotal.search_virustotal(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'virustotal')
elif engineitem == 'yahoo':
print('\033[94m[*] Searching Yahoo. \033[0m')
search = yahoosearch.search_yahoo(word, limit)
search.process()
hosts = search.get_hostnames()
emails = search.get_emails()
all_hosts.extend(filter(hosts))
all_emails.extend(filter(emails))
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'yahoo')
db.store_all(word, all_emails, 'email', 'yahoo')
elif engineitem == 'all':
print('\033[94m[*] Running a full harvest. \033[0m')
all_emails = []
all_hosts = []
try:
print('\033[94m\n[*] Searching Baidu. \033[0m')
search = baidusearch.SearchBaidu(word, limit)
search.process()
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'baidu')
db.store_all(word, all_emails, 'email', 'baidu')
except Exception:
pass
print('\033[94m\n[*] Searching Bing. \033[0m')
bingapi = 'no'
search = bingsearch.SearchBing(word, limit, start)
search.process(bingapi)
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'bing')
all_emails.extend(emails)
all_emails = sorted(set(all_emails))
db.store_all(word, all_emails, 'email', 'bing')
print('\033[94m\n[*] Searching Censys. \033[0m')
from discovery import censys
search = censys.SearchCensys(word, limit)
search.process()
ips = search.get_ipaddresses()
setips = set(ips)
uniqueips = list(setips) # Remove duplicates.
all_ip.extend(uniqueips)
hosts = filter(search.get_hostnames())
sethosts = set(hosts)
uniquehosts = list(sethosts) # Remove duplicates.
all_hosts.extend(uniquehosts)
db = stash.stash_manager()
db.store_all(word, uniquehosts, 'host', 'censys')
db.store_all(word, uniqueips, 'ip', 'censys')
print('\033[94m\n[*] Searching CRT.sh. \033[0m')
search = crtsh.search_crtsh(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'CRTsh')
# cymon
print('\033[94m\n[*] Searching Cymon. \033[0m')
from discovery import cymon
# Import locally or won't work.
search = cymon.search_cymon(word)
search.process()
all_ip = search.get_ipaddresses()
db = stash.stash_manager()
db.store_all(word, all_ip, 'ip', 'cymon')
print('\033[94m\n[*] Searching Dogpile. \033[0m')
search = dogpilesearch.SearchDogpile(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'dogpile')
db.store_all(word, all_hosts, 'host', 'dogpile')
print('\033[94m\n[*] Searching DuckDuckGo. \033[0m')
from discovery import duckduckgosearch
search = duckduckgosearch.SearchDuckDuckGo(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'duckduckgo')
db.store_all(word, all_hosts, 'host', 'duckduckgo')
print('\033[94m\n[*] Searching Google. \033[0m')
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_emails, 'email', 'google')
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'google')
print(
'\033[94m\n[*] Searching Google Certificate transparency report. \033[0m'
)
search = googlecertificates.SearchGoogleCertificates(
word, limit, start)
search.process()
domains = filter(search.get_domains())
all_hosts.extend(domains)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'google-certificates')
print('\033[94m\n[*] Searching Hunter. \033[0m')
from discovery import huntersearch
# Import locally.
try:
search = huntersearch.SearchHunter(word, limit, start)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'hunter')
all_emails.extend(emails)
all_emails = sorted(set(all_emails))
db.store_all(word, all_emails, 'email', 'hunter')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
print('\033[94m\n[*] Searching Linkedin. \033[0m')
search = linkedinsearch.SearchLinkedin(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'linkedin')
if len(people) == 0:
print('\n[*] No users found.\n\n')
else:
print('\n[*] Users found: ' + str(len(people)))
print('---------------------')
for user in sorted(list(set(people))):
print(user)
print('\033[94m\n[*] Searching Netcraft. \033[0m')
search = netcraft.SearchNetcraft(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'netcraft')
print('\033[94m\n[*] Searching PGP key server. \033[0m')
try:
search = pgpsearch.SearchPgp(word)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
sethosts = set(hosts)
uniquehosts = list(sethosts) # Remove duplicates.
all_hosts.extend(uniquehosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'PGP')
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_emails, 'email', 'PGP')
except Exception:
pass
print('\033[94m\n[*] Searching Threatcrowd. \033[0m')
try:
search = threatcrowd.search_threatcrowd(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'threatcrowd')
except Exception:
pass
print('\033[94m\n[*] Searching Trello. \033[0m')
from discovery import trello
# Import locally or won't work.
search = trello.search_trello(word, limit)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
info = search.get_urls()
hosts = filter(info[0])
trello_info = (info[1], True)
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'trello')
db.store_all(word, emails, 'email', 'trello')
try:
print('\033[94m\n[*] Searching Twitter. \033[0m')
search = twittersearch.search_twitter(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'twitter')
print('\nUsers from Twitter:')
print('-------------------')
for user in people:
print(user)
except Exception:
pass
print('\n[*] Virtual hosts:')
print('------------------')
for l in host_ip:
search = bingsearch.SearchBing(l, limit, start)
search.process_vhost()
res = search.get_allhostnames()
for x in res:
x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x)
x = re.sub('<', '', x)
x = re.sub('>', '', x)
print((l + '\t' + x))
vhost.append(l + ':' + x)
full.append(l + ':' + x)
vhost = sorted(set(vhost))
print('\033[94m\n[*] Searching VirusTotal. \033[0m')
search = virustotal.search_virustotal(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'virustotal')
print('\033[94m\n[*] Searching Yahoo. \033[0m')
search = yahoosearch.search_yahoo(word, limit)
search.process()
hosts = search.get_hostnames()
emails = search.get_emails()
all_hosts.extend(filter(hosts))
all_emails.extend(filter(emails))
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'yahoo')
db.store_all(word, all_emails, 'email', 'yahoo')
else:
print('\033[93m[!] Invalid source.\n\n \033[0m')
sys.exit(1)
# Sanity check to see if all_emails and all_hosts are defined.
try:
all_emails
except NameError:
print(
'\n\n\033[93m[!] No emails found because all_emails is not defined.\n\n \033[0m'
)
sys.exit(1)
try:
all_hosts
except NameError:
print(
'\n\n\033[93m[!] No hosts found because all_hosts is not defined.\n\n \033[0m'
)
sys.exit(1)
# Results
if len(all_ip) == 0:
print('\n[*] No IPs found.')
else:
print('\n[*] IPs found: ' + str(len(all_ip)))
print('-------------------')
ips = sorted(
ipaddress.ip_address(line.strip()) for line in set(all_ip))
print('\n'.join(map(str, ips)))
if len(all_emails) == 0:
print('\n[*] No emails found.')
else:
print('\n[*] Emails found: ' + str(len(all_emails)))
print('----------------------')
print(('\n'.join(sorted(list(set(all_emails))))))
if len(all_hosts) == 0:
print('\n[*] No hosts found.\n\n')
else:
print('\n[*] Hosts found: ' + str(len(all_hosts)))
print('---------------------')
all_hosts = sorted(list(set(all_hosts)))
full_host = hostchecker.Checker(all_hosts)
full = full_host.check()
for host in full:
ip = host.split(':')[1]
print(host)
if ip != 'empty':
if host_ip.count(ip.lower()):
pass
else:
host_ip.append(ip.lower())
db = stash.stash_manager()
db.store_all(word, host_ip, 'ip', 'DNS-resolver')
if trello_info[1] is True:
trello_urls = trello_info[0]
if trello_urls == []:
print('\n[*] No URLs found.')
else:
total = len(trello_urls)
print('\n[*] URLs found: ' + str(total))
print('--------------------')
for url in sorted(list(set(trello_urls))):
print(url)
# DNS brute force
dnsres = []
if dnsbrute is True:
print('\n[*] Starting DNS brute force.')
a = dnssearch.dns_force(word, dnsserver, verbose=True)
res = a.process()
print('\n[*] Hosts found after DNS brute force:')
print('-------------------------------------')
for y in res:
print(y)
dnsres.append(y.split(':')[0])
if y not in full:
full.append(y)
db = stash.stash_manager()
db.store_all(word, dnsres, 'host', 'dns_bruteforce')
# Port scanning
if ports_scanning is True:
print('\n\n[*] Scanning ports (active).\n')
for x in full:
host = x.split(':')[1]
domain = x.split(':')[0]
if host != 'empty':
print(('[*] Scanning ' + host))
ports = [21, 22, 80, 443, 8080]
try:
scan = port_scanner.PortScan(host, ports)
openports = scan.process()
if len(openports) > 1:
print(('\t[*] Detected open ports: ' +
','.join(str(e) for e in openports)))
takeover_check = 'True'
if takeover_check == 'True':
if len(openports) > 0:
search_take = takeover.take_over(domain)
search_take.process()
except Exception as e:
print(e)
# DNS reverse lookup
dnsrev = []
if dnslookup is True:
print('\n[*] Starting active queries.')
analyzed_ranges = []
for x in host_ip:
print(x)
ip = x.split(':')[0]
range = ip.split('.')
range[3] = '0/24'
s = '.'
range = s.join(range)
if not analyzed_ranges.count(range):
print('[*] Performing reverse lookup in ' + range)
a = dnssearch.dns_reverse(range, True)
a.list()
res = a.process()
analyzed_ranges.append(range)
else:
continue
for x in res:
if x.count(word):
dnsrev.append(x)
if x not in full:
full.append(x)
print('[*] Hosts found after reverse lookup (in target domain):')
print('--------------------------------------------------------')
for xh in dnsrev:
print(xh)
# DNS TLD expansion
dnstldres = []
if dnstld is True:
print('[*] Starting DNS TLD expansion.')
a = dnssearch.dns_tld(word, dnsserver, verbose=True)
res = a.process()
print('\n[*] Hosts found after DNS TLD expansion:')
print('----------------------------------------')
for y in res:
print(y)
dnstldres.append(y)
if y not in full:
full.append(y)
# Virtual hosts search
if virtual == 'basic':
print('\n[*] Virtual hosts:')
print('------------------')
for l in host_ip:
search = bingsearch.SearchBing(l, limit, start)
search.process_vhost()
res = search.get_allhostnames()
for x in res:
x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x)
x = re.sub('<', '', x)
x = re.sub('>', '', x)
print((l + '\t' + x))
vhost.append(l + ':' + x)
full.append(l + ':' + x)
vhost = sorted(set(vhost))
else:
pass
# Shodan
shodanres = []
if shodan is True:
import texttable
tab = texttable.Texttable()
header = [
'IP address', 'Hostname', 'Org', 'Services:Ports', 'Technologies'
]
tab.header(header)
tab.set_cols_align(['c', 'c', 'c', 'c', 'c'])
tab.set_cols_valign(['m', 'm', 'm', 'm', 'm'])
tab.set_chars(['-', '|', '+', '#'])
tab.set_cols_width([15, 20, 15, 15, 18])
host_ip = list(set(host_ip))
print('\033[94m[*] Searching Shodan. \033[0m')
try:
for ip in host_ip:
print(('\tSearching for ' + ip))
shodan = shodansearch.search_shodan()
rowdata = shodan.search_ip(ip)
time.sleep(2)
tab.add_row(rowdata)
printedtable = tab.draw()
print(printedtable)
except Exception as e:
print(
f'\033[93m[!] Error occurred in the Shodan search module: {e} \033[0m'
)
else:
pass
# Here we need to add explosion mode.
# We have to take out the TLDs to do this.
recursion = None
if recursion:
start = 0
for word in vhost:
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = search.get_emails()
hosts = search.get_hostnames()
print(emails)
print(hosts)
else:
pass
# Reporting
if filename != "":
try:
print('\nNEW REPORTING BEGINS.')
db = stash.stash_manager()
scanboarddata = db.getscanboarddata()
latestscanresults = db.getlatestscanresults(word)
previousscanresults = db.getlatestscanresults(word,
previousday=True)
latestscanchartdata = db.latestscanchartdata(word)
scanhistorydomain = db.getscanhistorydomain(word)
pluginscanstatistics = db.getpluginscanstatistics()
generator = statichtmlgenerator.htmlgenerator(word)
HTMLcode = generator.beginhtml()
HTMLcode += generator.generatelatestscanresults(latestscanresults)
HTMLcode += generator.generatepreviousscanresults(
previousscanresults)
graph = reportgraph.GraphGenerator(word)
HTMLcode += graph.drawlatestscangraph(word, latestscanchartdata)
HTMLcode += graph.drawscattergraphscanhistory(
word, scanhistorydomain)
HTMLcode += generator.generatepluginscanstatistics(
pluginscanstatistics)
HTMLcode += generator.generatedashboardcode(scanboarddata)
HTMLcode += '<p><span style="color: #000000;">Report generated on ' + str(
datetime.datetime.now()) + '</span></p>'
HTMLcode += '''
</body>
</html>
'''
Html_file = open('report.html', 'w')
Html_file.write(HTMLcode)
Html_file.close()
print('NEW REPORTING FINISHED!')
print('[*] Saving files.')
html = htmlExport.htmlExport(all_emails, full, vhost, dnsres,
dnsrev, filename, word, shodanres,
dnstldres)
save = html.writehtml()
except Exception as e:
print(e)
print('\n\033[93m[!] An error occurred creating the file.\033[0m')
try:
filename = filename.split('.')[0] + '.xml'
file = open(filename, 'w')
file.write('<?xml version="1.0" encoding="UTF-8"?><theHarvester>')
for x in all_emails:
file.write('<email>' + x + '</email>')
for x in full:
x = x.split(':')
if len(x) == 2:
file.write('<host>' + '<ip>' + x[1] + '</ip><hostname>' +
x[0] + '</hostname>' + '</host>')
else:
file.write('<host>' + x + '</host>')
for x in vhost:
x = x.split(':')
if len(x) == 2:
file.write('<vhost>' + '<ip>' + x[1] + '</ip><hostname>' +
x[0] + '</hostname>' + '</vhost>')
else:
file.write('<vhost>' + x + '</vhost>')
if shodanres != []:
shodanalysis = []
for x in shodanres:
res = x.split('SAPO')
file.write('<shodan>')
file.write('<host>' + res[0] + '</host>')
file.write('<port>' + res[2] + '</port>')
file.write('<banner><!--' + res[1] + '--></banner>')
reg_server = re.compile('Server:.*')
temp = reg_server.findall(res[1])
if temp != []:
shodanalysis.append(res[0] + ':' + temp[0])
file.write('</shodan>')
if shodanalysis != []:
shodanalysis = sorted(set(shodanalysis))
file.write('<servers>')
for x in shodanalysis:
file.write('<server>' + x + '</server>')
file.write('</servers>')
file.write('</theHarvester>')
file.flush()
file.close()
print('[*] Files saved.')
except Exception as er:
print(
f'\033[93m[!] An error occurred saving XML file: {er} \033[0m')
print('\n\n')
sys.exit(0)
def list_all(self, jails):
"""List all jails."""
self.full = True if self.plugin else self.full
jail_list = []
for jail in jails:
mountpoint = jail.properties["mountpoint"].value
try:
conf = iocage_lib.ioc_json.IOCJson(mountpoint).json_get_value(
'all')
state = ''
except (Exception, SystemExit):
# Jail is corrupt, we want all the keys to exist.
# So we will take the defaults and let the user
# know that they are not correct.
def_props = iocage_lib.ioc_json.IOCJson().json_get_value(
'all', default=True)
conf = {x: 'N/A' for x in def_props}
conf['host_hostuuid'] = \
f'{jail.name.split("/")[-1]}'
conf['release'] = 'N/A'
state = 'CORRUPT'
jid = '-'
if self.basejail_only and conf.get('basejail', 'no') != 'yes':
continue
uuid_full = conf["host_hostuuid"]
uuid = uuid_full
if not self.full:
# We only want to show the first 8 characters of a UUID,
# if it's not a UUID, we will show the whole name no matter
# what.
try:
uuid = str(_uuid.UUID(uuid, version=4))[:8]
except ValueError:
# We leave the "uuid" untouched, as it's not a valid
# UUID, but instead a named jail.
pass
full_ip4 = conf["ip4_addr"]
ip6 = conf["ip6_addr"]
try:
short_ip4 = ",".join([
item.split("|")[1].split("/")[0]
for item in full_ip4.split(",")
])
except IndexError:
short_ip4 = full_ip4 if full_ip4 != "none" else "-"
boot = conf["boot"]
jail_type = conf["type"]
full_release = conf["release"]
basejail = conf.get('basejail', 'no')
if "HBSD" in full_release:
full_release = re.sub(r"\W\w.", "-", full_release)
full_release = full_release.replace("--SD", "-STABLE-HBSD")
short_release = full_release.rstrip("-HBSD")
else:
short_release = "-".join(full_release.rsplit("-")[:2])
if full_ip4 == "none":
full_ip4 = "-"
if ip6 == "none":
ip6 = "-"
# Will be set already by a corrupt jail
status = False
if state != 'CORRUPT':
status, jid = self.list_get_jid(uuid_full)
if status:
state = "up"
else:
state = "down"
if conf["type"] == "template":
template = "-"
else:
jail_root = self.zfs.get_dataset(f"{jail.name}/root")
_origin_property = jail_root.properties["origin"]
if _origin_property and _origin_property.value != "":
template = jail_root.properties["origin"].value
template = template.rsplit("/root@", 1)[0].rsplit("/",
1)[-1]
else:
template = "-"
if "release" in template.lower() or "stable" in template.lower():
template = "-"
if conf["dhcp"] == "on" and status and os.geteuid() == 0:
interface = conf["interfaces"].split(",")[0].split(":")[0]
if interface == "vnet0":
# Inside jails they are epairNb
interface = f"{interface.replace('vnet', 'epair')}b"
short_ip4 = "DHCP"
full_ip4_cmd = [
"jexec", f"ioc-{uuid_full.replace('.', '_')}", "ifconfig",
interface, "inet"
]
try:
out = su.check_output(full_ip4_cmd)
full_ip4 = f'{interface}|' \
f'{out.splitlines()[2].split()[1].decode()}'
except (su.CalledProcessError, IndexError) as e:
short_ip4 += '(Network Issue)'
if isinstance(e, su.CalledProcessError):
full_ip4 = f'DHCP - Network Issue: {e}'
else:
full_ip4 = f'DHCP - Failed Parsing: {e}'
elif conf["dhcp"] == "on" and not status:
short_ip4 = "DHCP"
full_ip4 = "DHCP (not running)"
elif conf["dhcp"] == "on" and os.geteuid() != 0:
short_ip4 = "DHCP"
full_ip4 = "DHCP (running -- address requires root)"
# Append the JID and the NAME to the table
if self.full and self.plugin:
if jail_type != "plugin" and jail_type != "pluginv2":
# We only want plugin type jails to be apart of the
# list
continue
try:
with open(f"{mountpoint}/plugin/ui.json", "r") as u:
ip = full_ip4.split("|", 1)
ip = ip[1] if len(ip) != 1 else ip[0]
ip = ip.split("/", 1)[0] if "DHCP" not in full_ip4 \
else "DHCP"
ui_data = json.load(u)
admin_portal = ui_data["adminportal"]
admin_portal = admin_portal.replace("%%IP%%", ip)
try:
ph = ui_data["adminportal_placeholders"].items()
for placeholder, prop in ph:
admin_portal = admin_portal.replace(
placeholder,
iocage_lib.ioc_json.IOCJson(
mountpoint).json_plugin_get_value(
prop.split(".")))
except KeyError:
pass
except iocage_lib.ioc_exceptions.CommandNeedsRoot:
admin_portal = "Admin Portal requires root"
except iocage_lib.ioc_exceptions.CommandFailed as e:
admin_portal = b' '.join(e.message).decode()
except FileNotFoundError:
# They just didn't set a admin portal.
admin_portal = "-"
jail_list.append([
jid, uuid, boot, state, jail_type, full_release, full_ip4,
ip6, template, admin_portal
])
elif self.full:
jail_list.append([
jid, uuid, boot, state, jail_type, full_release, full_ip4,
ip6, template, basejail
])
else:
jail_list.append([jid, uuid, state, short_release, short_ip4])
list_type = "list_full" if self.full else "list_short"
sort = iocage_lib.ioc_common.ioc_sort(list_type,
self.sort,
data=jail_list)
jail_list.sort(key=sort)
# return the list...
if not self.header:
flat_jail = [j for j in jail_list]
return flat_jail
# Prints the table
table = texttable.Texttable(max_width=0)
if self.full:
if self.plugin:
table.set_cols_dtype(
["t", "t", "t", "t", "t", "t", "t", "t", "t", "t"])
jail_list.insert(0, [
"JID", "NAME", "BOOT", "STATE", "TYPE", "RELEASE", "IP4",
"IP6", "TEMPLATE", "PORTAL"
])
else:
# We get an infinite float otherwise.
table.set_cols_dtype(
["t", "t", "t", "t", "t", "t", "t", "t", "t", "t"])
jail_list.insert(0, [
"JID", "NAME", "BOOT", "STATE", "TYPE", "RELEASE", "IP4",
"IP6", "TEMPLATE", 'BASEJAIL'
])
else:
# We get an infinite float otherwise.
table.set_cols_dtype(["t", "t", "t", "t", "t"])
jail_list.insert(0, ["JID", "NAME", "STATE", "RELEASE", "IP4"])
table.add_rows(jail_list)
return table.draw()
def cli(prop, _type, _pool, jail, recursive, header, plugin, force):
"""Get a list of jails and print the property."""
table = texttable.Texttable(max_width=0)
if _type:
# Confusing I know.
jail = prop
prop = _type
elif _pool:
pool = ioc.IOCage(skip_jails=True).get('', pool=True)
ioc_common.logit({'level': 'INFO', 'message': pool})
exit()
else:
if not jail and not recursive:
ioc_common.logit({
'level': 'EXCEPTION',
'message': 'You must specify a jail!'
})
if _type == 'all' and recursive:
# TODO: Port this back
ioc_common.logit({
'level':
'EXCEPTION',
'message':
'You cannot use --all (-a) and --recursive (-r) '
'together. '
})
if _type == 'all' and not jail:
ioc_common.logit({
'level':
'EXCEPTION',
'message':
'Please specify a jail name when using -a flag.'
})
if not recursive:
if prop == 'state' or _type == 'state':
state = ioc.IOCage(jail=jail).get(prop)
ioc_common.logit({'level': 'INFO', 'message': state})
elif prop == 'jid' or _type == 'jid':
jid = ioc.IOCage(jail=jail).list('jid', uuid=jail)[1]
ioc_common.logit({'level': 'INFO', 'message': jid})
elif plugin:
_plugin = ioc.IOCage(jail=jail,
skip_jails=True).get(prop,
plugin=True,
start_jail=force)
ioc_common.logit({'level': 'INFO', 'message': _plugin})
elif prop == 'all':
props = ioc.IOCage(jail=jail, skip_jails=True).get(prop)
for p, v in props.items():
ioc_common.logit({'level': 'INFO', 'message': f'{p}:{v}'})
else:
p = ioc.IOCage(jail=jail, skip_jails=True).get(prop)
ioc_common.logit({'level': 'INFO', 'message': p})
else:
jails = ioc.IOCage().get(prop, recursive=True)
table.header(['NAME', f'PROP - {prop}'])
for jail_dict in jails:
for jail, prop in jail_dict.items():
if header:
table.add_row([jail, prop])
else:
ioc_common.logit({
'level': 'INFO',
'message': f'{jail}\t{prop}'
})
if header:
# Prints the table
ioc_common.logit({'level': 'INFO', 'message': table.draw()})
def _print_scores(y_true, y_pred, metrics):
tab = tt.Texttable()
print('\nTable: Evaluation Scores')
headings = ['Metric', 'Score']
tab.header(headings)
for m in metrics:
score = None
try:
if m == 'accuracy':
y_pred = optimize_threshold(y_true, y_pred, accuracy_score)
score = accuracy_score(y_true, y_pred)
elif m == 'precision':
y_pred = optimize_threshold(y_true, y_pred, precision_score)
score = precision_score(y_true, y_pred)
elif m == 'precision.micro':
y_pred = optimize_threshold(y_true,
y_pred,
precision_score,
average='micro')
score = precision_score(y_true, y_pred, average='micro')
elif m == 'precision.macro':
y_pred = optimize_threshold(y_true,
y_pred,
precision_score,
average='macro')
score = precision_score(y_true, y_pred, average='macro')
elif m == 'recall':
score = recall_score(y_true, y_pred)
elif m == 'recall.micro':
y_pred = optimize_threshold(y_true,
y_pred,
recall_score,
average='micro')
score = recall_score(y_true, y_pred, average='micro')
elif m == 'recall.macro':
y_pred = optimize_threshold(y_true,
y_pred,
recall_score,
average='macro')
score = recall_score(y_true, y_pred, average='macro')
elif m == 'f1':
y_pred = optimize_threshold(y_true, y_pred, f1_score)
score = f1_score(y_true, y_pred)
elif m == 'f1.micro':
y_pred = optimize_threshold(y_true,
y_pred,
f1_score,
average='micro')
score = f1_score(y_true, y_pred, average='micro')
elif m == 'f1.macro':
y_pred = optimize_threshold(y_true,
y_pred,
f1_score,
average='macro')
score = f1_score(y_true, y_pred, average='macro')
elif m == 'pearson_corr':
score = pearson_corr(y_true, y_pred)
elif m == 'spearman_corr':
score = spearman_corr(y_true, y_pred)
tab.add_row([m, score])
except Exception as e:
tab.add_row([m, 'Invalid'])
s = tab.draw()
print(s)
def status_report(jobs, width, height=None, tmp_prefix='', dst_prefix=''):
'''height, if provided, will limit the number of rows in the table,
showing first and last rows, row numbers and an elipsis in the middle.'''
abbreviate_jobs_list = False
n_begin_rows = 0
n_end_rows = 0
if height and height < len(jobs) + 1: # One row for header
abbreviate_jobs_list = True
if abbreviate_jobs_list:
n_rows = height - 2 # One for header, one for elipsis
n_begin_rows = int(n_rows / 2)
n_end_rows = n_rows - n_begin_rows
tab = tt.Texttable()
headings = [
'plot id', 'k', 'tmp', 'dst', 'wall', 'phase', 'tmp', 'pid', 'stat',
'mem', 'user', 'sys', 'io'
]
if height:
headings.insert(0, '#')
tab.header(headings)
tab.set_cols_dtype('t' * len(headings))
tab.set_cols_align('r' * len(headings))
tab.set_header_align('r' * len(headings))
for i, j in enumerate(sorted(jobs, key=job.Job.get_time_wall)):
# Elipsis row
if abbreviate_jobs_list and i == n_begin_rows:
row = ['...'] + ([''] * 13)
# Omitted row
elif abbreviate_jobs_list and i > n_begin_rows and i < (len(jobs) -
n_end_rows):
continue
# Regular row
else:
try:
row = [
j.plot_id[:8] + '...', j.k,
abbr_path(j.tmpdir, tmp_prefix),
abbr_path(j.dstdir, dst_prefix),
plot_util.time_format(j.get_time_wall()),
'%d:%d' % j.progress(),
plot_util.human_format(j.get_tmp_usage(), 0), j.proc.pid,
j.get_run_status(),
plot_util.human_format(j.get_mem_usage(), 1),
plot_util.time_format(j.get_time_user()),
plot_util.time_format(j.get_time_sys()),
plot_util.time_format(j.get_time_iowait())
]
except psutil.NoSuchProcess:
# In case the job has disappeared
row = [j.plot_id[:8] + '...'] + (['--'] * 12)
if height:
row.insert(0, '%3d' % i)
tab.add_row(row)
tab.set_max_width(width)
tab.set_deco(0) # No borders
# return ('tmp dir prefix: %s ; dst dir prefix: %s\n' % (tmp_prefix, dst_prefix)
return tab.draw()
def start():
parser = argparse.ArgumentParser(
description=
'theHarvester is used to gather open source intelligence (OSINT) on a\n'
'company or domain.')
parser.add_argument('-d',
'--domain',
help='company name or domain to search',
required=True)
parser.add_argument('-l',
'--limit',
help='limit the number of search results, default=500',
default=500,
type=int)
parser.add_argument('-S',
'--start',
help='start with result number X, default=0',
default=0,
type=int)
parser.add_argument('-g',
'--google-dork',
help='use Google Dorks for Google search',
default=False,
action='store_true')
parser.add_argument(
'-p',
'--port-scan',
help=
'scan the detected hosts and check for Takeovers (21,22,80,443,8080)',
default=False,
action='store_true')
parser.add_argument('-s',
'--shodan',
help='use Shodan to query discovered hosts',
default=False,
action='store_true')
parser.add_argument(
'-v',
'--virtual-host',
help='verify host name via DNS resolution and search for virtual hosts',
action='store_const',
const='basic',
default=False)
parser.add_argument('-e',
'--dns-server',
help='DNS server to use for lookup')
parser.add_argument(
'-t',
'--dns-tld',
help='perform a DNS TLD expansion discovery, default False',
default=False)
parser.add_argument('-n',
'--dns-lookup',
help='enable DNS server lookup, default False',
default=False,
action='store_true')
parser.add_argument('-c',
'--dns-brute',
help='perform a DNS brute force on the domain',
default=False,
action='store_true')
parser.add_argument('-f',
'--filename',
help='save the results to an HTML and/or XML file',
default='',
type=str)
parser.add_argument(
'-b',
'--source',
help='''baidu, bing, bingapi, censys, crtsh, dnsdumpster,
dogpile, duckduckgo, github-code, google,
hunter, intelx,
linkedin, linkedin_links, netcraft, securityTrails, threatcrowd,
trello, twitter, vhost, virustotal, yahoo''')
args = parser.parse_args()
try:
db = stash.stash_manager()
db.do_init()
except Exception:
pass
all_emails = []
all_hosts = []
all_ip = []
dnsbrute = args.dns_brute
dnslookup = args.dns_lookup
dnsserver = args.dns_server
dnstld = args.dns_tld
filename = args.filename # type: str
full = []
google_dorking = args.google_dork
host_ip = []
limit = args.limit # type: int
ports_scanning = args.port_scan
shodan = args.shodan
start = args.start # type: int
takeover_check = False
trello_info = ([], False)
vhost = []
virtual = args.virtual_host
word = args.domain # type: str
if args.source is not None:
engines = set(map(str.strip, args.source.split(',')))
if set(engines).issubset(Core.get_supportedengines()):
print(f'\033[94m[*] Target: {word} \n \033[0m')
for engineitem in engines:
if engineitem == 'baidu':
print('\033[94m[*] Searching Baidu. \033[0m')
from theHarvester.discovery import baidusearch
try:
baidu_search = baidusearch.SearchBaidu(word, limit)
baidu_search.process()
all_emails = filter(baidu_search.get_emails())
hosts = filter(baidu_search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'baidu')
db.store_all(word, all_emails, 'email', 'baidu')
except Exception:
pass
elif engineitem == 'bing' or engineitem == 'bingapi':
print('\033[94m[*] Searching Bing. \033[0m')
from theHarvester.discovery import bingsearch
try:
bing_search = bingsearch.SearchBing(word, limit, start)
bingapi = ''
if engineitem == 'bingapi':
bingapi += 'yes'
else:
bingapi += 'no'
bing_search.process(bingapi)
all_emails = filter(bing_search.get_emails())
hosts = filter(bing_search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'bing')
db.store_all(word, all_hosts, 'host', 'bing')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
elif engineitem == 'censys':
print('\033[94m[*] Searching Censys. \033[0m')
from theHarvester.discovery import censys
# Import locally or won't work
censys_search = censys.SearchCensys(word, limit)
censys_search.process()
all_ip = censys_search.get_ipaddresses()
hosts = filter(censys_search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'censys')
db.store_all(word, all_ip, 'ip', 'censys')
elif engineitem == 'crtsh':
try:
print('\033[94m[*] Searching CRT.sh. \033[0m')
from theHarvester.discovery import crtsh
crtsh_search = crtsh.SearchCrtsh(word)
crtsh_search.process()
hosts = filter(crtsh_search.get_data())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'CRTsh')
except Exception:
print(
f'\033[93m[!] An timeout occurred with crtsh, cannot find {args.domain}\033[0m'
)
elif engineitem == 'dnsdumpster':
try:
print('\033[94m[*] Searching DNSdumpster. \033[0m')
from theHarvester.discovery import dnsdumpster
dns_dumpster_search = dnsdumpster.SearchDnsDumpster(
word)
dns_dumpster_search.process()
hosts = filter(dns_dumpster_search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'dnsdumpster')
except Exception as e:
print(
f'\033[93m[!] An error occurred with dnsdumpster: {e} \033[0m'
)
elif engineitem == 'dogpile':
try:
print('\033[94m[*] Searching Dogpile. \033[0m')
from theHarvester.discovery import dogpilesearch
dogpile_search = dogpilesearch.SearchDogpile(
word, limit)
dogpile_search.process()
emails = filter(dogpile_search.get_emails())
hosts = filter(dogpile_search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'dogpile')
db.store_all(word, all_hosts, 'host', 'dogpile')
except Exception as e:
print(
f'\033[93m[!] An error occurred with Dogpile: {e} \033[0m'
)
elif engineitem == 'duckduckgo':
print('\033[94m[*] Searching DuckDuckGo. \033[0m')
from theHarvester.discovery import duckduckgosearch
duckduckgo_search = duckduckgosearch.SearchDuckDuckGo(
word, limit)
duckduckgo_search.process()
emails = filter(duckduckgo_search.get_emails())
hosts = filter(duckduckgo_search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'duckduckgo')
db.store_all(word, all_hosts, 'host', 'duckduckgo')
elif engineitem == 'github-code':
print('\033[94m[*] Searching Github (code). \033[0m')
try:
from theHarvester.discovery import githubcode
github_search = githubcode.SearchGithubCode(
word, limit)
github_search.process()
emails = filter(github_search.get_emails())
all_emails.extend(emails)
hosts = filter(github_search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'github-code')
db.store_all(word, all_emails, 'email', 'github-code')
except MissingKey as ex:
print(ex)
else:
pass
elif engineitem == 'exalead':
print('\033[94m[*] Searching Exalead \033[0m')
from theHarvester.discovery import exaleadsearch
exalead_search = exaleadsearch.SearchExalead(
word, limit, start)
exalead_search.process()
emails = filter(exalead_search.get_emails())
all_emails.extend(emails)
hosts = filter(exalead_search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'exalead')
db.store_all(word, all_emails, 'email', 'exalead')
elif engineitem == 'google':
print('\033[94m[*] Searching Google. \033[0m')
from theHarvester.discovery import googlesearch
google_search = googlesearch.SearchGoogle(
word, limit, start)
google_search.process(google_dorking)
emails = filter(google_search.get_emails())
all_emails.extend(emails)
hosts = filter(google_search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'google')
db.store_all(word, all_emails, 'email', 'google')
elif engineitem == 'hunter':
print('\033[94m[*] Searching Hunter. \033[0m')
from theHarvester.discovery import huntersearch
# Import locally or won't work.
try:
hunter_search = huntersearch.SearchHunter(
word, limit, start)
hunter_search.process()
emails = filter(hunter_search.get_emails())
all_emails.extend(emails)
hosts = filter(hunter_search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'hunter')
db.store_all(word, all_emails, 'email', 'hunter')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
elif engineitem == 'intelx':
print('\033[94m[*] Searching Intelx. \033[0m')
from theHarvester.discovery import intelxsearch
# Import locally or won't work.
try:
intelx_search = intelxsearch.SearchIntelx(word, limit)
intelx_search.process()
emails = filter(intelx_search.get_emails())
all_emails.extend(emails)
hosts = filter(intelx_search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'intelx')
db.store_all(word, all_emails, 'email', 'intelx')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
print(e)
elif engineitem == 'linkedin':
print('\033[94m[*] Searching Linkedin. \033[0m')
from theHarvester.discovery import linkedinsearch
linkedin_search = linkedinsearch.SearchLinkedin(
word, limit)
linkedin_search.process()
people = linkedin_search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'linkedin')
if len(people) == 0:
print('\n[*] No users found Linkedin.\n\n')
else:
print(f'\n[*] Users found: {len(people)}')
print('---------------------')
for user in sorted(list(set(people))):
print(user)
elif engineitem == 'linkedin_links':
print('\033[94m[*] Searching Linkedin. \033[0m')
from theHarvester.discovery import linkedinsearch
linkedin_links_search = linkedinsearch.SearchLinkedin(
word, limit)
linkedin_links_search.process()
people = linkedin_links_search.get_links()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'linkedin')
if len(people) == 0:
print('\n[*] No links found Linkedin.\n\n')
else:
print(f'\n[*] Links found: {len(people)}')
print('---------------------')
for user in sorted(list(set(people))):
print(user)
elif engineitem == 'netcraft':
print('\033[94m[*] Searching Netcraft. \033[0m')
from theHarvester.discovery import netcraft
netcraft_search = netcraft.SearchNetcraft(word)
netcraft_search.process()
hosts = filter(netcraft_search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'netcraft')
elif engineitem == 'securityTrails':
print('\033[94m[*] Searching SecurityTrails. \033[0m')
from theHarvester.discovery import securitytrailssearch
try:
securitytrails_search = securitytrailssearch.SearchSecuritytrail(
word)
securitytrails_search.process()
hosts = filter(securitytrails_search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'securityTrails')
ips = securitytrails_search.get_ips()
all_ip.extend(ips)
db = stash.stash_manager()
db.store_all(word, ips, 'ip', 'securityTrails')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
elif engineitem == 'threatcrowd':
print('\033[94m[*] Searching Threatcrowd. \033[0m')
from theHarvester.discovery import threatcrowd
try:
threatcrowd_search = threatcrowd.SearchThreatcrowd(
word)
threatcrowd_search.process()
hosts = filter(threatcrowd_search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'threatcrowd')
except Exception as e:
print(e)
elif engineitem == 'trello':
print('\033[94m[*] Searching Trello. \033[0m')
from theHarvester.discovery import trello
# Import locally or won't work.
trello_search = trello.SearchTrello(word, limit)
trello_search.process()
emails = filter(trello_search.get_emails())
all_emails.extend(emails)
info = trello_search.get_urls()
hosts = filter(info[0])
trello_info = (info[1], True)
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'trello')
db.store_all(word, emails, 'email', 'trello')
elif engineitem == 'twitter':
print(
'\033[94m[*] Searching Twitter usernames using Google. \033[0m'
)
from theHarvester.discovery import twittersearch
twitter_search = twittersearch.SearchTwitter(word, limit)
twitter_search.process()
people = twitter_search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'twitter')
if len(people) == 0:
print('\n[*] No users found.\n\n')
else:
print('\n[*] Users found: ' + str(len(people)))
print('---------------------')
for user in sorted(list(set(people))):
print(user)
elif engineitem == 'virustotal':
print('\033[94m[*] Searching VirusTotal. \033[0m')
from theHarvester.discovery import virustotal
virustotal_search = virustotal.SearchVirustotal(word)
virustotal_search.process()
hosts = filter(virustotal_search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'virustotal')
elif engineitem == 'yahoo':
print('\033[94m[*] Searching Yahoo. \033[0m')
from theHarvester.discovery import yahoosearch
yahoo_search = yahoosearch.SearchYahoo(word, limit)
yahoo_search.process()
hosts = yahoo_search.get_hostnames()
emails = yahoo_search.get_emails()
all_hosts.extend(filter(hosts))
all_emails.extend(filter(emails))
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'yahoo')
db.store_all(word, all_emails, 'email', 'yahoo')
else:
print('\033[93m[!] Invalid source.\n\n \033[0m')
sys.exit(1)
# Sanity check to see if all_emails and all_hosts are defined.
try:
all_emails
except NameError:
print(
'\n\n\033[93m[!] No emails found because all_emails is not defined.\n\n \033[0m'
)
sys.exit(1)
try:
all_hosts
except NameError:
print(
'\n\n\033[93m[!] No hosts found because all_hosts is not defined.\n\n \033[0m'
)
sys.exit(1)
# Results
if len(all_ip) == 0:
print('\n[*] No IPs found.')
else:
print('\n[*] IPs found: ' + str(len(all_ip)))
print('-------------------')
ips = sorted(
ipaddress.ip_address(line.strip()) for line in set(all_ip))
print('\n'.join(map(str, ips)))
if len(all_emails) == 0:
print('\n[*] No emails found.')
else:
print('\n[*] Emails found: ' + str(len(all_emails)))
print('----------------------')
print(('\n'.join(sorted(list(set(all_emails))))))
if len(all_hosts) == 0:
print('\n[*] No hosts found.\n\n')
else:
print('\n[*] Hosts found: ' + str(len(all_hosts)))
print('---------------------')
all_hosts = sorted(list(set(all_hosts)))
full_host = hostchecker.Checker(all_hosts)
full = full_host.check()
for host in full:
ip = host.split(':')[1]
print(host)
if ip != 'empty':
if host_ip.count(ip.lower()):
pass
else:
host_ip.append(ip.lower())
db = stash.stash_manager()
db.store_all(word, host_ip, 'ip', 'DNS-resolver')
if trello_info[1] is True:
trello_urls = trello_info[0]
if trello_urls is []:
print('\n[*] No URLs found.')
else:
total = len(trello_urls)
print('\n[*] URLs found: ' + str(total))
print('--------------------')
for url in sorted(list(set(trello_urls))):
print(url)
# DNS brute force
# dnsres = []
if dnsbrute is True:
print('\n[*] Starting DNS brute force.')
a = dnssearch.DnsForce(word, dnsserver, verbose=True)
res = a.process()
# print('\n[*] Hosts found after DNS brute force:')
# for y in res:
# print('-------------------------------------')
# print(y)
# dnsres.append(y.split(':')[0])
# if y not in full:
# full.append(y)
# db = stash.stash_manager()
# db.store_all(word, dnsres, 'host', 'dns_bruteforce')
# Port scanning
if ports_scanning is True:
print('\n\n[*] Scanning ports (active).\n')
for x in full:
host = x.split(':')[1]
domain = x.split(':')[0]
if host != 'empty':
print(('[*] Scanning ' + host))
ports = [21, 22, 80, 443, 8080]
try:
scan = port_scanner.PortScan(host, ports)
openports = scan.process()
if len(openports) > 1:
print(('\t[*] Detected open ports: ' +
','.join(str(e) for e in openports)))
takeover_check = 'True'
if takeover_check == 'True' and len(openports) > 0:
search_take = takeover.TakeOver(domain)
search_take.process()
except Exception as e:
print(e)
# DNS reverse lookup
dnsrev = []
if dnslookup is True:
print('\n[*] Starting active queries.')
analyzed_ranges = []
for entry in host_ip:
print(entry)
ip = entry.split(':')[0]
ip_range = ip.split('.')
ip_range[3] = '0/24'
s = '.'
ip_range = s.join(ip_range)
if not analyzed_ranges.count(ip_range):
print('[*] Performing reverse lookup in ' + ip_range)
a = dnssearch.DnsReverse(ip_range, True)
a.list()
res = a.process()
analyzed_ranges.append(ip_range)
else:
continue
for entries in res:
if entries.count(word):
dnsrev.append(entries)
if entries not in full:
full.append(entries)
print('[*] Hosts found after reverse lookup (in target domain):')
print('--------------------------------------------------------')
for xh in dnsrev:
print(xh)
# DNS TLD expansion
dnstldres = []
if dnstld is True:
print('[*] Starting DNS TLD expansion.')
a = dnssearch.DnsTld(word, dnsserver, verbose=True)
res = a.process()
print('\n[*] Hosts found after DNS TLD expansion:')
print('----------------------------------------')
for y in res:
print(y)
dnstldres.append(y)
if y not in full:
full.append(y)
# Virtual hosts search
if virtual == 'basic':
print('\n[*] Virtual hosts:')
print('------------------')
for l in host_ip:
basic_search = bingsearch.SearchBing(l, limit, start)
basic_search.process_vhost()
results = basic_search.get_allhostnames()
for result in results:
result = re.sub(r'[[\<\/?]*[\w]*>]*', '', result)
result = re.sub('<', '', result)
result = re.sub('>', '', result)
print((l + '\t' + result))
vhost.append(l + ':' + result)
full.append(l + ':' + result)
vhost = sorted(set(vhost))
else:
pass
# Shodan
shodanres = []
if shodan is True:
import texttable
tab = texttable.Texttable()
header = [
'IP address', 'Hostname', 'Org', 'Services:Ports', 'Technologies'
]
tab.header(header)
tab.set_cols_align(['c', 'c', 'c', 'c', 'c'])
tab.set_cols_valign(['m', 'm', 'm', 'm', 'm'])
tab.set_chars(['-', '|', '+', '#'])
tab.set_cols_width([15, 20, 15, 15, 18])
host_ip = list(set(host_ip))
print('\033[94m[*] Searching Shodan. \033[0m')
try:
for ip in host_ip:
print(('\tSearching for ' + ip))
shodan = shodansearch.SearchShodan()
rowdata = shodan.search_ip(ip)
time.sleep(2)
tab.add_row(rowdata)
printedtable = tab.draw()
print(printedtable)
except Exception as e:
print(f'\033[93m[!] An error occurred with Shodan: {e} \033[0m')
else:
pass
# Here we need to add explosion mode.
# We have to take out the TLDs to do this.
recursion = None
if recursion:
counter = 0
for word in vhost:
search = googlesearch.SearchGoogle(word, limit, counter)
search.process(google_dorking)
emails = search.get_emails()
hosts = search.get_hostnames()
print(emails)
print(hosts)
else:
pass
# Reporting
if filename != "":
try:
print('\n[*] Reporting started.')
db = stash.stash_manager()
scanboarddata = db.getscanboarddata()
latestscanresults = db.getlatestscanresults(word)
previousscanresults = db.getlatestscanresults(word,
previousday=True)
latestscanchartdata = db.latestscanchartdata(word)
scanhistorydomain = db.getscanhistorydomain(word)
pluginscanstatistics = db.getpluginscanstatistics()
generator = statichtmlgenerator.HtmlGenerator(word)
HTMLcode = generator.beginhtml()
HTMLcode += generator.generatelatestscanresults(latestscanresults)
HTMLcode += generator.generatepreviousscanresults(
previousscanresults)
graph = reportgraph.GraphGenerator(word)
HTMLcode += graph.drawlatestscangraph(word, latestscanchartdata)
HTMLcode += graph.drawscattergraphscanhistory(
word, scanhistorydomain)
HTMLcode += generator.generatepluginscanstatistics(
pluginscanstatistics)
HTMLcode += generator.generatedashboardcode(scanboarddata)
HTMLcode += '<p><span style="color: #000000;">Report generated on ' + str(
datetime.datetime.now()) + '</span></p>'
HTMLcode += '''
</body>
</html>
'''
Html_file = open(filename, 'w')
Html_file.write(HTMLcode)
Html_file.close()
print('[*] Reporting finished.')
print('[*] Saving files.')
except Exception as e:
print(e)
print(
'\n\033[93m[!] An error occurred while creating the output file.\n\n \033[0m'
)
sys.exit(1)
try:
filename = filename.split('.')[0] + '.xml'
file = open(filename, 'w')
file.write('<?xml version="1.0" encoding="UTF-8"?><theHarvester>')
for x in all_emails:
file.write('<email>' + x + '</email>')
for x in full:
x = x.split(':')
if len(x) == 2:
file.write('<host>' + '<ip>' + x[1] + '</ip><hostname>' +
x[0] + '</hostname>' + '</host>')
else:
file.write('<host>' + x + '</host>')
for x in vhost:
x = x.split(':')
if len(x) == 2:
file.write('<vhost>' + '<ip>' + x[1] + '</ip><hostname>' +
x[0] + '</hostname>' + '</vhost>')
else:
file.write('<vhost>' + x + '</vhost>')
if shodanres != []:
shodanalysis = []
for x in shodanres:
res = x.split('SAPO')
file.write('<shodan>')
file.write('<host>' + res[0] + '</host>')
file.write('<port>' + res[2] + '</port>')
file.write('<banner><!--' + res[1] + '--></banner>')
reg_server = re.compile('Server:.*')
temp = reg_server.findall(res[1])
if temp != []:
shodanalysis.append(res[0] + ':' + temp[0])
file.write('</shodan>')
if shodanalysis != []:
shodanalysis = sorted(set(shodanalysis))
file.write('<servers>')
for x in shodanalysis:
file.write('<server>' + x + '</server>')
file.write('</servers>')
file.write('</theHarvester>')
file.flush()
file.close()
print('[*] Files saved.')
except Exception as er:
print(
f'\033[93m[!] An error occurred while saving the XML file: {er} \033[0m'
)
print('\n\n')
sys.exit(0)
def main():
# The first, and only argument needs to be a directory
parser = argparse.ArgumentParser(
description=
"Checks for duplicated images in a directory tree. Compares just image data, metadata is ignored, so physically different files may be reported as duplicates if they have different metadata (tags, titles, JPEG rotation, EXIF info...)."
)
parser.add_argument('directory', help="Base directory to check")
parser.add_argument('-1',
'--sameline',
help="List each set of matches in a single line",
action='store_true',
required=False)
parser.add_argument(
'-d',
'--delete',
help="Prompt user for files to preserve, deleting all others",
action='store_true',
required=False)
parser.add_argument(
'-a',
'--auto',
help=
"If -d or --delete flag is enabled, always select the 'auto' action to select the file to preserve.",
action='store_true',
required=False)
parser.add_argument('-c',
'--clean',
help="Don't write to disk signatures cache",
action='store_true',
required=False)
parser.add_argument(
'-m',
'--method',
help=
"Hash method to use. Default is MD5, but CRC might be faster on slower CPUs where process is not I/O bound",
default="MD5",
choices=["MD5", "CRC"],
required=False)
parser.add_argument('--version',
action='version',
version='%(prog)s ' + VERSION)
args = parser.parse_args()
if args.auto and not args.delete:
sys.stderr.write(
"'-a' or '--auto' only makes sense when deleting files with '-d' or '--delete'\n"
)
exit(1)
pwd = os.getcwd()
# Check if jpeginfo is installed
havejpeginfo = True
try:
devnull = open(os.devnull, 'w')
check_call(["jpeginfo", "--version"], stdout=devnull, stderr=devnull)
sys.stderr.write(
"jpeginfo found in system, will be used to check JPEG file integrity\n"
)
except (sub.CalledProcessError, FileNotFoundError):
sys.stderr.write(
"jpeginfo not found in system, please install it for smarter JPEG file integrity detection\n"
)
havejpeginfo = False
try:
os.chdir(args.directory)
except:
sys.stderr.write("Directory %s doesn't exist\n" % args.directory)
exit(1)
# Extensiones admitidas (case insensitive)
extensiones = ('jpg', 'jpeg')
# Diccionario con información sobre los ficheros
jpegs = {}
# Flag para saber si hay que volver a escribir la caché por cambios
modif = False
# Recuperar información de los ficheros generada previamente, si existe
rootDir = '.'
fsigs = '.signatures'
if os.path.isfile(fsigs):
cache = open(fsigs, 'rb')
try:
sys.stderr.write("Signatures cache detected, loading...\n")
jpegs = pickle.load(file=cache)
cache.close()
# Clean up non-existing entries
sys.stderr.write("Cleaning up deleted files from cache...\n")
jpegs = dict(
[x for x in iter(jpegs.items()) if os.path.exists(x[0])])
except (pickle.UnpicklingError, KeyError, EOFError, ImportError,
IndexError):
# Si el fichero no es válido, ignorarlo y marcar que hay que escribir cambios
jpegs = {}
modif = True
cache.close()
# Create process pool for parallel hash calculation
pool = Pool()
count = 1
for dirName, subdirList, fileList in os.walk(rootDir):
sys.stderr.write('Exploring %s\n' % dirName)
for fname in fileList:
# Update signatures cache every 100 files
if modif and ((count % 100) == 0):
writecache(jpegs, args, fsigs)
modif = False
if fname.lower().endswith(extensiones):
ruta = os.path.join(dirName, fname)
# Si el fichero no está en la caché, o está pero con tamaño diferente, añadirlo
if (ruta not in jpegs) or (
(ruta in jpegs) and
(jpegs[ruta]['size'] != os.path.getsize(ruta))):
sys.stderr.write(" Calculating hash of %s...\n" % ruta)
jpegs[ruta] = {
'name': fname,
'dir': dirName,
'hash': hashcalc(ruta, pool, args.method,
havejpeginfo),
'size': os.path.getsize(ruta)
}
modif = True
count += 1
# Write hash cache to disk
if modif: writecache(jpegs, args, fsigs)
# Check for duplicates
# Create a new dict indexed by hash
# Initialize dict with an empty list for every possible hash
hashes = {}
for f in jpegs:
for h in jpegs[f]['hash']:
hashes[h] = []
# Group files with the same hash together
for f in jpegs:
for h in jpegs[f]['hash']:
hashes[h].append(jpegs[f])
# Discard hashes without duplicates
dupes = {}
for h in hashes:
if len(hashes[h]) > 1:
dupes[h] = hashes[h]
# Delete entries whose hash couldn't be generated, so they're not reported as duplicates
if 'ERR' in dupes: del dupes['ERR']
# Discard duplicated sets (probably a lot if --rotations is activated)
nodupes = []
for elem in list(dupes.values()):
if not elem in nodupes:
nodupes.append(elem)
# Cleanup. Not strictly necessary, but if there're a lot of files these can get quite big
del hashes, dupes
nset = 1
tmpdirs = []
for dupset in nodupes:
# Add path field (for convenience) and sort by file path
for d in dupset:
d.update({"path": os.path.join(d["dir"], d["name"])})
dupset.sort(key=lambda k: k['path'])
print()
if args.delete:
# Calculate best guess for auto mode
dupaux = [d['path'] for d in dupset]
# Sort by path length (probably not needed as dupset is already sorted, but just in case)
dupaux.sort(key=len)
# Best guess is the entry with most tags, or the one with shorter path if tags are equal (due to previous sort)
bestguess = dupaux.index(
max(dupaux, key=lambda k: len(metadata_summary(k)["tags"])))
optselected = False
while not optselected:
# Prompt for which duplicated file to keep, delete the others
t = tt.Texttable()
t.set_cols_align(["c", "r", "l", "l", "l", "l", "l", "l"])
t.set_chars(['-', '|', '+', '-'])
t.set_deco(t.HEADER)
# Get terminal width in order to set column sizes.
colsize = int(os.popen('stty size', 'r').read().split()[1])
t.set_cols_width(
[1, 1, 50, 20, 11, 10, 10, colsize - 103 - 30])
rws = [[
"", "#", "file", "date", "orientation", "title",
"software", "tags"
]]
for i in range(len(dupset)):
aux = dupset[i]
md = metadata_summary(aux["path"])
rws.append([
"*" if i == bestguess else " ", i + 1,
dupset[i]["path"], md["date"], md["orientation"],
md["title"], md["software"], ", ".join(md["tags"])
])
t.add_rows(rws)
print(("\n" + t.draw()))
if args.auto:
answer = 'auto'
else:
answer = input(
"\nSet %d of %d, preserve files [%d - %d, all, auto, show, detail, help, quit] (default: auto): "
% (nset, len(nodupes), 1, len(dupset)))
if answer in ["detail", "d"]:
# Show detailed differences in EXIF tags
filelist = [
os.path.join(x['dir'], x['name']) for x in dupset
]
metadata_comp_table(filelist)
elif answer in ["help", "h"]:
print()
print("[0-9]: Keep the selected file, delete the rest")
print("(a)ll: Keep all files, don't delete anything")
print(
"auto: Keep picture with most tags, or shorter path. If equal, don't delete anything"
)
print(
"(s)how: Copy duplicated files to a temporary directory and open in a file manager window (desktop default)"
)
print(
"(d)etail: Show a detailed table with metadata differences between files"
)
print("(h)elp: Show this screen")
print("(q)uit: Exit program")
print()
elif answer in ["quit", "q"]:
# If asked, write changes, delete temps and quit
if modif: writecache(jpegs, args, fsigs)
rmtemps(tmpdirs)
exit(0)
elif answer in ["show", "s"]:
# Create a temporary directory, copy duplicated files and open a file manager
tmpdir = tempfile.mkdtemp()
tmpdirs.append(tmpdir)
for i in range(len(dupset)):
p = dupset[i]['path']
ntemp = "%d_%s" % (i, dupset[i]['name'])
shutil.copyfile(p, os.path.join(tmpdir, ntemp))
sub.Popen(["xdg-open", tmpdir], stdout=None, stderr=None)
elif answer in ["all", "a"]:
# Don't delete anything
sys.stderr.write(
"Skipping deletion, all duplicates remain\n")
optselected = True
elif answer in ["auto", ""]:
answer = bestguess
else:
# If it's no option, assume it's a number and convert it to an array index
answer = int(answer) - 1
# If we have a valid number as an answer, delete all but the selected file
try:
answer = int(answer)
for i in range(len(dupset)):
if i != answer:
p = dupset[i]['path']
os.remove(p)
del jpegs[p]
modif = True
sys.stderr.write("Kept %s, deleted others\n" %
(dupset[answer]["name"]))
optselected = True
except ValueError:
pass
nset += 1
else:
# Just show duplicates
for f in dupset:
print(f['path'], end=' ')
if not args.sameline:
print("\n", end=' ')
# Final update of the cache in order to remove signatures of deleted files
if modif: writecache(jpegs, args, fsigs)
# Delete temps
rmtemps(tmpdirs)
# Restore directory
os.chdir(pwd)
def run(self):
citationMap = {
'Mike': {
'Mike': 0,
'Jim': 0,
'Mary': 0,
'Bob': 0,
'Ann': 0,
'Joe': 0,
'Nancy': 0
},
'Jim': {
'Mike': 20,
'Jim': 0,
'Mary': 20,
'Bob': 20,
'Ann': 0,
'Joe': 20,
'Nancy': 0
},
'Mary': {
'Mike': 1,
'Jim': 10,
'Mary': 0,
'Bob': 1,
'Ann': 0,
'Joe': 1,
'Nancy': 0
},
'Bob': {
'Mike': 1,
'Jim': 10,
'Mary': 1,
'Bob': 0,
'Ann': 0,
'Joe': 1,
'Nancy': 0
},
'Ann': {
'Mike': 0,
'Jim': 0,
'Mary': 0,
'Bob': 0,
'Ann': 0,
'Joe': 0,
'Nancy': 0
},
'Joe': {
'Mike': 0,
'Jim': 0,
'Mary': 0,
'Bob': 0,
'Ann': 0,
'Joe': 0,
'Nancy': 0
},
'Nancy': {
'Mike': 1,
'Jim': 10,
'Mary': 1,
'Bob': 1,
'Ann': 0,
'Joe': 1,
'Nancy': 0
}
}
self.graph, authorMap, conferenceMap =\
SampleGraphUtility.constructPathSimExampleThree(extraAuthorsAndCitations=True, citationMap = citationMap)
# Get the nodes we care about
conferences = [
conferenceMap['SIGMOD'], conferenceMap['VLDB'],
conferenceMap['ICDE'], conferenceMap['KDD']
]
authors = [
authorMap['Mike'],
authorMap['Jim'],
authorMap['Mary'],
authorMap['Bob'],
authorMap['Ann'],
authorMap['Joe'],
authorMap['Nancy'],
]
metaPathUtility = EdgeBasedMetaPathUtility()
# Project a 2-typed heterogeneous graph over adapted PathSim example
publicationProjectedGraph = metaPathUtility.createHeterogeneousProjection(
self.graph, [Author, Paper, Conference], symmetric=True)
self.output('\nAdjacency Matrix (Projected):')
adjMatrixTable = texttable.Texttable()
rows = [['Author'] + [conference.name for conference in conferences]]
rows += [[author.name] + [
publicationProjectedGraph.getNumberOfEdges(author, conference)
for conference in conferences
] for author in authors]
adjMatrixTable.add_rows(rows)
self.output(adjMatrixTable.draw())
# Project a homogeneous citation graph over adapted PathSim example
citationProjectedGraph = metaPathUtility.createHomogeneousProjection(
self.graph, [Author, Paper, Paper, Author])
self.output('\nCitation Matrix:')
adjMatrixTable = texttable.Texttable()
rows = [['Author'] + [author.name for author in authors]]
rows += [[author.name] + [
citationProjectedGraph.getNumberOfEdges(author, otherAuthor)
for otherAuthor in authors
] for author in authors]
adjMatrixTable.add_rows(rows)
self.output(adjMatrixTable.draw())
# Output total out/in citations
self.output('\nCitations Total:')
totalCitationsTable = texttable.Texttable()
rows = [['Author', 'In', 'Out']]
for author in authors:
inCount = sum(
citationProjectedGraph.getNumberOfEdges(otherAuthor, author)
for otherAuthor in authors)
outCount = sum(
citationProjectedGraph.getNumberOfEdges(author, otherAuthor)
for otherAuthor in authors)
rows += [[author.name, inCount, outCount]]
totalCitationsTable.add_rows(rows)
self.output(totalCitationsTable.draw())
# Get PathSim similarity scores
pathSimStrategy = PathSimStrategy(
self.graph, [Author, Paper, Conference, Paper, Author], True)
self.outputSimilarityScores(authorMap, authors, pathSimStrategy,
'APCPA PathSim')
# Output SimRank-related scores
strategy = SimRankStrategy(self.graph, [Author, Paper, Paper, Author],
symmetric=True)
self.outputSimilarityScores(authorMap, authors, strategy, "SimRank")
# Output the projected PageRank/HITS similarity scores
for name, algorithm in zip(
['PageRank', 'HITS'],
[PageRankDistanceStrategy, HITSDistanceStrategy]):
strategy = algorithm(self.graph, [Author, Paper, Paper, Author],
symmetric=True)
self.outputSimilarityScores(authorMap, authors, strategy,
"%s-Distance" % name)
# Get NeighborSim similarity scores
inNeighborSimStrategy = NeighborSimStrategy(
self.graph, [Author, Paper, Paper, Author])
self.outputSimilarityScores(authorMap, authors, inNeighborSimStrategy,
'APPA NeighborSim-In')
outNeighborSimStrategy = NeighborSimStrategy(
self.graph, [Author, Paper, Paper, Author],
reversed=True,
smoothed=True)
self.outputSimilarityScores(authorMap, authors, outNeighborSimStrategy,
'APPA NeighborSim-Out')
# Combined best PR-distance algorithm
simRankStrategy = SimRankStrategy(self.graph,
[Author, Paper, Paper, Author],
symmetric=True)
simRank = AggregateSimilarityStrategy(
self.graph, [pathSimStrategy, simRankStrategy], [0.5, 0.5])
self.outputSimilarityScores(authorMap, authors, simRank,
'APCPA Pathsim, APPA SimRank')
# Combined best neighborsim score
combinedNeighborSim = AggregateSimilarityStrategy(
self.graph,
[pathSimStrategy, inNeighborSimStrategy, outNeighborSimStrategy],
[0.6, 0.2, 0.2])
self.outputSimilarityScores(
authorMap, authors, combinedNeighborSim,
'APCPA Pathsim, APPA NeighborSim-Combined')
def analyze(logfilenames: typing.List[str], clipterminals: bool, bytmp: bool,
bybitfield: bool) -> None:
data: typing.Dict[str, typing.Dict[str, typing.List[float]]] = {}
for logfilename in logfilenames:
with open(logfilename, 'r') as f:
# Record of slicing and data associated with the slice
sl = 'x' # Slice key
phase_time: typing.Dict[str,
float] = {} # Map from phase index to time
n_sorts = 0
n_uniform = 0
is_first_last = False
# Read the logfile, triggering various behaviors on various
# regex matches.
for line in f:
# Beginning of plot job. We may encounter this multiple
# times, if a job was run with -n > 1. Sample log line:
# 2021-04-08T13:33:43.542 chia.plotting.create_plots : INFO Starting plot 1/5
m = re.search(r'Starting plot (\d*)/(\d*)', line)
if m:
# (re)-initialize data structures
sl = 'x' # Slice key
phase_time = {} # Map from phase index to time
n_sorts = 0
n_uniform = 0
seq_num = int(m.group(1))
seq_total = int(m.group(2))
is_first_last = seq_num == 1 or seq_num == seq_total
# Temp dirs. Sample log line:
# Starting plotting progress into temporary dirs: /mnt/tmp/01 and /mnt/tmp/a
m = re.search(r'^Starting plotting.*dirs: (.*) and (.*)', line)
if m:
# Record tmpdir, if slicing by it
if bytmp:
tmpdir = m.group(1)
sl += '-' + tmpdir
# Bitfield marker. Sample log line(s):
# Starting phase 2/4: Backpropagation without bitfield into tmp files... Mon Mar 1 03:56:11 2021
# or
# Starting phase 2/4: Backpropagation into tmp files... Fri Apr 2 03:17:32 2021
m = re.search(r'^Starting phase 2/4: Backpropagation', line)
if bybitfield and m:
if 'without bitfield' in line:
sl += '-nobitfield'
else:
sl += '-bitfield'
# CHIA: Phase timing. Sample log line:
# Time for phase 1 = 22796.7 seconds. CPU (98%) Tue Sep 29 17:57:19 2020
for phase in ['1', '2', '3', '4']:
m = re.search(
r'^Time for phase ' + phase +
' = (\d+.\d+) seconds..*', line)
if m:
phase_time[phase] = float(m.group(1))
# MADMAX: Phase timing. Sample log line: "Phase 2 took 2193.37 sec"
for phase in ['1', '2', '3', '4']:
m = re.search(r'^Phase ' + phase + ' took (\d+.\d+) sec.*',
line)
if m:
phase_time[phase] = float(m.group(1))
# Uniform sort. Sample log line:
# Bucket 267 uniform sort. Ram: 0.920GiB, u_sort min: 0.688GiB, qs min: 0.172GiB.
# or
# ....?....
# or
# Bucket 511 QS. Ram: 0.920GiB, u_sort min: 0.375GiB, qs min: 0.094GiB. force_qs: 1
m = re.search(r'Bucket \d+ ([^\.]+)\..*', line)
if m and not 'force_qs' in line:
sorter = m.group(1)
n_sorts += 1
if sorter == 'uniform sort':
n_uniform += 1
elif sorter == 'QS':
pass
else:
print('Warning: unrecognized sort ' + sorter)
# CHIA: Job completion. Record total time in sliced data store.
# Sample log line:
# Total time = 49487.1 seconds. CPU (97.26%) Wed Sep 30 01:22:10 2020
m = re.search(r'^Total time = (\d+.\d+) seconds.*', line)
if m:
if clipterminals and is_first_last:
pass # Drop this data; omit from statistics.
else:
data.setdefault(sl, {}).setdefault(
'total time', []).append(float(m.group(1)))
for phase in ['1', '2', '3', '4']:
data.setdefault(sl, {}).setdefault(
'phase ' + phase, []).append(phase_time[phase])
data.setdefault(sl, {}).setdefault(
'%usort', []).append(100 * n_uniform // n_sorts)
# MADMAX: Job completion. Record total time in sliced data store.
# Sample log line: "Total plot creation time was 2530.76 sec"
m = re.search(r'^Total plot creation time was (\d+.\d+) sec.*',
line)
if m:
data.setdefault(sl, {}).setdefault('total time',
[]).append(
float(m.group(1)))
for phase in ['1', '2', '3', '4']:
data.setdefault(sl, {}).setdefault(
'phase ' + phase, []).append(phase_time[phase])
data.setdefault(sl, {}).setdefault('%usort', []).append(
0) # Not available for MADMAX
# Prepare report
tab = tt.Texttable()
all_measures = [
'%usort', 'phase 1', 'phase 2', 'phase 3', 'phase 4', 'total time'
]
headings = ['Slice', 'n'] + all_measures
tab.header(headings)
for sl in data.keys():
row = [sl]
# Sample size
sample_sizes = []
for measure in all_measures:
values = data.get(sl, {}).get(measure, [])
sample_sizes.append(len(values))
sample_size_lower_bound = min(sample_sizes)
sample_size_upper_bound = max(sample_sizes)
if sample_size_lower_bound == sample_size_upper_bound:
row.append('%d' % sample_size_lower_bound)
else:
row.append('%d-%d' %
(sample_size_lower_bound, sample_size_upper_bound))
# Phase timings
for measure in all_measures:
values = data.get(sl, {}).get(measure, [])
if (len(values) > 1):
row.append(
'μ=%s σ=%s' %
(plot_util.human_format(statistics.mean(values), 1),
plot_util.human_format(statistics.stdev(values), 0)))
elif (len(values) == 1):
row.append(plot_util.human_format(values[0], 1))
else:
row.append('N/A')
tab.add_row(row)
(rows, columns) = os.popen('stty size', 'r').read().split()
tab.set_max_width(int(columns))
s = tab.draw()
print(s)
def format_table(rows):
table = texttable.Texttable(max_width=0)
table.header(('dBV', 'dBu', 'Volts RMS', 'Volts peak-to-peak'))
table.add_rows(rows, header=False)
table.set_cols_align(['r', 'r', 'r', 'r'])
return table.draw()
def fetch_plugin_index(self,
props,
_list=False,
list_header=False,
list_long=False,
accept_license=False,
icon=False,
official=False):
if self.server == "download.freebsd.org":
git_server = "https://github.com/freenas/iocage-ix-plugins.git"
else:
git_server = self.server
git_working_dir = f"{self.iocroot}/.plugin_index"
# list --plugins won't often be root.
if os.geteuid() == 0:
try:
self.__clone_repo(git_server, git_working_dir)
except Exception as err:
iocage_lib.ioc_common.logit(
{
"level": "EXCEPTION",
"message": err
},
_callback=self.callback)
with open(f"{self.iocroot}/.plugin_index/INDEX", "r") as plugins:
plugins = json.load(plugins)
_plugins = self.__fetch_sort_plugin__(plugins, official=official)
if self.plugin is None and not _list:
for p in _plugins:
iocage_lib.ioc_common.logit(
{
"level": "INFO",
"message": f"[{_plugins.index(p)}] {p}"
},
_callback=self.callback,
silent=self.silent)
if _list:
plugin_list = []
for p in _plugins:
p = p.split("-", 1)
name = p[0]
desc, pkg = re.sub(r'[()]', '', p[1]).rsplit(" ", 1)
license = plugins[pkg].get("license", "")
_official = str(plugins[pkg].get("official", False))
icon_path = plugins[pkg].get("icon", None)
p = [name, desc, pkg]
if not list_header:
p += [license, _official]
if icon:
p += [icon_path]
if official and _official == "False":
continue
plugin_list.append(p)
if not list_header:
return plugin_list
else:
if list_long:
table = texttable.Texttable(max_width=0)
else:
table = texttable.Texttable(max_width=80)
list_header = ["NAME", "DESCRIPTION", "PKG"]
if icon:
list_header += ["ICON"]
plugin_list.insert(0, list_header)
table.add_rows(plugin_list)
return table.draw()
if self.plugin is None:
self.plugin = input("\nType the number of the desired"
" plugin\nPress [Enter] or type EXIT to"
" quit: ")
self.plugin = self.__fetch_validate_plugin__(self.plugin.lower(),
_plugins)
self.fetch_plugin(f"{self.iocroot}/.plugin_index/{self.plugin}.json",
props, 0, accept_license)
if isinstance(self.spec[k],list):
for v in self.spec[k]:
if hasattr(v, 'prettyprint'):
v = v.prettyprint(max_width=max_width-20)
else:
v = str(v)
summary += v + "\n"
else:
v = self.spec[k]
if hasattr(v, 'prettyprint'):
summary = v.prettyprint(max_width=max_width-20)
else:
summary = str(v)
rows.append((k, summary))
table = texttable.Texttable(max_width=max_width)
table.add_rows(rows)
return table.draw()
<<<<<<< HEAD
=======
class SpecManagerBase():
"""
Manage a specification directory
"""
def __init__(self, *args, **kwargs):
self.specs = []
"""
Accept path and read list of spec files available in path.
required prefix: spec_
import texttable as tt
carlist = [['DT-LT-87', 'Citroen', 'XM', '1999-09-23', 34500],
['GF-NX-07', 'Volkswagen', 'Polo', "1999-07-12", 78000],
['GF-PD-34', 'Volkswagen', 'Polo', "1999-07-22", 57500],
['KR-RT-65', 'Volkswagen', 'Golf', "1999-08-08", 42000],
['PT-ER-45', 'Ford', 'Fiesta', "1999-03-02", 25000],
['TT-PR-73', 'Citroen', 'XM', '1999-03-02', 1200],
['TT-RW-01', 'Volkswagen', 'Polo', "1999-11-14", 4500]]
tab = tt.Texttable()
header = ['Kenteken', 'Merk', 'Type', 'DatumAPK', 'Kilometerstand']
tab.header(header)
for c in carlist:
tab.add_row(c)
draw = tab.draw()
print(draw)
def list_all(self, jails):
"""List all jails."""
self.full = True if self.plugin else self.full
jail_list = []
plugin_index_data = {}
for jail in jails:
try:
mountpoint = jail.properties['mountpoint']
except KeyError:
iocage_lib.ioc_common.logit(
{
'level':
'ERROR',
'message':
f'{jail.name} mountpoint is misconfigured. '
'Please correct this.'
},
_callback=self.callback,
silent=self.silent)
continue
try:
conf = iocage_lib.ioc_json.IOCJson(mountpoint).json_get_value(
'all')
state = ''
except (Exception, SystemExit):
# Jail is corrupt, we want all the keys to exist.
# So we will take the defaults and let the user
# know that they are not correct.
def_props = iocage_lib.ioc_json.IOCJson().json_get_value(
'all', default=True)
conf = {x: 'N/A' for x in def_props}
conf['host_hostuuid'] = \
f'{jail.name.split("/")[-1]}'
conf['release'] = 'N/A'
state = 'CORRUPT'
jid = '-'
if self.basejail_only and not iocage_lib.ioc_common.check_truthy(
conf.get('basejail', 0)):
continue
uuid_full = conf["host_hostuuid"]
uuid = uuid_full
if not self.full:
# We only want to show the first 8 characters of a UUID,
# if it's not a UUID, we will show the whole name no matter
# what.
try:
uuid = str(_uuid.UUID(uuid, version=4))[:8]
except ValueError:
# We leave the "uuid" untouched, as it's not a valid
# UUID, but instead a named jail.
pass
full_ip4 = conf["ip4_addr"]
ip6 = conf["ip6_addr"]
try:
short_ip4 = ",".join([
item.split("|")[1].split("/")[0]
for item in full_ip4.split(",")
])
except IndexError:
short_ip4 = full_ip4 if full_ip4 != "none" else "-"
boot = 'on' if iocage_lib.ioc_common.check_truthy(
conf.get('boot', 0)) else 'off'
jail_type = conf["type"]
full_release = conf["release"]
basejail = 'yes' if iocage_lib.ioc_common.check_truthy(
conf.get('basejail', 0)) else 'no'
if "HBSD" in full_release:
full_release = re.sub(r"\W\w.", "-", full_release)
full_release = full_release.replace("--SD", "-STABLE-HBSD")
short_release = full_release.rstrip("-HBSD")
else:
short_release = "-".join(full_release.rsplit("-")[:2])
if full_ip4 == "none":
full_ip4 = "-"
if ip6 == "none":
ip6 = "-"
# Will be set already by a corrupt jail
status = False
if state != 'CORRUPT':
status, jid = self.list_get_jid(uuid_full)
if status:
state = "up"
else:
state = "down"
if conf["type"] == "template":
template = "-"
else:
jail_root = Dataset(f'{jail.name}/root')
if jail_root.exists:
_origin_property = jail_root.properties.get('origin')
else:
_origin_property = None
if _origin_property:
template = _origin_property
template = template.rsplit("/root@", 1)[0].rsplit("/",
1)[-1]
else:
template = "-"
if "release" in template.lower() or "stable" in template.lower():
template = "-"
if iocage_lib.ioc_common.check_truthy(
conf['dhcp']) and status and os.geteuid() == 0:
interface = conf["interfaces"].split(",")[0].split(":")[0]
if interface == "vnet0":
# Inside jails they are epairNb
interface = f"{interface.replace('vnet', 'epair')}b"
short_ip4 = "DHCP"
full_ip4_cmd = [
"jexec", f"ioc-{uuid_full.replace('.', '_')}", "ifconfig",
interface, "inet"
]
try:
out = su.check_output(full_ip4_cmd)
full_ip4 = f'{interface}|' \
f'{out.splitlines()[2].split()[1].decode()}'
except (su.CalledProcessError, IndexError) as e:
short_ip4 += '(Network Issue)'
if isinstance(e, su.CalledProcessError):
full_ip4 = f'DHCP - Network Issue: {e}'
else:
full_ip4 = f'DHCP - Failed Parsing: {e}'
elif iocage_lib.ioc_common.check_truthy(
conf['dhcp']) and not status:
short_ip4 = "DHCP"
full_ip4 = "DHCP (not running)"
elif iocage_lib.ioc_common.check_truthy(
conf['dhcp']) and os.geteuid() != 0:
short_ip4 = "DHCP"
full_ip4 = "DHCP (running -- address requires root)"
# Append the JID and the NAME to the table
if self.full and self.plugin:
if jail_type != "plugin" and jail_type != "pluginv2":
# We only want plugin type jails to be apart of the
# list
continue
try:
with open(f"{mountpoint}/plugin/ui.json", "r") as u:
# We want to ensure that we show the correct NAT
# ports for nat based plugins and when NAT isn't
# desired, we don't show them at all. In all these
# variable values, what persists across NAT/DHCP/Static
# ip based plugins is that the internal ports of the
# jail don't change. For example if a plugin jail has
# nginx running on port 4000, it will still want to
# have it running on 4000 regardless of the fact
# how user configures to start the plugin jail. We take
# this fact, and search for an explicit specified port
# number in the admin portal, if none is found, that
# means that it is ( 80 - default for http ).
nat_forwards_dict = {}
nat_forwards = conf.get('nat_forwards', 'none')
for rule in nat_forwards.split(
',') if nat_forwards != 'none' else ():
# Rule can be proto(port), proto(in/out), port
if rule.isdigit():
jail = host = rule
else:
rule = rule.split('(')[-1].strip(')')
if ':' in rule:
jail, host = rule.split(':')
else:
# only one port provided
jail = host = rule
nat_forwards_dict[int(jail)] = int(host)
if not conf.get('nat'):
all_ips = map(
lambda v: 'DHCP'
if 'dhcp' in v.lower() else v, [
i.split('|')[-1].split('/')[0].strip()
for i in full_ip4.split(',')
])
else:
default_gateways = \
iocage_lib.ioc_common.get_host_gateways()
all_ips = [
f['addr'] for k in default_gateways
if default_gateways[k]['interface']
for f in netifaces.ifaddresses(
default_gateways[k]['interface'])
[netifaces.AF_INET if k ==
'ipv4' else netifaces.AF_INET6]
]
ui_data = json.load(u)
admin_portal = ui_data["adminportal"]
admin_portals = []
for portal in admin_portal.split(','):
if conf.get('nat'):
portal_uri = urllib.parse.urlparse(portal)
portal_port = portal_uri.port or 80
# We do this safely as it's possible
# dev hasn't added it to plugin's json yet
nat_port = nat_forwards_dict.get(portal_port)
if nat_port:
uri = portal_uri._replace(
netloc=f'{portal_uri._hostinfo[0]}:'
f'{nat_port}').geturl()
else:
uri = portal
else:
uri = portal
admin_portals.append(','.join(
map(lambda v: uri.replace('%%IP%%', v),
all_ips)))
admin_portal = ','.join(admin_portals)
try:
ph = ui_data["adminportal_placeholders"].items()
if ph and not status:
admin_portal = f"{uuid} is not running!"
else:
for placeholder, prop in ph:
admin_portal = admin_portal.replace(
placeholder,
iocage_lib.ioc_json.IOCJson(
mountpoint).json_plugin_get_value(
prop.split(".")))
except KeyError:
pass
except iocage_lib.ioc_exceptions.CommandNeedsRoot:
admin_portal = "Admin Portal requires root"
except iocage_lib.ioc_exceptions.CommandFailed as e:
admin_portal = b' '.join(e.message).decode()
doc_url = ui_data.get('docurl', '-')
except FileNotFoundError:
# They just didn't set a admin portal.
admin_portal = doc_url = '-'
jail_list.append([
jid, uuid, boot, state, jail_type, full_release, full_ip4,
ip6, template, admin_portal, doc_url
])
if self.plugin_data:
if conf['plugin_repository'] not in plugin_index_data:
repo_obj = iocage_lib.ioc_plugin.IOCPlugin(
git_repository=conf['plugin_repository'])
if not os.path.exists(repo_obj.git_destination):
repo_obj.pull_clone_git_repo()
with open(
os.path.join(repo_obj.git_destination,
'INDEX')) as f:
plugin_index_data[conf['plugin_repository']] = \
json.loads(f.read())
index_plugin_conf = plugin_index_data[
conf['plugin_repository']].get(conf['plugin_name'], {})
jail_list[-1].extend([
conf['plugin_name'],
conf['plugin_repository'],
index_plugin_conf.get('primary_pkg'),
index_plugin_conf.get('category'),
index_plugin_conf.get('maintainer'),
])
elif self.full:
jail_list.append([
jid, uuid, boot, state, jail_type, full_release, full_ip4,
ip6, template, basejail
])
else:
jail_list.append([jid, uuid, state, short_release, short_ip4])
list_type = "list_full" if self.full else "list_short"
sort = iocage_lib.ioc_common.ioc_sort(list_type,
self.sort,
data=jail_list)
jail_list.sort(key=sort)
# return the list...
if not self.header:
flat_jail = [j for j in jail_list]
return flat_jail
# Prints the table
table = texttable.Texttable(max_width=0)
if self.full:
if self.plugin:
table.set_cols_dtype(
["t", "t", "t", "t", "t", "t", "t", "t", "t", "t", "t"])
jail_list.insert(0, [
"JID", "NAME", "BOOT", "STATE", "TYPE", "RELEASE", "IP4",
"IP6", "TEMPLATE", "PORTAL", "DOC_URL"
])
else:
# We get an infinite float otherwise.
table.set_cols_dtype(
["t", "t", "t", "t", "t", "t", "t", "t", "t", "t"])
jail_list.insert(0, [
"JID", "NAME", "BOOT", "STATE", "TYPE", "RELEASE", "IP4",
"IP6", "TEMPLATE", 'BASEJAIL'
])
else:
# We get an infinite float otherwise.
table.set_cols_dtype(["t", "t", "t", "t", "t"])
jail_list.insert(0, ["JID", "NAME", "STATE", "RELEASE", "IP4"])
table.add_rows(jail_list)
return table.draw()
