class Nmap(DockerCollector):
config = {
'name': 'nmap',
'docker': {'build_context': get_actual_dir()},
}
def callbacks(self):
return {
Domain: self.from_domain,
IPv4: self.from_ip,
}
def _scan(self, target):
data = self.run_container(command=['-oX', '-', '-sS', '-T3', target])
yield
for proto, port, service in self.findall_regex(
data,
r'port protocol="(.*)" portid="(.*)"><state \
state=.* reason=.*service name="(.*)" method=',
):
yield Socket(proto=proto, port=port, service_name=service)
def from_domain(self, domain):
yield from self._scan(domain.fqdn)
def from_ip(self, ip):
yield from self._scan(ip.address)
class BlackWidow(DockerCollector):
config = {
'name': 'black-widow',
'docker': {'build_context': get_actual_dir()},
}
def callbacks(self):
return {
Domain: self.scan,
}
def scan(self, domain):
data = self.run_container(command=['-d', domain.fqdn, '-l', '5', '-v', 'y'])
for email in self.findall_regex(data, r'Email found! (.*) '):
yield Email(address=email)
for number in self.findall_regex(data, r'Telephone # found! (.*) '):
yield Phone(number=number)
for url in self.findall_regex(
data,
r'(https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]\
{2,}|www\.[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|\
https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9]+\.[^\s]{2,}|www\.[a-zA-Z0-9]+\.[^\s]{2,})',
):
yield Uri(location=url)
class TheHarester(DockerCollector):
config = {
'name': 'harvester',
'docker': {'build_context': get_actual_dir()},
}
def callbacks(self):
return {
Domain: self.from_domain,
Company: self.from_company,
}
def from_company(self, company):
yield from self.scan(company.name)
def from_domain(self, domain):
yield from self.scan(domain.fqdn)
def scan(self, target):
data = self.run_container(
command=[
'-d',
target,
'--source',
'baidu,bing,bufferoverun,certspotter,crtsh,dnsdumpster,duckduckgo,\
exalead,google,linkedin,linkedin_links,netcraft,\
omnisint,otx,qwant,rapiddns,threatminer,twitter,urlscan,yahoo',
],
)
for item, _ in self.findall_regex(
data,
r'\[\*\] IPs found: \d+\n-------------------\
\n((.|\n)*)\n\[\*\] Emails found',
):
for ip in item.split('\n'):
if ip:
yield IPv4(address=ip)
for item, _ in self.findall_regex(
data,
r'\[\*\] Emails found: \d+\n----------------------\n((.|\n)*)\n\[\*\] Hosts found',
):
for email in item.split('\n'):
if email:
yield Email(address=email)
for item, _ in self.findall_regex(
data,
r'\[\*\] Hosts found: \d+\n---------------------\n((.|\n)*)',
):
for host in item.split('\n'):
if not host:
continue
if ':' in host:
domain, ip = host.split(':')
yield Domain(fqdn=domain, address=ip)
yield IPv4(address=ip, dns=domain)
else:
yield Domain(fqdn=host)
class ReconNG(DockerCollector):
config = {
'name': 'recon-ng',
'docker': {
'build_context': get_actual_dir()
},
}
def callbacks(self):
return {
Domain: self.from_domain,
Username: self.from_username,
}
def from_domain(self, domain):
data = self.run_container(command=[
'-m',
'recon/domains-hosts/hackertarget',
'-o',
f'SOURCE={domain.fqdn}',
'-x',
], )
for item in self.findall_regex(data, r'Host: (.*)'):
yield Domain(fqdn=item)
def from_username(self, username):
data = self.run_container(
command=['-m', 'profiler', '-o', f'SOURCE={username.name}',
'-x'], )
for category, resource, url in self.findall_regex(
data,
r'Category: (.*)\n.*\n.*Resource: (.*)\n.*Url: (.*)',
):
yield Profile(url=url, category=category, resource=resource)
class PhoneInfoga(DockerCollector):
config = {
'name': 'phone-infoga',
'docker': {
'build_context': get_actual_dir()
},
}
def callbacks(self):
return {Phone: self.scan}
def scan(self, phone):
data = self.run_container(command=['scan', '-n', phone.number])
for item in self.findall_regex(
data,
r'\[i\] Running local scan...\n\[\+\] Local format: (.*)\n\[\+\] \
E164 format:.*\n\[\+\] International format: (.*)\n\[\+\] \
Country found:.*\((.*)\)\n\[\+\] Carrier: (.*)',
):
local_format, international_format, country_code, carrier = item
for item in self.findall_regex(
data,
r'\[i\] Running Numverify.com scan...\n\[\+\] Valid: \
(.*)\n\[\+\] Number:.*\n\[\+\] Local format: (.*)\n\[\+\] International format: \
(.*)\n\[\+\] Country code: (.*) \(.*\n\[\+\] Country: (.*)\n\[\+\] Location: \
(.*)\n\[\+\] Carrier: (.*)\n\[\+\] Line type: (.*)\n',
):
(
valid,
nv_local_format,
nv_international_format,
nv_country_code,
country,
location,
nv_carrier,
line_type,
) = item
local_format = local_format or nv_local_format
international_format = international_format or nv_international_format
country_code = country_code or nv_country_code
carrier = carrier or nv_carrier
yield Phone(
number=phone.number,
valid=valid,
local_format=local_format,
international_format=international_format,
country_code=country_code,
country=country,
location=location,
carrier=carrier,
line_type=line_type,
)
class TheHarvester(DockerCollector):
config = {
"name": "the-harvester",
"docker": {
"build_context": get_actual_dir()
},
}
def callbacks(self):
return {
Domain: self.from_domain,
}
def from_domain(self, domain):
yield from self.scan(domain.fqdn)
def scan(self, target):
data = self.run_container(command=[
"-d",
target,
"--source",
"anubis,baidu,bing,bufferoverun,certspotter,crtsh,dnsdumpster,duckduckgo,google,hackertarget,linkedin,linkedin_links,n45ht,omnisint,qwant,rapiddns,threatcrowd,threatminer,trello,twitter,urlscan,yahoo",
], )
for item, _ in self.findall_regex(
data,
r"\[\*\] IPs found: \d+\n-------------------\n((.|\n)*)\n\[\*\] Emails found",
):
for ip in item.split("\n"):
if ip:
yield from [IPv4(address=i.strip()) for i in ip.split(",")]
for item, _ in self.findall_regex(
data,
r"\[\*\] Emails found: \d+\n----------------------\n((.|\n)*)\n\[\*\] Hosts found",
):
for email in item.split("\n"):
if email:
yield Email(address=email)
for item, _ in self.findall_regex(
data,
r"\[\*\] Hosts found: \d+\n---------------------\n((.|\n)*)",
):
for host in item.split("\n"):
if not host:
continue
if ":" in host:
domain, ip = host.split(":")
yield Domain(fqdn=domain, address=ip)
yield from [
IPv4(address=i.strip(), domain=domain)
for i in ip.split(",")
]
else:
yield Domain(fqdn=host)
class Infoga(DockerCollector):
config = {
'name': 'infoga',
'docker': {'build_context': get_actual_dir()},
}
def callbacks(self):
return {Domain: self.scan}
def scan(self, domain):
data = self.run_container(command=['--domain', domain.fqdn, '-v', '1'])
for item in self.findall_regex(data, r'Email: (.*) \('):
yield Email(address=item)
class Subfinder(DockerCollector):
config = {
'name': 'subfinder',
'docker': {'build_context': get_actual_dir()},
}
def callbacks(self):
return {Domain: self.scan}
def scan(self, domain):
data = self.run_container(command=['-d', domain.fqdn, '-nC', '-silent'])
for domain in data.split('\n'):
yield Domain(fqdn=domain)
class Sherlock(DockerCollector):
config = {
'name': 'sherlock',
'docker': {
'build_context': get_actual_dir()
},
}
def callbacks(self):
return {Username: self.from_username}
def from_username(self, username):
data = self.run_container(command=[
username.name, '--no-color', '--print-found', '--timeout', '20'
], )
for item in self.findall_regex(data, r'\[\+\] .*: (.*)\n'):
yield Profile(url=item)
class Kupa3(DockerCollector):
config = {
'name': 'kupa3',
'docker': {
'build_context': get_actual_dir()
},
}
def callbacks(self):
return {
Domain: self.scan,
}
def scan(self, domain):
data = self.run_container(command=['ls', '-la'])
for item in self.findall_regex(data, r'(.*)'):
yield
class Sherlock(DockerCollector):
config = {
"name": "sherlock",
"docker": {
"build_context": get_actual_dir()
},
}
def callbacks(self):
return {Username: self.from_username}
def from_username(self, username):
data = self.run_container(command=[
username.name, "--no-color", "--print-found", "--timeout", "20"
], )
logger.debug(data)
for item in self.findall_regex(data, r"\[\+\] .*: (.*)\n"):
yield OnlineProfile(url=item)
class Dirsearch(DockerCollector):
config = {
'name': 'dirsearch',
'docker': {
'build_context': get_actual_dir()
},
}
def callbacks(self):
return {Domain: self.from_domain}
def from_domain(self, domain):
data = self.run_container(
command=['-u', domain.fqdn, '-F', '--timeout=5', '-q', '-t',
'4'], )
print('!!!!', data)
for item in self.findall_regex(data, r'2\d\d - .* - ([^\s]+)'):
yield Uri(location=item)
class Twint(DockerCollector):
config = {
'name': 'twint',
'docker': {
'build_context': get_actual_dir()
},
}
def callbacks(self):
return {Username: self.scan}
def scan(self, username):
regex = r'(\d+) (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*<.*> (.*)'
data = self.run_container(command=['-u', username.name, '--retweets'])
for tweet_id, date, content in self.findall_regex(data, regex):
yield Tweet(id=tweet_id, content=content, date=date, rt=True)
data = self.run_container(command=['-u', username.name])
for tweet_id, date, content in self.findall_regex(data, regex):
yield Tweet(id=tweet_id, content=content, date=date, rt=False)
class DummyDocker(DockerCollector):
config = {
'name': 'dummy-docker-collector',
'docker': {
'build_context': get_actual_dir()
},
}
def callbacks(self):
return {
Person: self.cb_person,
Email: self.cb_email,
}
def cb_person(self, person):
whoami = self.run_container(command='whoami')
yield Person(firstname='dummy docker', lastname=whoami)
yield Email(address='dummy@email')
def cb_email(self, email):
date = self.run_container(command='date')
yield Person(firstname='dummy docker', lastname=date)
yield Email(address='dummy@email')
class Dummy(DockerCollector):
config = {
"name": "dummy-docker-collector",
"limiter": [RequestRate(limit=1, interval=Duration.SECOND)],
"docker": {
"build_context": get_actual_dir()
}
}
def callbacks(self):
return {
Email: self.cb_email,
Username: self.cb_username,
}
def cb_username(self, username):
whoami = self.run_container(command="whoami")
yield Username(name=username.name + whoami)
yield Email(address="dummy@" + whoami)
def cb_email(self, email):
date = self.run_container(command="date")
yield Username(name=email.address + date)
yield Email(address="dummy@" + date)
class Zen(DockerCollector):
config = {
'name': 'zen',
'docker': {
'build_context': get_actual_dir()
},
}
def callbacks(self):
return {
Username: self.from_username,
Company: self.from_company,
}
def from_username(self, username):
data = self.run_container(command=[username.name])
for email in self.findall_regex(data, fr'{username.name} : (.*)'):
yield Email(address=email)
def from_company(self, company):
data = self.run_container(command=[company.name, '--org'])
for username, email in self.findall_regex(data, r'(.*) : (.*)'):
yield Username(name=username, email=email)
yield Email(address=email, username=username)
class DummyDocker(DockerCollector):
config = {
"name": "recon-ng",
"docker": {
"build_context": get_actual_dir()
},
}
def callbacks(self):
return {
Domain: self.from_domain,
Username: self.from_username,
}
def from_domain(self, domain):
data = self.run_container(command=[
"-m",
"recon/domains-hosts/hackertarget",
"-o",
f"SOURCE={domain.fqdn}",
"-x",
], )
logger.debug(data)
for item in self.findall_regex(data, r"Host: (.*)"):
yield Domain(fqdn=item)
def from_username(self, username):
data = self.run_container(
command=["-m", "profiler", "-o", f"SOURCE={username.name}",
"-x"], )
logger.debug(data)
for category, resource, url in self.findall_regex(
data,
r"Category: (.*)\n.*\n.*Resource: (.*)\n.*Url: (.*)",
):
yield OnlineProfile(url=url, category=category, resource=resource)
class Zen(DockerCollector):
config = {
"name": "zen",
"docker": {"build_context": get_actual_dir()},
}
def callbacks(self):
return {
Username: self.from_username,
Company: self.from_company,
}
def from_username(self, username):
data = self.run_container(command=[username.name])
logger.debug(data)
for email in self.findall_regex(data, f"{username.name} : (.*)"):
yield Email(address=email)
def from_company(self, company):
data = self.run_container(command=[company.name, "--org"])
logger.debug(data)
for username, email in self.findall_regex(data, r"(.*) : (.*)"):
yield Username(name=username, email=email)
yield Email(address=email, username=username)
