def collect_api(self, query: str, output_directory: str = None) -> None:
"""
This method collects information from the Censys IPv4 API
:param query: The query in Censys syntax to collect the desired information from Censys
:param output_directory: The directory where Censys results are stored
:return:
"""
query_results = []
if not output_directory or not os.path.isdir(output_directory):
raise NotADirectoryError(
"output directory '{}' does not exist".format(
output_directory))
print("[*] querying censys API")
api = censys.ipv4.CensysIPv4(api_id=self._api_uid,
api_secret=self._api_key)
try:
for overview in api.search(query, fields=["ip"]):
details = api.view(overview["ip"])
query_results.append(details)
except Exception as ex:
print(
" [E] The following error occurred while searching the censys API: {}"
.format(ex))
BaseUtils.add_json_results(self._command, query_results)
self.write_filesystem(query_results=query_results,
item=query,
output_directory=output_directory)
def collect_api(self, domain: str, output_directory: str = None) -> None:
"""
This method collects information from the Censys Certificate API
:param domain: The query in Censys syntax to collect the desired information from Censys
:param output_directory: The directory where Censys results are stored
:return:
"""
query_results = []
print("[*] querying censys API")
api = censys.certificates.CensysCertificates(api_id=self._api_uid,
api_secret=self._api_key)
try:
for overview in api.search("parsed.names.raw:{}".format(domain),
fields=["parsed.fingerprint_sha256"]):
details = api.view(overview["parsed.fingerprint_sha256"])
query_results.append(
{overview["parsed.fingerprint_sha256"]: details})
except Exception as ex:
print(
"[E] the following error occurred while searching the censys API: {}"
.format(ex))
BaseUtils.add_json_results(self._command, query_results)
self.write_filesystem(query_results=query_results,
item=domain,
output_directory=output_directory)
def collect_api(self, domain: str, output_directory: str = None) -> None:
"""
This method collects information from the host.io API
:param domain: The domain to collect information for
:param output_directory: The directory where the results are stored
:return:
"""
if not output_directory or not os.path.isdir(output_directory):
raise NotADirectoryError(
"output directory '{}' does not exist".format(
output_directory))
print("[*] querying certspotter.com API")
url = self._api_url if self._api_url[-1] != "/" else self._api_url[:-1]
response = self._get_request_info(api_url=url,
params={
"include_subdomains": True,
"expand": "cert",
"domain": domain
})
if response.status_code == 200:
query_results = json.loads(response.content)
BaseUtils.add_json_results(self._command, query_results)
self.write_filesystem(query_results=query_results,
item=domain,
output_directory=output_directory)
else:
raise ApiCollectionFailed("failed with status code: {}".format(
response.status_code))
def collect_api(self, item: str, output_directory: str = None) -> None:
"""
This method collects information from the host.io API
:param item: The item to collect information for
:param output_directory: The directory where the results are stored
:return:
"""
if not output_directory or not os.path.isdir(output_directory):
raise NotADirectoryError(
"output directory '{}' does not exist".format(
output_directory))
print("[*] querying crt.sh")
response = self._get_request_info(api_url="https://crt.sh/",
params={
"O": item,
"exclude": "expired"
})
if response.status_code == 200:
BaseUtils.add_binary_result(self._command, response.content)
self.write_filesystem(query_results=response.content,
item=item,
output_directory=output_directory)
self._parse_table(response.content)
else:
raise ApiCollectionFailed("failed with status code: {}".format(
response.status_code))
def collect_api(self, host: str, output_directory: str = None) -> None:
"""
This method collects information from the Shodan API
:param host: The IP address for which information shall be obtained from Shodan
:param output_directory: The directory where Shodan results are stored
:return:
"""
if not output_directory or not os.path.isdir(output_directory):
raise NotADirectoryError(
"output directory '{}' does not exist".format(
output_directory))
print("[*] querying shodan API")
api = shodan.Shodan(self._api_key)
query_results = api.host(host)
BaseUtils.add_json_results(self._command, query_results)
self.write_filesystem(query_results=query_results,
item=host,
output_directory=output_directory)
def collect_api(self, query: str, output_directory: str = None) -> None:
"""
This method collects information from the Shodan API
:param query: The query in Shodan syntax to collect the desired information from Shodan
:param output_directory: The directory where Shodan results are stored
:return:
"""
if not output_directory or not os.path.isdir(output_directory):
raise NotADirectoryError(
"output directory '{}' does not exist".format(
output_directory))
print("[*] querying shodan API")
api = shodan.Shodan(self._api_key)
query_results = api.search(query)
BaseUtils.add_json_results(self._command, query_results)
self.write_filesystem(query_results=query_results,
item=query,
output_directory=output_directory)
def collect_api(self, domain: str, output_directory: str = None) -> None:
"""
This method collects information from the host.io API
:param domain: The domain to collect information for
:param output_directory: The directory where the results are stored
:return:
"""
if not output_directory or not os.path.isdir(output_directory):
raise NotADirectoryError(
"output directory '{}' does not exist".format(
output_directory))
print("[*] querying virustotal.com API")
url = self._api_domain_url if self._api_domain_url[
-1] != "/" else self._api_domain_url[:-1]
response = self._get_request_info(api_url=url,
params={
"apikey": self._api_key,
"domain": domain
})
if response.status_code == 200:
query_results = json.loads(response.content)
BaseUtils.add_json_results(self._command, query_results)
self.write_filesystem(query_results=query_results,
item=domain,
output_directory=output_directory)
elif response.status_code == 204:
raise ApiCollectionFailed(
"too many requests ({}) - the rate limit has "
"been exceeded".format(response.status_code))
elif response.status_code == 400:
raise ApiCollectionFailed(
"bad request ({}) - the request was somehow "
"incorrect".format(response.status_code))
elif response.status_code == 403:
raise ApiCollectionFailed(
"forbidden ({}) - not enough privileges to make "
"the request".format(response.status_code))
elif response.status_code == 404:
raise ApiCollectionFailed(
"not found ({}) - the resource could not be found".format(
response.status_code))
else:
raise ApiCollectionFailed("failed with status code: {}".format(
response.status_code))
def collect_api(self, domain: str, output_directory: str = None) -> None:
"""This method collects information from the API"""
if not output_directory or not os.path.isdir(output_directory):
raise NotADirectoryError(
"output directory '{}' does not exist".format(
output_directory))
print("[*] querying securitytrails API")
response = self._get_request_info(
api_url=self._api_url,
url_extension="{}/subdomains".format(domain))
if response.status_code == 200:
query_results = json.loads(response.content)
BaseUtils.add_json_results(self._command, query_results)
self.write_filesystem(query_results=query_results,
item=domain,
output_directory=output_directory)
else:
raise ApiCollectionFailed("failed with status code: {}".format(
response.status_code))
def collect_api(self, company: str, output_directory: str = None) -> None:
"""
This method collects information from the host.io API
:param company: The company to collect information for
:param output_directory: The directory where the results are stored
:return:
"""
if not output_directory or not os.path.isdir(output_directory):
raise NotADirectoryError("output directory '{}' does not exist".format(output_directory))
print("[*] querying viewdns.info API")
response = self._get_request_info(api_url="https://viewdns.info/reversewhois/",
params={"q": company})
if response.status_code == 200:
BaseUtils.add_binary_result(self._command, response.content)
self.write_filesystem(query_results=response.content,
item=company,
output_directory=output_directory)
else:
raise ApiCollectionFailed("failed with status code: {}".format(response.status_code))
def collect_api(self,
email: str,
output_directory: str = None,
offset: int = 0,
limit: int = 1000) -> None:
"""This method collects information from the API"""
if not output_directory or not os.path.isdir(output_directory):
raise NotADirectoryError(
"output directory '{}' does not exist".format(
output_directory))
if not offset:
print("[*] querying haveibeenpwned.com API")
response = self._get_request_info(api_url=self._api_url,
url_extension=quote(email),
params={"truncateResponse": "true"})
if response.status_code == 200:
query_results = json.loads(response.content)
BaseUtils.add_json_results(self._command, query_results)
self.write_filesystem(query_results=query_results,
item=email,
output_directory=output_directory)
elif response.status_code == 400:
raise ApiCollectionFailed(
"Bad request ({}) - the account does not comply with an acceptable format "
"(i.e. it's an empty string)".format(response.status_code))
elif response.status_code == 403:
raise ApiCollectionFailed(
"Forbidden ({}) - no user agent has been specified "
"in the request".format(response.status_code))
elif response.status_code == 404:
raise ApiCollectionFailed(
"Not found ({}) - the account could not be found and has therefore "
"not been pwned".format(response.status_code))
elif response.status_code == 429:
raise ApiCollectionFailed(
"Too many requests ({}) - the rate limit has been "
"exceeded".format(response.status_code))
else:
raise ApiCollectionFailed("failed with status code: {}".format(
response.status_code))
def collect_api(self, domain: str, output_directory: str=None) -> None:
"""
This method collects information from the builtwith.com API
:param domain: The domain to collect information for
:param output_directory: The directory where the results are stored
:return:
"""
if not output_directory or not os.path.isdir(output_directory):
raise NotADirectoryError("output directory '{}' does not exist".format(output_directory))
print("[*] querying builtwith.com API")
response = self._get_request_info(api_url=self._api_url,
params={"KEY": self._api_key,
"api_key": self._api_key,
"LOOKUP": domain})
if response.status_code == 200:
query_results = json.loads(response.content)
BaseUtils.add_json_results(self._command, query_results)
self.write_filesystem(query_results=query_results,
item=domain,
output_directory=output_directory)
else:
raise ApiCollectionFailed("failed with status code: {}".format(response.status_code))
def collect_api(self,
domain: str,
output_directory: str = None,
offset: int = 0,
limit: int = 1000) -> None:
"""This method collects information from the API"""
if not output_directory or not os.path.isdir(output_directory):
raise NotADirectoryError(
"output directory '{}' does not exist".format(
output_directory))
if not offset:
print("[*] querying hunger.io API")
response = self._get_request_info(api_url=self._api_url,
params={
"domain": domain,
"api_key": self._api_key,
"limit": limit,
"offset": (offset * limit)
})
if response.status_code == 200:
query_results = json.loads(response.content)
BaseUtils.add_json_results(self._command, query_results)
self.write_filesystem(query_results=query_results,
item=domain,
number=offset,
output_directory=output_directory)
results = query_results["meta"]["results"]
offset += 1
if (offset * limit) <= results:
self.collect_api(domain=domain,
output_directory=output_directory,
offset=offset,
limit=limit)
else:
raise ApiCollectionFailed("failed with status code: {}".format(
response.status_code))
def get_csv(self) -> List[List[str]]:
"""
This method returns all information as CSV.
:return:
"""
rows = [[
"Workspace", "Network (NW)", "Scope (NW)", "Company (NW)",
"IP Address (IP)", "IP Summary", "In Scope (IP)",
"Host Names (HN)", "Protocol", "Port", "Service",
"Nmap Service Name", "Nessus Service Name", "Sources", "CVE",
"CVSSv3", "CVSSv2", "Plugin ID", "Description"
]]
additional_info = self._session.query(AdditionalInfo)\
.join(Service)\
.join(Host)\
.join(Workspace)\
.filter(Workspace.name.in_([item.name for item in self._workspaces]),
AdditionalInfo.name == "CVEs").all()
for item in additional_info:
if self._filter(item):
network = None
companies = None
network_scope = None
host = item.service.host
host_names = host.get_host_host_name_mappings_str([
DnsResourceRecordType.a, DnsResourceRecordType.aaaa,
DnsResourceRecordType.ptr
])
if host.ipv4_network:
network = host.ipv4_network.network
companies = host.ipv4_network.companies_str
network_scope = host.ipv4_network.scope_str
for entry in BaseUtils.get_csv_as_list(item.values):
tmp = [
host.workspace.name, network, network_scope, companies,
host.address, host.summary, host.in_scope, host_names,
item.service.protocol_str, item.service.port,
item.service.protocol_port_str,
item.service.nmap_service_name,
item.service.nessus_service_name, item.sources_str
]
tmp.extend(entry)
rows.append(tmp)
return rows