def set_data(self, data):
"""
Set / convert internal data.
For now it just selects a random set to show.
"""
entries = []
# Grab 5 random banners to display
for banner in random.sample(data, min(len(data), 5)):
desc = '{} -> {} / {}'.format(get_ip(banner), banner['port'],
banner['location']['country_code'])
if banner['location']['city']:
# Not all cities can be encoded in ASCII so ignore any errors
try:
desc += ' {}'.format(banner['location']['city'])
except Exception:
pass
if 'tags' in banner and banner['tags']:
desc += ' / {}'.format(','.join(banner['tags']))
entry = (
float(banner['location']['latitude']),
float(banner['location']['longitude']),
'*',
desc,
curses.A_BOLD,
'red',
)
entries.append(entry)
self.data = entries
def set_data(self, data):
"""
Set / convert internal data.
For now it just selects a random set to show.
"""
entries = []
# Grab 5 random banners to display
for banner in random.sample(data, min(len(data), 5)):
desc = '{} -> {} / {}'.format(get_ip(banner), banner['port'], banner['location']['country_code'])
if banner['location']['city']:
# Not all cities can be encoded in ASCII so ignore any errors
try:
desc += ' {}'.format(banner['location']['city'])
except Exception:
pass
if 'tags' in banner and banner['tags']:
desc += ' / {}'.format(','.join(banner['tags']))
entry = (
float(banner['location']['latitude']),
float(banner['location']['longitude']),
'*',
desc,
curses.A_BOLD,
'red',
)
entries.append(entry)
self.data = entries
def scan_internet(quiet, port, protocol):
"""Scan the Internet for a specific port and protocol using the Shodan infrastructure."""
key = get_api_key()
api = shodan.Shodan(key)
try:
# Submit the request to Shodan
click.echo('Submitting Internet scan to Shodan...', nl=False)
scan = api.scan_internet(port, protocol)
click.echo('Done')
# If the requested port is part of the regular Shodan crawling, then
# we don't know when the scan is done so lets return immediately and
# let the user decide when to stop waiting for further results.
official_ports = api.ports()
if port in official_ports:
click.echo('The requested port is already indexed by Shodan. A new scan for the port has been launched, please subscribe to the real-time stream for results.')
else:
# Create the output file
filename = '{0}-{1}.json.gz'.format(port, protocol)
counter = 0
with helpers.open_file(filename, 'w') as fout:
click.echo('Saving results to file: {0}'.format(filename))
# Start listening for results
done = False
# Keep listening for results until the scan is done
click.echo('Waiting for data, please stand by...')
while not done:
try:
for banner in api.stream.ports([port], timeout=90):
counter += 1
helpers.write_banner(fout, banner)
if not quiet:
click.echo('{0:<40} {1:<20} {2}'.format(
click.style(helpers.get_ip(banner), fg=COLORIZE_FIELDS['ip_str']),
click.style(str(banner['port']), fg=COLORIZE_FIELDS['port']),
';'.join(banner['hostnames']))
)
except shodan.APIError as e:
# We stop waiting for results if the scan has been processed by the crawlers and
# there haven't been new results in a while
if done:
break
scan = api.scan_status(scan['id'])
if scan['status'] == 'DONE':
done = True
except socket.timeout as e:
# We stop waiting for results if the scan has been processed by the crawlers and
# there haven't been new results in a while
if done:
break
scan = api.scan_status(scan['id'])
if scan['status'] == 'DONE':
done = True
except Exception as e:
raise click.ClickException(repr(e))
click.echo('Scan finished: {0} devices found'.format(counter))
except shodan.APIError as e:
raise click.ClickException(e.value)
def scan_submit(wait, filename, force, verbose, netblocks):
"""Scan an IP/ netblock using Shodan."""
key = get_api_key()
api = shodan.Shodan(key)
alert = None
# Submit the IPs for scanning
try:
# Submit the scan
scan = api.scan(netblocks, force=force)
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M')
click.echo('')
click.echo('Starting Shodan scan at {} - {} scan credits left'.format(now, scan['credits_left']))
if verbose:
click.echo('# Scan ID: {}'.format(scan['id']))
# Return immediately
if wait <= 0:
click.echo('Exiting now, not waiting for results. Use the API or website to retrieve the results of the scan.')
else:
# Setup an alert to wait for responses
alert = api.create_alert('Scan: {}'.format(', '.join(netblocks)), netblocks)
# Create the output file if necessary
filename = filename.strip()
fout = None
if filename != '':
# Add the appropriate extension if it's not there atm
if not filename.endswith('.json.gz'):
filename += '.json.gz'
fout = helpers.open_file(filename, 'w')
# Start a spinner
finished_event = threading.Event()
progress_bar_thread = threading.Thread(target=async_spinner, args=(finished_event,))
progress_bar_thread.start()
# Now wait a few seconds for items to get returned
hosts = collections.defaultdict(dict)
done = False
scan_start = time.time()
cache = {}
while not done:
try:
for banner in api.stream.alert(aid=alert['id'], timeout=wait):
ip = banner.get('ip', banner.get('ipv6', None))
if not ip:
continue
# Don't show duplicate banners
cache_key = '{}:{}'.format(ip, banner['port'])
if cache_key not in cache:
hosts[helpers.get_ip(banner)][banner['port']] = banner
cache[cache_key] = True
# If we've grabbed data for more than 60 seconds it might just be a busy network and we should move on
if time.time() - scan_start >= 60:
scan = api.scan_status(scan['id'])
if verbose:
click.echo('# Scan status: {}'.format(scan['status']))
if scan['status'] == 'DONE':
done = True
break
except shodan.APIError as e:
# If the connection timed out before the timeout, that means the streaming server
# that the user tried to reach is down. In that case, lets wait briefly and try
# to connect again!
if (time.time() - scan_start) < wait:
time.sleep(0.5)
continue
# Exit if the scan was flagged as done somehow
if done:
break
scan = api.scan_status(scan['id'])
if scan['status'] == 'DONE':
done = True
if verbose:
click.echo('# Scan status: {}'.format(scan['status']))
except socket.timeout as e:
# If the connection timed out before the timeout, that means the streaming server
# that the user tried to reach is down. In that case, lets wait a second and try
# to connect again!
if (time.time() - scan_start) < wait:
continue
done = True
except Exception as e:
finished_event.set()
progress_bar_thread.join()
raise click.ClickException(repr(e))
finished_event.set()
progress_bar_thread.join()
def print_field(name, value):
click.echo(' {:25s}{}'.format(name, value))
def print_banner(banner):
click.echo(' {:20s}'.format(click.style(str(banner['port']), fg='green') + '/' + banner['transport']), nl=False)
if 'product' in banner:
click.echo(banner['product'], nl=False)
if 'version' in banner:
click.echo(' ({})'.format(banner['version']), nl=False)
click.echo('')
# Show optional ssl info
if 'ssl' in banner:
if 'versions' in banner['ssl']:
# Only print SSL versions if they were successfully tested
versions = [version for version in sorted(banner['ssl']['versions']) if not version.startswith('-')]
if len(versions) > 0:
click.echo(' |-- SSL Versions: {}'.format(', '.join(versions)))
if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']:
click.echo(' |-- Diffie-Hellman Parameters:')
click.echo(' {:15s}{}\n {:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator']))
if 'fingerprint' in banner['ssl']['dhparams']:
click.echo(' {:15s}{}'.format('Fingerprint:', banner['ssl']['dhparams']['fingerprint']))
if hosts:
# Remove the remaining spinner character
click.echo('\b ')
for ip in sorted(hosts):
host = next(iter(hosts[ip].items()))[1]
click.echo(click.style(ip, fg='cyan'), nl=False)
if 'hostnames' in host and host['hostnames']:
click.echo(' ({})'.format(', '.join(host['hostnames'])), nl=False)
click.echo('')
if 'location' in host and 'country_name' in host['location'] and host['location']['country_name']:
print_field('Country', host['location']['country_name'])
if 'city' in host['location'] and host['location']['city']:
print_field('City', host['location']['city'])
if 'org' in host and host['org']:
print_field('Organization', host['org'])
if 'os' in host and host['os']:
print_field('Operating System', host['os'])
click.echo('')
# Output the vulnerabilities the host has
if 'vulns' in host and len(host['vulns']) > 0:
vulns = []
for vuln in host['vulns']:
if vuln.startswith('!'):
continue
if vuln.upper() == 'CVE-2014-0160':
vulns.append(click.style('Heartbleed', fg='red'))
else:
vulns.append(click.style(vuln, fg='red'))
if len(vulns) > 0:
click.echo(' {:25s}'.format('Vulnerabilities:'), nl=False)
for vuln in vulns:
click.echo(vuln + '\t', nl=False)
click.echo('')
# Print all the open ports:
click.echo(' Open Ports:')
for port in sorted(hosts[ip]):
print_banner(hosts[ip][port])
# Save the banner in a file if necessary
if fout:
helpers.write_banner(fout, hosts[ip][port])
click.echo('')
else:
# Prepend a \b to remove the spinner
click.echo('\bNo open ports found or the host has been recently crawled and cant get scanned again so soon.')
except shodan.APIError as e:
raise click.ClickException(e.value)
finally:
# Remove any alert
if alert:
api.delete_alert(alert['id'])
def host_print_pretty(host, history=False):
"""Show the host information in a user-friendly way and try to include
as much relevant information as possible."""
# General info
click.echo(click.style(get_ip(host), fg='green'))
if len(host['hostnames']) > 0:
click.echo(u'{:25s}{}'.format('Hostnames:',
';'.join(host['hostnames'])))
if 'city' in host and host['city']:
click.echo(u'{:25s}{}'.format('City:', host['city']))
if 'country_name' in host and host['country_name']:
click.echo(u'{:25s}{}'.format('Country:', host['country_name']))
if 'os' in host and host['os']:
click.echo(u'{:25s}{}'.format('Operating System:', host['os']))
if 'org' in host and host['org']:
click.echo(u'{:25s}{}'.format('Organization:', host['org']))
if 'last_update' in host and host['last_update']:
click.echo('{:25s}{}'.format('Updated:', host['last_update']))
click.echo('{:25s}{}'.format('Number of open ports:', len(host['ports'])))
# Output the vulnerabilities the host has
if 'vulns' in host and len(host['vulns']) > 0:
vulns = []
for vuln in host['vulns']:
if vuln.startswith('!'):
continue
if vuln.upper() == 'CVE-2014-0160':
vulns.append(click.style('Heartbleed', fg='red'))
else:
vulns.append(click.style(vuln, fg='red'))
if len(vulns) > 0:
click.echo('{:25s}'.format('Vulnerabilities:'), nl=False)
for vuln in vulns:
click.echo(vuln + '\t', nl=False)
click.echo('')
click.echo('')
# If the user doesn't have access to SSL/ Telnet results then we need
# to pad the host['data'] property with empty banners so they still see
# the port listed as open. (#63)
if len(host['ports']) != len(host['data']):
# Find the ports the user can't see the data for
ports = host['ports']
for banner in host['data']:
if banner['port'] in ports:
ports.remove(banner['port'])
# Add the placeholder banners
for port in ports:
banner = {
'port': port,
'transport': 'tcp', # All the filtered services use TCP
'timestamp': host['data'][-1]
['timestamp'], # Use the timestamp of the oldest banner
'placeholder':
True, # Don't store this banner when the file is saved
}
host['data'].append(banner)
click.echo('Ports:')
for banner in sorted(host['data'], key=lambda k: k['port']):
product = ''
version = ''
if 'product' in banner and banner['product']:
product = banner['product']
if 'version' in banner and banner['version']:
version = '({})'.format(banner['version'])
click.echo(click.style('{:>7d}'.format(banner['port']), fg='cyan'),
nl=False)
click.echo('/', nl=False)
click.echo(click.style('{} '.format(banner['transport']), fg='yellow'),
nl=False)
click.echo('{} {}'.format(product, version), nl=False)
if history:
# Format the timestamp to only show the year-month-day
date = banner['timestamp'][:10]
click.echo(click.style('\t\t({})'.format(date),
fg='white',
dim=True),
nl=False)
click.echo('')
# Show optional ssl info
if 'ssl' in banner:
if 'versions' in banner['ssl'] and banner['ssl']['versions']:
click.echo('\t|-- SSL Versions: {}'.format(', '.join([
version for version in sorted(banner['ssl']['versions'])
if not version.startswith('-')
])))
if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']:
click.echo('\t|-- Diffie-Hellman Parameters:')
click.echo('\t\t{:15s}{}\n\t\t{:15s}{}'.format(
'Bits:', banner['ssl']['dhparams']['bits'], 'Generator:',
banner['ssl']['dhparams']['generator']))
if 'fingerprint' in banner['ssl']['dhparams']:
click.echo('\t\t{:15s}{}'.format(
'Fingerprint:',
banner['ssl']['dhparams']['fingerprint']))
'PLEASE_READ_ME_XYZ', 'jacpos', 'jackpos', 'jackposv1', 'jackposv2',
'jackposprivate12', 'alina', 'topkek112', 'README', 'WRITE_ME',
'WE_HAVE_YOUR_DATA', 'your_data_has_been_backed_up', 'REQUEST_YOUR_DATA',
'DB_HAS_BEEN_DROPPED', 'Warning', 'Attention',
'send_bitcoin_to_retrieve_the_data', 'DATA_HAS_BEEN_BACKED_UP',
'REQUEST_ME', 'CONTACTME', 'BACKUP_DB', 'db_has_been_backed_up',
'PLEASE_READ', 'please_read', 'warning', 'DB_H4CK3D', 'CONTACTME',
'PLEASE_READ_ME', 'DB_DELETED', 'DB_DROPPED', 'PLEASEREAD',
'NODATA4U_SECUREYOURSHIT', 'SECUREYOURSHIT', 'pleasereadthis', 'readme',
'PLEASE_SECURE_THIS_INSTALLATION', 'ReadmePlease', 'JUST_READ_ME',
'README_MISSING_DATABASES', 'README_YOU_DB_IS_INSECURE',
'PWNED_SECURE_YOUR_STUFF_SILLY', 'WARNING_ALERT', 'pleaseread'
]
for banner in iterate_files(argv[1:]):
ip = get_ip(banner)
org = banner['org']
try:
product = banner['product']
except:
pass
try:
if product == "MongoDB":
data = banner['data'].replace('MongoDB Server Information\n',
'').split('\n},\n')[2]
data = json.loads(data + '}')
for db in data['databases']:
if db['name'] in pwnedDBs:
print('{}:{}:{}'.format(ip, db['name'], product))
elif product == "Elastic":
data = banner['elastic']
def host_print_pretty(host, history=False):
"""Show the host information in a user-friendly way and try to include
as much relevant information as possible."""
# General info
click.echo(click.style(get_ip(host), fg='green'))
if len(host['hostnames']) > 0:
click.echo(u'{:25s}{}'.format('Hostnames:', ';'.join(host['hostnames'])))
if 'city' in host and host['city']:
click.echo(u'{:25s}{}'.format('City:', host['city']))
if 'country_name' in host and host['country_name']:
click.echo(u'{:25s}{}'.format('Country:', host['country_name']))
if 'os' in host and host['os']:
click.echo(u'{:25s}{}'.format('Operating System:', host['os']))
if 'org' in host and host['org']:
click.echo(u'{:25s}{}'.format('Organization:', host['org']))
if 'last_update' in host and host['last_update']:
click.echo('{:25s}{}'.format('Updated:', host['last_update']))
click.echo('{:25s}{}'.format('Number of open ports:', len(host['ports'])))
# Output the vulnerabilities the host has
if 'vulns' in host and len(host['vulns']) > 0:
vulns = []
for vuln in host['vulns']:
if vuln.startswith('!'):
continue
if vuln.upper() == 'CVE-2014-0160':
vulns.append(click.style('Heartbleed', fg='red'))
else:
vulns.append(click.style(vuln, fg='red'))
if len(vulns) > 0:
click.echo('{:25s}'.format('Vulnerabilities:'), nl=False)
for vuln in vulns:
click.echo(vuln + '\t', nl=False)
click.echo('')
click.echo('')
# If the user doesn't have access to SSL/ Telnet results then we need
# to pad the host['data'] property with empty banners so they still see
# the port listed as open. (#63)
if len(host['ports']) != len(host['data']):
# Find the ports the user can't see the data for
ports = host['ports']
for banner in host['data']:
if banner['port'] in ports:
ports.remove(banner['port'])
# Add the placeholder banners
for port in ports:
banner = {
'port': port,
'transport': 'tcp', # All the filtered services use TCP
'timestamp': host['data'][-1]['timestamp'], # Use the timestamp of the oldest banner
'placeholder': True, # Don't store this banner when the file is saved
}
host['data'].append(banner)
click.echo('Ports:')
for banner in sorted(host['data'], key=lambda k: k['port']):
product = ''
version = ''
if 'product' in banner and banner['product']:
product = banner['product']
if 'version' in banner and banner['version']:
version = '({})'.format(banner['version'])
click.echo(click.style('{:>7d}'.format(banner['port']), fg='cyan'), nl=False)
if 'transport' in banner:
click.echo('/', nl=False)
click.echo(click.style('{} '.format(banner['transport']), fg='yellow'), nl=False)
click.echo('{} {}'.format(product, version), nl=False)
if history:
# Format the timestamp to only show the year-month-day
date = banner['timestamp'][:10]
click.echo(click.style('\t\t({})'.format(date), fg='white', dim=True), nl=False)
click.echo('')
# Show optional ssl info
if 'ssl' in banner:
if 'versions' in banner['ssl'] and banner['ssl']['versions']:
click.echo('\t|-- SSL Versions: {}'.format(', '.join([item for item in sorted(banner['ssl']['versions']) if not version.startswith('-')])))
if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']:
click.echo('\t|-- Diffie-Hellman Parameters:')
click.echo('\t\t{:15s}{}\n\t\t{:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator']))
if 'fingerprint' in banner['ssl']['dhparams']:
click.echo('\t\t{:15s}{}'.format('Fingerprint:', banner['ssl']['dhparams']['fingerprint']))
# Setup the Shodan API object
api = shodan.Shodan(SHODAN_API_KEY)
# Show a progress indicator
BAR_FORMAT = '{desc}{percentage:3.0f}%[{bar}] {n_fmt}/{total_fmt} '
'[{elapsed}<{remaining}, {rate_fmt}{postfix}]'
pbar = tqdm.tqdm(total=res_count, ascii=True, bar_format=BAR_FORMAT)
pbar.set_description(desc='Сollecting snapshots from the JSON')
# Loop over all of the Shodan data files the user provided
for banner in helpers.iterate_files(f'{city.lower()}-data.json.gz'):
# See whether the current banner has a screenshot,
# if it does then lets lookup more information about this IP
has_screenshot = helpers.get_screenshot(banner)
if has_screenshot:
try:
ip = helpers.get_ip(banner)
pbar.write(f'Looking up {ip}')
host = api.host(ip, history=True)
country = host['country_name']
city = host['city']
# Store all the historical screenshots for this IP
screenshots = []
for tmp_banner in host['data']:
# Try to extract the image from the banner data
screenshot = helpers.get_screenshot(tmp_banner)
if screenshot:
# Sort the images by the time they were
# collected so the GIF will loop based on
# the local time regardless of which day the
# banner was taken.
