def host(format, history, filename, save, ip):
"""View all available information for an IP address"""
key = get_api_key()
api = shodan.Shodan(key)
try:
host = api.host(ip, history=history)
# Print the host information to the terminal using the user-specified format
HOST_PRINT[format](host, history=history)
# Store the results
if filename or save:
if save:
filename = '{}.json.gz'.format(ip)
# Add the appropriate extension if it's not there atm
if not filename.endswith('.json.gz'):
filename += '.json.gz'
# Create/ append to the file
fout = helpers.open_file(filename)
for banner in sorted(host['data'], key=lambda k: k['port']):
if 'placeholder' not in banner:
helpers.write_banner(fout, banner)
except shodan.APIError as e:
raise click.ClickException(e.value)
def host(format, history, filename, save, ip):
"""View all available information for an IP address"""
key = get_api_key()
api = shodan.Shodan(key)
try:
host = api.host(ip, history=history)
# Print the host information to the terminal using the user-specified format
HOST_PRINT[format](host, history=history)
# Store the results
if filename or save:
if save:
filename = '{}.json.gz'.format(ip)
# Add the appropriate extension if it's not there atm
if not filename.endswith('.json.gz'):
filename += '.json.gz'
# Create/ append to the file
fout = helpers.open_file(filename)
for banner in sorted(host['data'], key=lambda k: k['port']):
if 'placeholder' not in banner:
helpers.write_banner(fout, banner)
except shodan.APIError as e:
raise click.ClickException(e.value)
def parse(color, fields, filters, filename, separator, filenames):
"""Extract information out of compressed JSON files."""
# Strip out any whitespace in the fields and turn them into an array
fields = [item.strip() for item in fields.split(',')]
if len(fields) == 0:
raise click.ClickException('Please define at least one property to show')
has_filters = len(filters) > 0
# Setup the output file handle
fout = None
if filename:
# If no filters were provided raise an error since it doesn't make much sense w/out them
if not has_filters:
raise click.ClickException('Output file specified without any filters. Need to use filters with this option.')
# Add the appropriate extension if it's not there atm
if not filename.endswith('.json.gz'):
filename += '.json.gz'
fout = helpers.open_file(filename)
for banner in helpers.iterate_files(filenames):
row = u''
# Validate the banner against any provided filters
if has_filters and not match_filters(banner, filters):
continue
# Append the data
if fout:
helpers.write_banner(fout, banner)
# Loop over all the fields and print the banner as a row
for i, field in enumerate(fields):
tmp = u''
value = get_banner_field(banner, field)
if value:
field_type = type(value)
# If the field is an array then merge it together
if field_type == list:
tmp = u';'.join(value)
elif field_type in [int, float]:
tmp = u'{}'.format(value)
else:
tmp = escape_data(value)
# Colorize certain fields if the user wants it
if color:
tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white'))
# Add the field information to the row
if i > 0:
row += separator
row += tmp
click.echo(row)
def download(limit, filename, query):
"""Download search results and save them in a compressed JSON file."""
key = get_api_key()
# Create the query string out of the provided tuple
query = ' '.join(query).strip()
# Make sure the user didn't supply an empty string
if query == '':
raise click.ClickException('Empty search query')
filename = filename.strip()
if filename == '':
raise click.ClickException('Empty filename')
# Add the appropriate extension if it's not there atm
if not filename.endswith('.json.gz'):
filename += '.json.gz'
# Perform the search
api = shodan.Shodan(key)
try:
total = api.count(query)['total']
info = api.info()
except:
raise click.ClickException('The Shodan API is unresponsive at the moment, please try again later.')
# Print some summary information about the download request
click.echo('Search query:\t\t\t%s' % query)
click.echo('Total number of results:\t%s' % total)
click.echo('Query credits left:\t\t%s' % info['unlocked_left'])
click.echo('Output file:\t\t\t%s' % filename)
if limit > total:
limit = total
# A limit of -1 means that we should download all the data
if limit <= 0:
limit = total
with helpers.open_file(filename, 'w') as fout:
count = 0
try:
cursor = api.search_cursor(query, minify=False)
with click.progressbar(cursor, length=limit) as bar:
for banner in bar:
helpers.write_banner(fout, banner)
count += 1
if count >= limit:
break
except Exception:
pass
# Let the user know we're done
if count < limit:
click.echo(click.style('Notice: fewer results were saved than requested', 'yellow'))
click.echo(click.style(u'Saved {} results into file {}'.format(count, filename), 'green'))
def domain_info(domain, details, save, history, type):
"""View all available information for a domain"""
key = get_api_key()
api = shodan.Shodan(key)
try:
info = api.dns.domain_info(domain, history=history, type=type)
except shodan.APIError as e:
raise click.ClickException(e.value)
# Grab the host information for any IP records that were returned
hosts = {}
if details:
ips = [record['value'] for record in info['data'] if record['type'] in ['A', 'AAAA']]
ips = set(ips)
fout = None
if save:
filename = u'{}-hosts.json.gz'.format(domain)
fout = helpers.open_file(filename)
for ip in ips:
try:
hosts[ip] = api.host(ip)
# Store the banners if requested
if fout:
for banner in hosts[ip]['data']:
if 'placeholder' not in banner:
helpers.write_banner(fout, banner)
except shodan.APIError:
pass # Ignore any API lookup errors as this isn't critical information
# Save the DNS data
if save:
filename = u'{}.json.gz'.format(domain)
fout = helpers.open_file(filename)
for record in info['data']:
helpers.write_banner(fout, record)
click.secho(info['domain'].upper(), fg='green')
click.echo('')
for record in info['data']:
click.echo(
u'{:32} {:14} {}'.format(
click.style(record['subdomain'], fg='cyan'),
click.style(record['type'], fg='yellow'),
record['value']
),
nl=False,
)
if record['value'] in hosts:
host = hosts[record['value']]
click.secho(u' Ports: {}'.format(', '.join([str(port) for port in sorted(host['ports'])])), fg='blue', nl=False)
click.echo('')
def download(limit, filename, query):
"""Download search results and save them in a compressed JSON file."""
key = get_api_key()
# Create the query string out of the provided tuple
query = ' '.join(query).strip()
# Make sure the user didn't supply an empty string
if query == '':
raise click.ClickException('Empty search query')
filename = filename.strip()
if filename == '':
raise click.ClickException('Empty filename')
# Add the appropriate extension if it's not there atm
if not filename.endswith('.json.gz'):
filename += '.json.gz'
# Perform the search
api = shodan.Shodan(key)
try:
total = api.count(query)['total']
info = api.info()
except Exception:
raise click.ClickException('The Shodan API is unresponsive at the moment, please try again later.')
# Print some summary information about the download request
click.echo('Search query:\t\t\t%s' % query)
click.echo('Total number of results:\t%s' % total)
click.echo('Query credits left:\t\t%s' % info['unlocked_left'])
click.echo('Output file:\t\t\t%s' % filename)
if limit > total:
limit = total
# A limit of -1 means that we should download all the data
if limit <= 0:
limit = total
with helpers.open_file(filename, 'w') as fout:
count = 0
try:
cursor = api.search_cursor(query, minify=False)
with click.progressbar(cursor, length=limit) as bar:
for banner in bar:
helpers.write_banner(fout, banner)
count += 1
if count >= limit:
break
except Exception:
pass
# Let the user know we're done
if count < limit:
click.echo(click.style('Notice: fewer results were saved than requested', 'yellow'))
click.echo(click.style(u'Saved {} results into file {}'.format(count, filename), 'green'))
def parse(color, fields, filters, filename, separator, filenames):
"""Extract information out of compressed JSON files."""
# Strip out any whitespace in the fields and turn them into an array
fields = [item.strip() for item in fields.split(',')]
if len(fields) == 0:
raise click.ClickException('Please define at least one property to show')
has_filters = len(filters) > 0
# Setup the output file handle
fout = None
if filename:
# If no filters were provided raise an error since it doesn't make much sense w/out them
if not has_filters:
raise click.ClickException('Output file specified without any filters. Need to use filters with this option.')
# Add the appropriate extension if it's not there atm
if not filename.endswith('.json.gz'):
filename += '.json.gz'
fout = helpers.open_file(filename)
for banner in helpers.iterate_files(filenames):
row = u''
# Validate the banner against any provided filters
if has_filters and not match_filters(banner, filters):
continue
# Append the data
if fout:
helpers.write_banner(fout, banner)
# Loop over all the fields and print the banner as a row
for i, field in enumerate(fields):
tmp = u''
value = get_banner_field(banner, field)
if value:
field_type = type(value)
# If the field is an array then merge it together
if field_type == list:
tmp = u';'.join(value)
elif field_type in [int, float]:
tmp = u'{}'.format(value)
else:
tmp = escape_data(value)
# Colorize certain fields if the user wants it
if color:
tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white'))
# Add the field information to the row
if i > 0:
row += separator
row += tmp
click.echo(row)
def scan_internet(quiet, port, protocol):
"""Scan the Internet for a specific port and protocol using the Shodan infrastructure."""
key = get_api_key()
api = shodan.Shodan(key)
try:
# Submit the request to Shodan
click.echo('Submitting Internet scan to Shodan...', nl=False)
scan = api.scan_internet(port, protocol)
click.echo('Done')
# If the requested port is part of the regular Shodan crawling, then
# we don't know when the scan is done so lets return immediately and
# let the user decide when to stop waiting for further results.
official_ports = api.ports()
if port in official_ports:
click.echo('The requested port is already indexed by Shodan. A new scan for the port has been launched, please subscribe to the real-time stream for results.')
else:
# Create the output file
filename = '{0}-{1}.json.gz'.format(port, protocol)
counter = 0
with helpers.open_file(filename, 'w') as fout:
click.echo('Saving results to file: {0}'.format(filename))
# Start listening for results
done = False
# Keep listening for results until the scan is done
click.echo('Waiting for data, please stand by...')
while not done:
try:
for banner in api.stream.ports([port], timeout=90):
counter += 1
helpers.write_banner(fout, banner)
if not quiet:
click.echo('{0:<40} {1:<20} {2}'.format(
click.style(helpers.get_ip(banner), fg=COLORIZE_FIELDS['ip_str']),
click.style(str(banner['port']), fg=COLORIZE_FIELDS['port']),
';'.join(banner['hostnames']))
)
except shodan.APIError as e:
# We stop waiting for results if the scan has been processed by the crawlers and
# there haven't been new results in a while
if done:
break
scan = api.scan_status(scan['id'])
if scan['status'] == 'DONE':
done = True
except socket.timeout as e:
# We stop waiting for results if the scan has been processed by the crawlers and
# there haven't been new results in a while
if done:
break
scan = api.scan_status(scan['id'])
if scan['status'] == 'DONE':
done = True
except Exception as e:
raise click.ClickException(repr(e))
click.echo('Scan finished: {0} devices found'.format(counter))
except shodan.APIError as e:
raise click.ClickException(e.value)
def scan_submit(wait, filename, force, verbose, netblocks):
"""Scan an IP/ netblock using Shodan."""
key = get_api_key()
api = shodan.Shodan(key)
alert = None
# Submit the IPs for scanning
try:
# Submit the scan
scan = api.scan(netblocks, force=force)
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M')
click.echo('')
click.echo('Starting Shodan scan at {} - {} scan credits left'.format(now, scan['credits_left']))
if verbose:
click.echo('# Scan ID: {}'.format(scan['id']))
# Return immediately
if wait <= 0:
click.echo('Exiting now, not waiting for results. Use the API or website to retrieve the results of the scan.')
else:
# Setup an alert to wait for responses
alert = api.create_alert('Scan: {}'.format(', '.join(netblocks)), netblocks)
# Create the output file if necessary
filename = filename.strip()
fout = None
if filename != '':
# Add the appropriate extension if it's not there atm
if not filename.endswith('.json.gz'):
filename += '.json.gz'
fout = helpers.open_file(filename, 'w')
# Start a spinner
finished_event = threading.Event()
progress_bar_thread = threading.Thread(target=async_spinner, args=(finished_event,))
progress_bar_thread.start()
# Now wait a few seconds for items to get returned
hosts = collections.defaultdict(dict)
done = False
scan_start = time.time()
cache = {}
while not done:
try:
for banner in api.stream.alert(aid=alert['id'], timeout=wait):
ip = banner.get('ip', banner.get('ipv6', None))
if not ip:
continue
# Don't show duplicate banners
cache_key = '{}:{}'.format(ip, banner['port'])
if cache_key not in cache:
hosts[helpers.get_ip(banner)][banner['port']] = banner
cache[cache_key] = True
# If we've grabbed data for more than 60 seconds it might just be a busy network and we should move on
if time.time() - scan_start >= 60:
scan = api.scan_status(scan['id'])
if verbose:
click.echo('# Scan status: {}'.format(scan['status']))
if scan['status'] == 'DONE':
done = True
break
except shodan.APIError as e:
# If the connection timed out before the timeout, that means the streaming server
# that the user tried to reach is down. In that case, lets wait briefly and try
# to connect again!
if (time.time() - scan_start) < wait:
time.sleep(0.5)
continue
# Exit if the scan was flagged as done somehow
if done:
break
scan = api.scan_status(scan['id'])
if scan['status'] == 'DONE':
done = True
if verbose:
click.echo('# Scan status: {}'.format(scan['status']))
except socket.timeout as e:
# If the connection timed out before the timeout, that means the streaming server
# that the user tried to reach is down. In that case, lets wait a second and try
# to connect again!
if (time.time() - scan_start) < wait:
continue
done = True
except Exception as e:
finished_event.set()
progress_bar_thread.join()
raise click.ClickException(repr(e))
finished_event.set()
progress_bar_thread.join()
def print_field(name, value):
click.echo(' {:25s}{}'.format(name, value))
def print_banner(banner):
click.echo(' {:20s}'.format(click.style(str(banner['port']), fg='green') + '/' + banner['transport']), nl=False)
if 'product' in banner:
click.echo(banner['product'], nl=False)
if 'version' in banner:
click.echo(' ({})'.format(banner['version']), nl=False)
click.echo('')
# Show optional ssl info
if 'ssl' in banner:
if 'versions' in banner['ssl']:
# Only print SSL versions if they were successfully tested
versions = [version for version in sorted(banner['ssl']['versions']) if not version.startswith('-')]
if len(versions) > 0:
click.echo(' |-- SSL Versions: {}'.format(', '.join(versions)))
if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']:
click.echo(' |-- Diffie-Hellman Parameters:')
click.echo(' {:15s}{}\n {:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator']))
if 'fingerprint' in banner['ssl']['dhparams']:
click.echo(' {:15s}{}'.format('Fingerprint:', banner['ssl']['dhparams']['fingerprint']))
if hosts:
# Remove the remaining spinner character
click.echo('\b ')
for ip in sorted(hosts):
host = next(iter(hosts[ip].items()))[1]
click.echo(click.style(ip, fg='cyan'), nl=False)
if 'hostnames' in host and host['hostnames']:
click.echo(' ({})'.format(', '.join(host['hostnames'])), nl=False)
click.echo('')
if 'location' in host and 'country_name' in host['location'] and host['location']['country_name']:
print_field('Country', host['location']['country_name'])
if 'city' in host['location'] and host['location']['city']:
print_field('City', host['location']['city'])
if 'org' in host and host['org']:
print_field('Organization', host['org'])
if 'os' in host and host['os']:
print_field('Operating System', host['os'])
click.echo('')
# Output the vulnerabilities the host has
if 'vulns' in host and len(host['vulns']) > 0:
vulns = []
for vuln in host['vulns']:
if vuln.startswith('!'):
continue
if vuln.upper() == 'CVE-2014-0160':
vulns.append(click.style('Heartbleed', fg='red'))
else:
vulns.append(click.style(vuln, fg='red'))
if len(vulns) > 0:
click.echo(' {:25s}'.format('Vulnerabilities:'), nl=False)
for vuln in vulns:
click.echo(vuln + '\t', nl=False)
click.echo('')
# Print all the open ports:
click.echo(' Open Ports:')
for port in sorted(hosts[ip]):
print_banner(hosts[ip][port])
# Save the banner in a file if necessary
if fout:
helpers.write_banner(fout, hosts[ip][port])
click.echo('')
else:
# Prepend a \b to remove the spinner
click.echo('\bNo open ports found or the host has been recently crawled and cant get scanned again so soon.')
except shodan.APIError as e:
raise click.ClickException(e.value)
finally:
# Remove any alert
if alert:
api.delete_alert(alert['id'])
def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries, asn, alert, compresslevel):
"""Stream data in real-time."""
# Setup the Shodan API
key = get_api_key()
api = shodan.Shodan(key)
# Temporarily change the baseurl
api.stream.base_url = streamer
# Strip out any whitespace in the fields and turn them into an array
fields = [item.strip() for item in fields.split(',')]
if len(fields) == 0:
raise click.ClickException('Please define at least one property to show')
# The user must choose "ports", "countries", "asn" or nothing - can't select multiple
# filtered streams at once.
stream_type = []
if ports:
stream_type.append('ports')
if countries:
stream_type.append('countries')
if asn:
stream_type.append('asn')
if alert:
stream_type.append('alert')
if len(stream_type) > 1:
raise click.ClickException('Please use --ports, --countries OR --asn. You cant subscribe to multiple filtered streams at once.')
stream_args = None
# Turn the list of ports into integers
if ports:
try:
stream_args = [int(item.strip()) for item in ports.split(',')]
except ValueError:
raise click.ClickException('Invalid list of ports')
if alert:
alert = alert.strip()
if alert.lower() != 'all':
stream_args = alert
if asn:
stream_args = asn.split(',')
if countries:
stream_args = countries.split(',')
# Flatten the list of stream types
# Possible values are:
# - all
# - asn
# - countries
# - ports
if len(stream_type) == 1:
stream_type = stream_type[0]
else:
stream_type = 'all'
# Decide which stream to subscribe to based on whether or not ports were selected
def _create_stream(name, args, timeout):
return {
'all': api.stream.banners(timeout=timeout),
'alert': api.stream.alert(args, timeout=timeout),
'asn': api.stream.asn(args, timeout=timeout),
'countries': api.stream.countries(args, timeout=timeout),
'ports': api.stream.ports(args, timeout=timeout),
}.get(name, 'all')
stream = _create_stream(stream_type, stream_args, timeout=timeout)
counter = 0
quit = False
last_time = timestr()
fout = None
if datadir:
fout = open_streaming_file(datadir, last_time, compresslevel)
while not quit:
try:
for banner in stream:
# Limit the number of results to output
if limit > 0:
counter += 1
if counter > limit:
quit = True
break
# Write the data to the file
if datadir:
cur_time = timestr()
if cur_time != last_time:
last_time = cur_time
fout.close()
fout = open_streaming_file(datadir, last_time)
helpers.write_banner(fout, banner)
# Print the banner information to stdout
if not quiet:
row = u''
# Loop over all the fields and print the banner as a row
for field in fields:
tmp = u''
value = get_banner_field(banner, field)
if value:
field_type = type(value)
# If the field is an array then merge it together
if field_type == list:
tmp = u';'.join(value)
elif field_type in [int, float]:
tmp = u'{}'.format(value)
else:
tmp = escape_data(value)
# Colorize certain fields if the user wants it
if color:
tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white'))
# Add the field information to the row
row += tmp
row += separator
click.echo(row)
except requests.exceptions.Timeout:
raise click.ClickException('Connection timed out')
except KeyboardInterrupt:
quit = True
except shodan.APIError as e:
raise click.ClickException(e.value)
except Exception:
# For other errors lets just wait a bit and try to reconnect again
time.sleep(1)
# Create a new stream object to subscribe to
stream = _create_stream(stream_type, stream_args, timeout=timeout)
def host(format, history, filename, save, ip):
"""View all available information for an IP address"""
key = get_api_key()
api = shodan.Shodan(key)
try:
host = api.host(ip, history=history)
# General info
click.echo(click.style(ip, fg='green'))
if len(host['hostnames']) > 0:
click.echo('{:25s}{}'.format('Hostnames:', ';'.join(host['hostnames'])))
if 'city' in host and host['city']:
click.echo('{:25s}{}'.format('City:', host['city']))
if 'country_name' in host and host['country_name']:
click.echo('{:25s}{}'.format('Country:', host['country_name']))
if 'os' in host and host['os']:
click.echo('{:25s}{}'.format('Operating System:', host['os']))
if 'org' in host and host['org']:
click.echo('{:25s}{}'.format('Organization:', host['org']))
if 'last_update' in host and host['last_update']:
click.echo('{:25s}{}'.format('Updated:', host['last_update']))
click.echo('{:25s}{}'.format('Number of open ports:', len(host['ports'])))
# Output the vulnerabilities the host has
if 'vulns' in host and len(host['vulns']) > 0:
vulns = []
for vuln in host['vulns']:
if vuln.startswith('!'):
continue
if vuln.upper() == 'CVE-2014-0160':
vulns.append(click.style('Heartbleed', fg='red'))
else:
vulns.append(click.style(vuln, fg='red'))
if len(vulns) > 0:
click.echo('{:25s}'.format('Vulnerabilities:'), nl=False)
for vuln in vulns:
click.echo(vuln + '\t', nl=False)
click.echo('')
click.echo('')
# If the user doesn't have access to SSL/ Telnet results then we need
# to pad the host['data'] property with empty banners so they still see
# the port listed as open. (#63)
if len(host['ports']) != len(host['data']):
# Find the ports the user can't see the data for
ports = host['ports']
for banner in host['data']:
if banner['port'] in ports:
ports.remove(banner['port'])
# Add the placeholder banners
for port in ports:
banner = {
'port': port,
'transport': 'tcp', # All the filtered services use TCP
'timestamp': host['data'][-1]['timestamp'], # Use the timestamp of the oldest banner
'placeholder': True, # Don't store this banner when the file is saved
}
host['data'].append(banner)
click.echo('Ports:')
for banner in sorted(host['data'], key=lambda k: k['port']):
product = ''
version = ''
if 'product' in banner and banner['product']:
product = banner['product']
if 'version' in banner and banner['version']:
version = '({})'.format(banner['version'])
click.echo(click.style('{:>7d}'.format(banner['port']), fg='cyan'), nl=False)
click.echo('/', nl=False)
click.echo(click.style('{} '.format(banner['transport']), fg='yellow'), nl=False)
click.echo('{} {}'.format(product, version), nl=False)
if history:
# Format the timestamp to only show the year-month-day
date = banner['timestamp'][:10]
click.echo(click.style('\t\t({})'.format(date), fg='white', dim=True), nl=False)
click.echo('')
# Show optional ssl info
if 'ssl' in banner:
if 'versions' in banner['ssl'] and banner['ssl']['versions']:
click.echo('\t|-- SSL Versions: {}'.format(', '.join([version for version in sorted(banner['ssl']['versions']) if not version.startswith('-')])))
if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']:
click.echo('\t|-- Diffie-Hellman Parameters:')
click.echo('\t\t{:15s}{}\n\t\t{:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator']))
if 'fingerprint' in banner['ssl']['dhparams']:
click.echo('\t\t{:15s}{}'.format('Fingerprint:', banner['ssl']['dhparams']['fingerprint']))
# Store the results
if filename or save:
if save:
filename = '{}.json.gz'.format(ip)
# Add the appropriate extension if it's not there atm
if not filename.endswith('.json.gz'):
filename += '.json.gz'
# Create/ append to the file
fout = helpers.open_file(filename)
for banner in sorted(host['data'], key=lambda k: k['port']):
if 'placeholder' not in banner:
helpers.write_banner(fout, banner)
except shodan.APIError as e:
raise click.ClickException(e.value)
# Use this script to parse ip addresses in a file and filters by tags listed in the array below. It then makes the results readable and digestible.
# This tool should be used in conjunction with the other Shodan scripts in the repo.
# Import the method that helps us parse the data file
from shodan.helpers import iterate_files, open_file, write_banner
# Standard Python libraries
from pprint import pprint
from sys import argv, exit
# Settings
OUTPUT_FILENAME = 'iterate-query.json.gz'
# The user has to provide at least 1 file
if len(argv) == 1:
print('Usage: {} <file1.json.gz> [file2.json.gz] ...'.format(argv[0]))
exit(1)
# Create the output file
with open_file(OUTPUT_FILENAME) as fout:
# Iterate over all of the provided data files
for banner in iterate_files(argv[1:]):
# Is the service listed?
if 'tags' in banner and 'vpn' or 'http' or 'https' or 'ftp' or 'telnet' or 'smtp' or 'ssh' or 'mysql' or 'mssql' or 'snmp' in banner['tags']:
# Show the banner
pprint(banner)
# Store it in the output file
write_banner(fout, banner)
# Whether or not to look up historical information for the IPs
use_history = False
if len(argv) == 4:
use_history = True
# Must have initialized the CLI before running this script
key = get_api_key()
# Create the API connection
api = Shodan(key)
# Create the output file
fout = open_file(output_filename, 'w')
# Open the file containing the list of IPs
with open(input_filename, 'r') as fin:
# Loop over all the IPs in the file
for line in fin:
ip = line.strip() # Remove any trailing whitespace/ newlines
# Wrap the API calls to nicely skip IPs which don't have data
try:
print('Processing: {}'.format(ip))
info = api.host(ip, history=use_history)
# All the banners are stored in the "data" property
for banner in info['data']:
write_banner(fout, banner)
except:
pass # No data
def host(format, history, filename, save, ip):
"""View all available information for an IP address"""
key = get_api_key()
api = shodan.Shodan(key)
try:
host = api.host(ip, history=history)
# General info
click.echo(click.style(ip, fg='green'))
if len(host['hostnames']) > 0:
click.echo('{:25s}{}'.format('Hostnames:',
';'.join(host['hostnames'])))
if 'city' in host and host['city']:
click.echo('{:25s}{}'.format('City:', host['city']))
if 'country_name' in host and host['country_name']:
click.echo('{:25s}{}'.format('Country:', host['country_name']))
if 'os' in host and host['os']:
click.echo('{:25s}{}'.format('Operating System:', host['os']))
if 'org' in host and host['org']:
click.echo('{:25s}{}'.format('Organization:', host['org']))
click.echo('{:25s}{}'.format('Number of open ports:',
len(host['ports'])))
# Output the vulnerabilities the host has
if 'vulns' in host and len(host['vulns']) > 0:
vulns = []
for vuln in host['vulns']:
if vuln.startswith('!'):
continue
if vuln.upper() == 'CVE-2014-0160':
vulns.append(click.style('Heartbleed', fg='red'))
else:
vulns.append(click.style(vuln, fg='red'))
if len(vulns) > 0:
click.echo('{:25s}'.format('Vulnerabilities:'), nl=False)
for vuln in vulns:
click.echo(vuln + '\t', nl=False)
click.echo('')
click.echo('')
click.echo('Ports:')
for banner in sorted(host['data'], key=lambda k: k['port']):
product = ''
version = ''
if 'product' in banner and banner['product']:
product = banner['product']
if 'version' in banner and banner['version']:
version = '({})'.format(banner['version'])
click.echo(click.style('{:>7d} '.format(banner['port']),
fg='cyan'),
nl=False)
click.echo('{} {}'.format(product, version), nl=False)
if history:
# Format the timestamp to only show the year-month-day
date = banner['timestamp'][:10]
click.echo(click.style('\t\t({})'.format(date),
fg='white',
dim=True),
nl=False)
click.echo('')
# Show optional ssl info
if 'ssl' in banner:
if 'versions' in banner['ssl'] and banner['ssl']['versions']:
click.echo('\t|-- SSL Versions: {}'.format(', '.join([
version
for version in sorted(banner['ssl']['versions'])
if not version.startswith('-')
])))
if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']:
click.echo('\t|-- Diffie-Hellman Parameters:')
click.echo('\t\t{:15s}{}\n\t\t{:15s}{}'.format(
'Bits:', banner['ssl']['dhparams']['bits'],
'Generator:', banner['ssl']['dhparams']['generator']))
if 'fingerprint' in banner['ssl']['dhparams']:
click.echo('\t\t{:15s}{}'.format(
'Fingerprint:',
banner['ssl']['dhparams']['fingerprint']))
# Store the results
if filename or save:
if save:
filename = '{}.json.gz'.format(ip)
# Add the appropriate extension if it's not there atm
if not filename.endswith('.json.gz'):
filename += '.json.gz'
# Create/ append to the file
fout = helpers.open_file(filename)
for banner in sorted(host['data'], key=lambda k: k['port']):
helpers.write_banner(fout, banner)
except shodan.APIError as e:
raise click.ClickException(e.value)
def alert_download(filename, alert_id):
"""Download all information for monitored networks/ IPs."""
key = get_api_key()
api = shodan.Shodan(key)
ips = set()
networks = set()
# Helper method to process batches of IPs
def batch(iterable, size=1):
iter_length = len(iterable)
for ndx in range(0, iter_length, size):
yield iterable[ndx:min(ndx + size, iter_length)]
try:
# Get the list of alerts for the user
click.echo('Looking up alert information...')
if alert_id:
alerts = [api.alerts(aid=alert_id.strip())]
else:
alerts = api.alerts()
click.echo('Compiling list of networks/ IPs to download...')
for alert in alerts:
for net in alert['filters']['ip']:
if '/' in net:
networks.add(net)
else:
ips.add(net)
click.echo('Downloading...')
with open_file(filename) as fout:
# Check if the user is able to use batch IP lookups
batch_size = 1
if len(ips) > 0:
api_info = api.info()
if api_info['plan'] in ['corp', 'stream-100']:
batch_size = 100
# Convert it to a list so we can index into it
ips = list(ips)
# Grab all the IP information
for ip in batch(ips, size=batch_size):
try:
click.echo(ip)
results = api.host(ip)
if not isinstance(results, list):
results = [results]
for host in results:
for banner in host['data']:
write_banner(fout, banner)
except APIError:
pass
sleep(
1
) # Slow down a bit to make sure we don't hit the rate limit
# Grab all the network ranges
for net in networks:
try:
counter = 0
click.echo(net)
for banner in api.search_cursor('net:{}'.format(net)):
write_banner(fout, banner)
# Slow down a bit to make sure we don't hit the rate limit
if counter % 100 == 0:
sleep(1)
counter += 1
except APIError:
pass
except shodan.APIError as e:
raise click.ClickException(e.value)
click.secho('Successfully downloaded results into: {}'.format(filename),
fg='green')
def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries, asn, alert, compresslevel):
"""Stream data in real-time."""
# Setup the Shodan API
key = get_api_key()
api = shodan.Shodan(key)
# Temporarily change the baseurl
api.stream.base_url = streamer
# Strip out any whitespace in the fields and turn them into an array
fields = [item.strip() for item in fields.split(',')]
if len(fields) == 0:
raise click.ClickException('Please define at least one property to show')
# The user must choose "ports", "countries", "asn" or nothing - can't select multiple
# filtered streams at once.
stream_type = []
if ports:
stream_type.append('ports')
if countries:
stream_type.append('countries')
if asn:
stream_type.append('asn')
if alert:
stream_type.append('alert')
if len(stream_type) > 1:
raise click.ClickException('Please use --ports, --countries OR --asn. You cant subscribe to multiple filtered streams at once.')
stream_args = None
# Turn the list of ports into integers
if ports:
try:
stream_args = [int(item.strip()) for item in ports.split(',')]
except ValueError:
raise click.ClickException('Invalid list of ports')
if alert:
alert = alert.strip()
if alert.lower() != 'all':
stream_args = alert
if asn:
stream_args = asn.split(',')
if countries:
stream_args = countries.split(',')
# Flatten the list of stream types
# Possible values are:
# - all
# - asn
# - countries
# - ports
if len(stream_type) == 1:
stream_type = stream_type[0]
else:
stream_type = 'all'
# Decide which stream to subscribe to based on whether or not ports were selected
def _create_stream(name, args, timeout):
return {
'all': api.stream.banners(timeout=timeout),
'alert': api.stream.alert(args, timeout=timeout),
'asn': api.stream.asn(args, timeout=timeout),
'countries': api.stream.countries(args, timeout=timeout),
'ports': api.stream.ports(args, timeout=timeout),
}.get(name, 'all')
stream = _create_stream(stream_type, stream_args, timeout=timeout)
counter = 0
quit = False
last_time = timestr()
fout = None
if datadir:
fout = open_streaming_file(datadir, last_time, compresslevel)
while not quit:
try:
for banner in stream:
# Limit the number of results to output
if limit > 0:
counter += 1
if counter > limit:
quit = True
break
# Write the data to the file
if datadir:
cur_time = timestr()
if cur_time != last_time:
last_time = cur_time
fout.close()
fout = open_streaming_file(datadir, last_time)
helpers.write_banner(fout, banner)
# Print the banner information to stdout
if not quiet:
row = u''
# Loop over all the fields and print the banner as a row
for field in fields:
tmp = u''
value = get_banner_field(banner, field)
if value:
field_type = type(value)
# If the field is an array then merge it together
if field_type == list:
tmp = u';'.join(value)
elif field_type in [int, float]:
tmp = u'{}'.format(value)
else:
tmp = escape_data(value)
# Colorize certain fields if the user wants it
if color:
tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white'))
# Add the field information to the row
row += tmp
row += separator
click.echo(row)
except requests.exceptions.Timeout:
raise click.ClickException('Connection timed out')
except KeyboardInterrupt:
quit = True
except shodan.APIError as e:
raise click.ClickException(e.value)
except Exception:
# For other errors lets just wait a bit and try to reconnect again
time.sleep(1)
# Create a new stream object to subscribe to
stream = _create_stream(stream_type, stream_args, timeout=timeout)
