def preloadBannerTimestamps(input_file):
bannerTSDict = {}
if not input_file.endswith('.json.gz'):
input_file += '.json.gz'
for result in helpers.iterate_files(input_file):
bannerTSDict[result['ip_str']] = result['timestamp']
return bannerTSDict
def preloadBannerTimestamps(input_file):
bannerTSDict = {}
if not input_file.endswith('.json.gz'):
input_file += '.json.gz'
for result in helpers.iterate_files(input_file):
bannerTSDict[result['ip_str']] = result['timestamp']
return bannerTSDict
def parse(color, fields, filters, filename, separator, filenames):
"""Extract information out of compressed JSON files."""
# Strip out any whitespace in the fields and turn them into an array
fields = [item.strip() for item in fields.split(',')]
if len(fields) == 0:
raise click.ClickException('Please define at least one property to show')
has_filters = len(filters) > 0
# Setup the output file handle
fout = None
if filename:
# If no filters were provided raise an error since it doesn't make much sense w/out them
if not has_filters:
raise click.ClickException('Output file specified without any filters. Need to use filters with this option.')
# Add the appropriate extension if it's not there atm
if not filename.endswith('.json.gz'):
filename += '.json.gz'
fout = helpers.open_file(filename)
for banner in helpers.iterate_files(filenames):
row = u''
# Validate the banner against any provided filters
if has_filters and not match_filters(banner, filters):
continue
# Append the data
if fout:
helpers.write_banner(fout, banner)
# Loop over all the fields and print the banner as a row
for i, field in enumerate(fields):
tmp = u''
value = get_banner_field(banner, field)
if value:
field_type = type(value)
# If the field is an array then merge it together
if field_type == list:
tmp = u';'.join(value)
elif field_type in [int, float]:
tmp = u'{}'.format(value)
else:
tmp = escape_data(value)
# Colorize certain fields if the user wants it
if color:
tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white'))
# Add the field information to the row
if i > 0:
row += separator
row += tmp
click.echo(row)
def parse(color, fields, filters, filename, separator, filenames):
"""Extract information out of compressed JSON files."""
# Strip out any whitespace in the fields and turn them into an array
fields = [item.strip() for item in fields.split(',')]
if len(fields) == 0:
raise click.ClickException('Please define at least one property to show')
has_filters = len(filters) > 0
# Setup the output file handle
fout = None
if filename:
# If no filters were provided raise an error since it doesn't make much sense w/out them
if not has_filters:
raise click.ClickException('Output file specified without any filters. Need to use filters with this option.')
# Add the appropriate extension if it's not there atm
if not filename.endswith('.json.gz'):
filename += '.json.gz'
fout = helpers.open_file(filename)
for banner in helpers.iterate_files(filenames):
row = u''
# Validate the banner against any provided filters
if has_filters and not match_filters(banner, filters):
continue
# Append the data
if fout:
helpers.write_banner(fout, banner)
# Loop over all the fields and print the banner as a row
for i, field in enumerate(fields):
tmp = u''
value = get_banner_field(banner, field)
if value:
field_type = type(value)
# If the field is an array then merge it together
if field_type == list:
tmp = u';'.join(value)
elif field_type in [int, float]:
tmp = u'{}'.format(value)
else:
tmp = escape_data(value)
# Colorize certain fields if the user wants it
if color:
tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white'))
# Add the field information to the row
if i > 0:
row += separator
row += tmp
click.echo(row)
def main(args):
# Basic input validation
if len(args) <= 1:
print('Usage: {} <data.json.gz> [list of data files...]'.args[0])
return 1
# Keep track of the frequency of each key
tracker = defaultdict(int)
# Loop over all the banners in the provided Shodan data files
for banner in iterate_files(args[1:]):
# Find "Keys" section of the banner text
start = banner['data'].find('# Keys\r\n')
if start < 0:
continue
start += len('# Keys\r\n')
# Find the start of the next section
end = banner['data'].find('\r\n\r\n', start)
# Grab the text in between
keys = banner['data'][start:end]
# Parse the keys line by line
keys = keys.split('\r\n')
for key in keys:
# Skip comments or empty keys
if key.startswith('#') or key.strip() == '':
continue
# Increment the frequency counter for this key
tracker[key] += 1
# Lets print the results
print('Top {} Redis Keys\n'.format(TOP_RESULTS))
# Turn the defaultdict into a list of items (iteritems())
# Sort it in reverse (i.e. most common value will be at the start of the list)
# And sort it based on the value instead of the key name (lambda ...)
sorted_items = sorted(tracker.iteritems(),
reverse=True,
key=lambda (k, v): v)
# Use the enumerate() method to give us a counter and start the counter at 1
for i, item in enumerate(sorted_items, start=1):
# The key names are padded to a length of 30 characters so the output aligns better
print('#{}\t{:30s}\t{}'.format(i, item[0], item[1]))
# Stop looping over results once we've printed the top X values
if i >= TOP_RESULTS:
break
def populateMetadata(image):
print 'In populateMetadata function'
for result in helpers.iterate_files('screenshots.json.gz'):
if result['ip_str'] == image:
print 'Found matching result!!!!!*****#########'
global ipAddress
if result['ip_str'] is not None:
ipAddress = result['ip_str'] + '<br/>'
else:
ipAddress = ''
global country
if result['location']['country_name'] is not None:
country = result['location']['country_name']
else:
country = ''
global city
if result['location']['city'] is not None:
city = ', ' + result['location']['city']
else:
city = ''
global hostnames
if result['hostnames'] is not None:
hostnames = str(result['hostnames']).replace("[", "").replace(
"]", "").replace("u", "") + '<br/>'
else:
hostnames = ''
global isp
if result['isp'] is not None:
isp = result['isp'] + '<br/>'
else:
isp = ''
global timestamp
if result['timestamp'] is not None:
timestamp = result['timestamp'].replace("T", " ") + '<br/>'
else:
timestamp = ''
def populateMetadata(image):
print 'In populateMetadata function'
for result in helpers.iterate_files('screenshots.json.gz'):
if result['ip_str'] == image:
print 'Found matching result!!!!!*****#########'
global ipAddress
if result['ip_str'] is not None:
ipAddress = result['ip_str'] + '<br/>'
else:
ipAddress = ''
global country
if result['location']['country_name'] is not None:
country = result['location']['country_name']
else:
country = ''
global city
if result['location']['city'] is not None:
city = ', ' + result['location']['city']
else:
city = ''
global hostnames
if result['hostnames'] is not None:
hostnames = str(result['hostnames']).replace("[", "").replace("]", "").replace("u", "") + '<br/>'
else:
hostnames = ''
global isp
if result['isp'] is not None:
isp = result['isp'] + '<br/>'
else:
isp = ''
global timestamp
if result['timestamp'] is not None:
timestamp = result['timestamp'].replace("T", " ") + '<br/>'
else:
timestamp = ''
# Use this script to parse ip addresses in a file and filters by tags listed in the array below. It then makes the results readable and digestible.
# This tool should be used in conjunction with the other Shodan scripts in the repo.
# Import the method that helps us parse the data file
from shodan.helpers import iterate_files, open_file, write_banner
# Standard Python libraries
from pprint import pprint
from sys import argv, exit
# Settings
OUTPUT_FILENAME = 'iterate-query.json.gz'
# The user has to provide at least 1 file
if len(argv) == 1:
print('Usage: {} <file1.json.gz> [file2.json.gz] ...'.format(argv[0]))
exit(1)
# Create the output file
with open_file(OUTPUT_FILENAME) as fout:
# Iterate over all of the provided data files
for banner in iterate_files(argv[1:]):
# Is the service listed?
if 'tags' in banner and 'vpn' or 'http' or 'https' or 'ftp' or 'telnet' or 'smtp' or 'ssh' or 'mysql' or 'mssql' or 'snmp' in banner['tags']:
# Show the banner
pprint(banner)
# Store it in the output file
write_banner(fout, banner)
# Settings
API_KEY = ''
MIN_SCREENS = 5 # Number of screenshots that Shodan needs to have in order to make a GIF
MAX_SCREENS = 24
if len(sys.argv) != 2:
print('Usage: {} <shodan-data.json.gz>'.format(sys.argv[0]))
sys.exit(1)
# GIFs are stored in the local "data" directory
os.mkdir('data')
# We need to connect to the API to lookup the historical host information
api = shodan.Shodan(API_KEY)
for result in helpers.iterate_files(sys.argv[1]):
# Get the historic info
host = api.host(result['ip_str'], history=True)
# Count how many screenshots this host has
screenshots = []
for banner in host['data']:
if 'opts' in banner and 'screenshot' in banner['opts']:
timestamp = arrow.get(banner['timestamp']).time()
sort_key = timestamp.hour
screenshots.append(
(sort_key, banner['opts']['screenshot']['data']))
if len(screenshots) >= MAX_SCREENS:
break
import shodan
import shodan.helpers as helpers
import os
import sys
# Usage
if len(sys.argv) != 3:
print('Usage: {} <Shodan json.gz> <output directory>'.format(sys.argv[0]))
sys.exit(-1)
input_file = sys.argv[1]
output_dir = sys.argv[2]
# Make sure the directory exists
if not os.path.exists(output_dir):
os.mkdir(output_dir)
for banner in helpers.iterate_files(input_file):
# Try to grab the screenshot from the banner
screenshot = helpers.get_screenshot(banner)
# If we found a screenshot then create a file w/ the data
if screenshot:
# Create the file handle
image = open('{}/{}.jpg'.format(output_dir, banner['ip_str']), 'w')
# Write the image data which is stored using base64 encoding
image.write(screenshot['data'].decode('base64'))
# GIFs are stored in the local "snapshots" directory
try:
os.makedirs('snapshots/tmp')
except OSError:
pass
# Setup the Shodan API object
api = shodan.Shodan(SHODAN_API_KEY)
# Show a progress indicator
BAR_FORMAT = '{desc}{percentage:3.0f}%[{bar}] {n_fmt}/{total_fmt} '
'[{elapsed}<{remaining}, {rate_fmt}{postfix}]'
pbar = tqdm.tqdm(total=res_count, ascii=True, bar_format=BAR_FORMAT)
pbar.set_description(desc='Сollecting snapshots from the JSON')
# Loop over all of the Shodan data files the user provided
for banner in helpers.iterate_files(f'{city.lower()}-data.json.gz'):
# See whether the current banner has a screenshot,
# if it does then lets lookup more information about this IP
has_screenshot = helpers.get_screenshot(banner)
if has_screenshot:
try:
ip = helpers.get_ip(banner)
pbar.write(f'Looking up {ip}')
host = api.host(ip, history=True)
country = host['country_name']
city = host['city']
# Store all the historical screenshots for this IP
screenshots = []
for tmp_banner in host['data']:
# Try to extract the image from the banner data
API_KEY = ''
MIN_SCREENS = 5 # Number of screenshots that Shodan needs to have in order to make a GIF
MAX_SCREENS = 24
if len(sys.argv) != 2:
print('Usage: {} <shodan-data.json.gz>'.format(sys.argv[0]))
sys.exit(1)
# GIFs are stored in the local "data" directory
os.mkdir('data')
# We need to connect to the API to lookup the historical host information
api = shodan.Shodan(API_KEY)
# Use the shodan.helpers.iterate_files() method to loop over the Shodan data file
for result in helpers.iterate_files(sys.argv[1]):
# Get the historic info
host = api.host(result['ip_str'], history=True)
# Count how many screenshots this host has
screenshots = []
for banner in host['data']:
# Extract the image from the banner data
if 'opts' in banner and 'screenshot' in banner['opts']:
# Sort the images by the time they were collected so the GIF will loop
# based on the local time regardless of which day the banner was taken.
timestamp = arrow.get(banner['timestamp']).time()
sort_key = timestamp.hour
screenshots.append((
sort_key,
banner['opts']['screenshot']['data']
