def preloadBannerTimestamps(input_file): bannerTSDict = {} if not input_file.endswith('.json.gz'): input_file += '.json.gz' for result in helpers.iterate_files(input_file): bannerTSDict[result['ip_str']] = result['timestamp'] return bannerTSDict
def preloadBannerTimestamps(input_file): bannerTSDict = {} if not input_file.endswith('.json.gz'): input_file += '.json.gz' for result in helpers.iterate_files(input_file): bannerTSDict[result['ip_str']] = result['timestamp'] return bannerTSDict
def parse(color, fields, filters, filename, separator, filenames): """Extract information out of compressed JSON files.""" # Strip out any whitespace in the fields and turn them into an array fields = [item.strip() for item in fields.split(',')] if len(fields) == 0: raise click.ClickException('Please define at least one property to show') has_filters = len(filters) > 0 # Setup the output file handle fout = None if filename: # If no filters were provided raise an error since it doesn't make much sense w/out them if not has_filters: raise click.ClickException('Output file specified without any filters. Need to use filters with this option.') # Add the appropriate extension if it's not there atm if not filename.endswith('.json.gz'): filename += '.json.gz' fout = helpers.open_file(filename) for banner in helpers.iterate_files(filenames): row = u'' # Validate the banner against any provided filters if has_filters and not match_filters(banner, filters): continue # Append the data if fout: helpers.write_banner(fout, banner) # Loop over all the fields and print the banner as a row for i, field in enumerate(fields): tmp = u'' value = get_banner_field(banner, field) if value: field_type = type(value) # If the field is an array then merge it together if field_type == list: tmp = u';'.join(value) elif field_type in [int, float]: tmp = u'{}'.format(value) else: tmp = escape_data(value) # Colorize certain fields if the user wants it if color: tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white')) # Add the field information to the row if i > 0: row += separator row += tmp click.echo(row)
def parse(color, fields, filters, filename, separator, filenames): """Extract information out of compressed JSON files.""" # Strip out any whitespace in the fields and turn them into an array fields = [item.strip() for item in fields.split(',')] if len(fields) == 0: raise click.ClickException('Please define at least one property to show') has_filters = len(filters) > 0 # Setup the output file handle fout = None if filename: # If no filters were provided raise an error since it doesn't make much sense w/out them if not has_filters: raise click.ClickException('Output file specified without any filters. Need to use filters with this option.') # Add the appropriate extension if it's not there atm if not filename.endswith('.json.gz'): filename += '.json.gz' fout = helpers.open_file(filename) for banner in helpers.iterate_files(filenames): row = u'' # Validate the banner against any provided filters if has_filters and not match_filters(banner, filters): continue # Append the data if fout: helpers.write_banner(fout, banner) # Loop over all the fields and print the banner as a row for i, field in enumerate(fields): tmp = u'' value = get_banner_field(banner, field) if value: field_type = type(value) # If the field is an array then merge it together if field_type == list: tmp = u';'.join(value) elif field_type in [int, float]: tmp = u'{}'.format(value) else: tmp = escape_data(value) # Colorize certain fields if the user wants it if color: tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white')) # Add the field information to the row if i > 0: row += separator row += tmp click.echo(row)
def main(args): # Basic input validation if len(args) <= 1: print('Usage: {} <data.json.gz> [list of data files...]'.args[0]) return 1 # Keep track of the frequency of each key tracker = defaultdict(int) # Loop over all the banners in the provided Shodan data files for banner in iterate_files(args[1:]): # Find "Keys" section of the banner text start = banner['data'].find('# Keys\r\n') if start < 0: continue start += len('# Keys\r\n') # Find the start of the next section end = banner['data'].find('\r\n\r\n', start) # Grab the text in between keys = banner['data'][start:end] # Parse the keys line by line keys = keys.split('\r\n') for key in keys: # Skip comments or empty keys if key.startswith('#') or key.strip() == '': continue # Increment the frequency counter for this key tracker[key] += 1 # Lets print the results print('Top {} Redis Keys\n'.format(TOP_RESULTS)) # Turn the defaultdict into a list of items (iteritems()) # Sort it in reverse (i.e. most common value will be at the start of the list) # And sort it based on the value instead of the key name (lambda ...) sorted_items = sorted(tracker.iteritems(), reverse=True, key=lambda (k, v): v) # Use the enumerate() method to give us a counter and start the counter at 1 for i, item in enumerate(sorted_items, start=1): # The key names are padded to a length of 30 characters so the output aligns better print('#{}\t{:30s}\t{}'.format(i, item[0], item[1])) # Stop looping over results once we've printed the top X values if i >= TOP_RESULTS: break
def populateMetadata(image): print 'In populateMetadata function' for result in helpers.iterate_files('screenshots.json.gz'): if result['ip_str'] == image: print 'Found matching result!!!!!*****#########' global ipAddress if result['ip_str'] is not None: ipAddress = result['ip_str'] + '<br/>' else: ipAddress = '' global country if result['location']['country_name'] is not None: country = result['location']['country_name'] else: country = '' global city if result['location']['city'] is not None: city = ', ' + result['location']['city'] else: city = '' global hostnames if result['hostnames'] is not None: hostnames = str(result['hostnames']).replace("[", "").replace( "]", "").replace("u", "") + '<br/>' else: hostnames = '' global isp if result['isp'] is not None: isp = result['isp'] + '<br/>' else: isp = '' global timestamp if result['timestamp'] is not None: timestamp = result['timestamp'].replace("T", " ") + '<br/>' else: timestamp = ''
def populateMetadata(image): print 'In populateMetadata function' for result in helpers.iterate_files('screenshots.json.gz'): if result['ip_str'] == image: print 'Found matching result!!!!!*****#########' global ipAddress if result['ip_str'] is not None: ipAddress = result['ip_str'] + '<br/>' else: ipAddress = '' global country if result['location']['country_name'] is not None: country = result['location']['country_name'] else: country = '' global city if result['location']['city'] is not None: city = ', ' + result['location']['city'] else: city = '' global hostnames if result['hostnames'] is not None: hostnames = str(result['hostnames']).replace("[", "").replace("]", "").replace("u", "") + '<br/>' else: hostnames = '' global isp if result['isp'] is not None: isp = result['isp'] + '<br/>' else: isp = '' global timestamp if result['timestamp'] is not None: timestamp = result['timestamp'].replace("T", " ") + '<br/>' else: timestamp = ''
# Use this script to parse ip addresses in a file and filters by tags listed in the array below. It then makes the results readable and digestible. # This tool should be used in conjunction with the other Shodan scripts in the repo. # Import the method that helps us parse the data file from shodan.helpers import iterate_files, open_file, write_banner # Standard Python libraries from pprint import pprint from sys import argv, exit # Settings OUTPUT_FILENAME = 'iterate-query.json.gz' # The user has to provide at least 1 file if len(argv) == 1: print('Usage: {} <file1.json.gz> [file2.json.gz] ...'.format(argv[0])) exit(1) # Create the output file with open_file(OUTPUT_FILENAME) as fout: # Iterate over all of the provided data files for banner in iterate_files(argv[1:]): # Is the service listed? if 'tags' in banner and 'vpn' or 'http' or 'https' or 'ftp' or 'telnet' or 'smtp' or 'ssh' or 'mysql' or 'mssql' or 'snmp' in banner['tags']: # Show the banner pprint(banner) # Store it in the output file write_banner(fout, banner)
# Settings API_KEY = '' MIN_SCREENS = 5 # Number of screenshots that Shodan needs to have in order to make a GIF MAX_SCREENS = 24 if len(sys.argv) != 2: print('Usage: {} <shodan-data.json.gz>'.format(sys.argv[0])) sys.exit(1) # GIFs are stored in the local "data" directory os.mkdir('data') # We need to connect to the API to lookup the historical host information api = shodan.Shodan(API_KEY) for result in helpers.iterate_files(sys.argv[1]): # Get the historic info host = api.host(result['ip_str'], history=True) # Count how many screenshots this host has screenshots = [] for banner in host['data']: if 'opts' in banner and 'screenshot' in banner['opts']: timestamp = arrow.get(banner['timestamp']).time() sort_key = timestamp.hour screenshots.append( (sort_key, banner['opts']['screenshot']['data'])) if len(screenshots) >= MAX_SCREENS: break
import shodan import shodan.helpers as helpers import os import sys # Usage if len(sys.argv) != 3: print('Usage: {} <Shodan json.gz> <output directory>'.format(sys.argv[0])) sys.exit(-1) input_file = sys.argv[1] output_dir = sys.argv[2] # Make sure the directory exists if not os.path.exists(output_dir): os.mkdir(output_dir) for banner in helpers.iterate_files(input_file): # Try to grab the screenshot from the banner screenshot = helpers.get_screenshot(banner) # If we found a screenshot then create a file w/ the data if screenshot: # Create the file handle image = open('{}/{}.jpg'.format(output_dir, banner['ip_str']), 'w') # Write the image data which is stored using base64 encoding image.write(screenshot['data'].decode('base64'))
# GIFs are stored in the local "snapshots" directory try: os.makedirs('snapshots/tmp') except OSError: pass # Setup the Shodan API object api = shodan.Shodan(SHODAN_API_KEY) # Show a progress indicator BAR_FORMAT = '{desc}{percentage:3.0f}%[{bar}] {n_fmt}/{total_fmt} ' '[{elapsed}<{remaining}, {rate_fmt}{postfix}]' pbar = tqdm.tqdm(total=res_count, ascii=True, bar_format=BAR_FORMAT) pbar.set_description(desc='Сollecting snapshots from the JSON') # Loop over all of the Shodan data files the user provided for banner in helpers.iterate_files(f'{city.lower()}-data.json.gz'): # See whether the current banner has a screenshot, # if it does then lets lookup more information about this IP has_screenshot = helpers.get_screenshot(banner) if has_screenshot: try: ip = helpers.get_ip(banner) pbar.write(f'Looking up {ip}') host = api.host(ip, history=True) country = host['country_name'] city = host['city'] # Store all the historical screenshots for this IP screenshots = [] for tmp_banner in host['data']: # Try to extract the image from the banner data
API_KEY = '' MIN_SCREENS = 5 # Number of screenshots that Shodan needs to have in order to make a GIF MAX_SCREENS = 24 if len(sys.argv) != 2: print('Usage: {} <shodan-data.json.gz>'.format(sys.argv[0])) sys.exit(1) # GIFs are stored in the local "data" directory os.mkdir('data') # We need to connect to the API to lookup the historical host information api = shodan.Shodan(API_KEY) # Use the shodan.helpers.iterate_files() method to loop over the Shodan data file for result in helpers.iterate_files(sys.argv[1]): # Get the historic info host = api.host(result['ip_str'], history=True) # Count how many screenshots this host has screenshots = [] for banner in host['data']: # Extract the image from the banner data if 'opts' in banner and 'screenshot' in banner['opts']: # Sort the images by the time they were collected so the GIF will loop # based on the local time regardless of which day the banner was taken. timestamp = arrow.get(banner['timestamp']).time() sort_key = timestamp.hour screenshots.append(( sort_key, banner['opts']['screenshot']['data']