def host(format, history, filename, save, ip): """View all available information for an IP address""" key = get_api_key() api = shodan.Shodan(key) try: host = api.host(ip, history=history) # Print the host information to the terminal using the user-specified format HOST_PRINT[format](host, history=history) # Store the results if filename or save: if save: filename = '{}.json.gz'.format(ip) # Add the appropriate extension if it's not there atm if not filename.endswith('.json.gz'): filename += '.json.gz' # Create/ append to the file fout = helpers.open_file(filename) for banner in sorted(host['data'], key=lambda k: k['port']): if 'placeholder' not in banner: helpers.write_banner(fout, banner) except shodan.APIError as e: raise click.ClickException(e.value)
def scan_internet(quiet, port, protocol): """Scan the Internet for a specific port and protocol using the Shodan infrastructure.""" key = get_api_key() api = shodan.Shodan(key) try: # Submit the request to Shodan click.echo('Submitting Internet scan to Shodan...', nl=False) scan = api.scan_internet(port, protocol) click.echo('Done') # If the requested port is part of the regular Shodan crawling, then # we don't know when the scan is done so lets return immediately and # let the user decide when to stop waiting for further results. official_ports = api.ports() if port in official_ports: click.echo( 'The requested port is already indexed by Shodan. A new scan for the port has been launched, please subscribe to the real-time stream for results.' ) else: # Create the output file filename = '{0}-{1}.json.gz'.format(port, protocol) counter = 0 with helpers.open_file(filename, 'w') as fout: click.echo('Saving results to file: {0}'.format(filename)) # Start listening for results done = False # Keep listening for results until the scan is done click.echo('Waiting for data, please stand by...') while not done: try: for banner in api.stream.ports([port], timeout=90): counter += 1 helpers.write_banner(fout, banner) if not quiet: click.echo('{0:<40} {1:<20} {2}'.format( click.style(helpers.get_ip(banner), fg=COLORIZE_FIELDS['ip_str']), click.style(str(banner['port']), fg=COLORIZE_FIELDS['port']), ';'.join(banner['hostnames']))) except shodan.APIError: # We stop waiting for results if the scan has been processed by the crawlers and # there haven't been new results in a while if done: break scan = api.scan_status(scan['id']) if scan['status'] == 'DONE': done = True except socket.timeout: # We stop waiting for results if the scan has been processed by the crawlers and # there haven't been new results in a while if done: break scan = api.scan_status(scan['id']) if scan['status'] == 'DONE': done = True except Exception as e: raise click.ClickException(repr(e)) click.echo('Scan finished: {0} devices found'.format(counter)) except shodan.APIError as e: raise click.ClickException(e.value)
def scan_submit(wait, filename, force, verbose, netblocks): """Scan an IP/ netblock using Shodan.""" key = get_api_key() api = shodan.Shodan(key) alert = None # Submit the IPs for scanning try: # Submit the scan scan = api.scan(netblocks, force=force) now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M') click.echo('') click.echo('Starting Shodan scan at {} - {} scan credits left'.format( now, scan['credits_left'])) if verbose: click.echo('# Scan ID: {}'.format(scan['id'])) # Return immediately if wait <= 0: click.echo( 'Exiting now, not waiting for results. Use the API or website to retrieve the results of the scan.' ) else: # Setup an alert to wait for responses alert = api.create_alert('Scan: {}'.format(', '.join(netblocks)), netblocks) # Create the output file if necessary filename = filename.strip() fout = None if filename != '': # Add the appropriate extension if it's not there atm if not filename.endswith('.json.gz'): filename += '.json.gz' fout = helpers.open_file(filename, 'w') # Start a spinner finished_event = threading.Event() progress_bar_thread = threading.Thread(target=async_spinner, args=(finished_event, )) progress_bar_thread.start() # Now wait a few seconds for items to get returned hosts = collections.defaultdict(dict) done = False scan_start = time.time() cache = {} while not done: try: for banner in api.stream.alert(aid=alert['id'], timeout=wait): ip = banner.get('ip', banner.get('ipv6', None)) if not ip: continue # Don't show duplicate banners cache_key = '{}:{}'.format(ip, banner['port']) if cache_key not in cache: hosts[helpers.get_ip(banner)][ banner['port']] = banner cache[cache_key] = True # If we've grabbed data for more than 60 seconds it might just be a busy network and we should move on if time.time() - scan_start >= 60: scan = api.scan_status(scan['id']) if verbose: click.echo('# Scan status: {}'.format( scan['status'])) if scan['status'] == 'DONE': done = True break except shodan.APIError: # If the connection timed out before the timeout, that means the streaming server # that the user tried to reach is down. In that case, lets wait briefly and try # to connect again! if (time.time() - scan_start) < wait: time.sleep(0.5) continue # Exit if the scan was flagged as done somehow if done: break scan = api.scan_status(scan['id']) if scan['status'] == 'DONE': done = True if verbose: click.echo('# Scan status: {}'.format(scan['status'])) except socket.timeout: # If the connection timed out before the timeout, that means the streaming server # that the user tried to reach is down. In that case, lets wait a second and try # to connect again! if (time.time() - scan_start) < wait: continue done = True except Exception as e: finished_event.set() progress_bar_thread.join() raise click.ClickException(repr(e)) finished_event.set() progress_bar_thread.join() def print_field(name, value): click.echo(' {:25s}{}'.format(name, value)) def print_banner(banner): click.echo(' {:20s}'.format( click.style(str(banner['port']), fg='green') + '/' + banner['transport']), nl=False) if 'product' in banner: click.echo(banner['product'], nl=False) if 'version' in banner: click.echo(' ({})'.format(banner['version']), nl=False) click.echo('') # Show optional ssl info if 'ssl' in banner: if 'versions' in banner['ssl']: # Only print SSL versions if they were successfully tested versions = [ version for version in sorted(banner['ssl']['versions']) if not version.startswith('-') ] if len(versions) > 0: click.echo(' |-- SSL Versions: {}'.format( ', '.join(versions))) if 'dhparams' in banner['ssl'] and banner['ssl'][ 'dhparams']: click.echo(' |-- Diffie-Hellman Parameters:') click.echo(' {:15s}{}\n {:15s}{}'.format( 'Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator'])) if 'fingerprint' in banner['ssl']['dhparams']: click.echo(' {:15s}{}'.format( 'Fingerprint:', banner['ssl']['dhparams']['fingerprint'])) if hosts: # Remove the remaining spinner character click.echo('\b ') for ip in sorted(hosts): host = next(iter(hosts[ip].items()))[1] click.echo(click.style(ip, fg='cyan'), nl=False) if 'hostnames' in host and host['hostnames']: click.echo(' ({})'.format(', '.join( host['hostnames'])), nl=False) click.echo('') if 'location' in host and 'country_name' in host[ 'location'] and host['location']['country_name']: print_field('Country', host['location']['country_name']) if 'city' in host['location'] and host['location'][ 'city']: print_field('City', host['location']['city']) if 'org' in host and host['org']: print_field('Organization', host['org']) if 'os' in host and host['os']: print_field('Operating System', host['os']) click.echo('') # Output the vulnerabilities the host has if 'vulns' in host and len(host['vulns']) > 0: vulns = [] for vuln in host['vulns']: if vuln.startswith('!'): continue if vuln.upper() == 'CVE-2014-0160': vulns.append( click.style('Heartbleed', fg='red')) else: vulns.append(click.style(vuln, fg='red')) if len(vulns) > 0: click.echo(' {:25s}'.format('Vulnerabilities:'), nl=False) for vuln in vulns: click.echo(vuln + '\t', nl=False) click.echo('') # Print all the open ports: click.echo(' Open Ports:') for port in sorted(hosts[ip]): print_banner(hosts[ip][port]) # Save the banner in a file if necessary if fout: helpers.write_banner(fout, hosts[ip][port]) click.echo('') else: # Prepend a \b to remove the spinner click.echo( '\bNo open ports found or the host has been recently crawled and cant get scanned again so soon.' ) except shodan.APIError as e: raise click.ClickException(e.value) finally: # Remove any alert if alert: api.delete_alert(alert['id'])
def parse(color, fields, filters, filename, separator, filenames): """Extract information out of compressed JSON files.""" # Strip out any whitespace in the fields and turn them into an array fields = [item.strip() for item in fields.split(',')] if len(fields) == 0: raise click.ClickException( 'Please define at least one property to show') has_filters = len(filters) > 0 # Setup the output file handle fout = None if filename: # If no filters were provided raise an error since it doesn't make much sense w/out them if not has_filters: raise click.ClickException( 'Output file specified without any filters. Need to use filters with this option.' ) # Add the appropriate extension if it's not there atm if not filename.endswith('.json.gz'): filename += '.json.gz' fout = helpers.open_file(filename) for banner in helpers.iterate_files(filenames): row = u'' # Validate the banner against any provided filters if has_filters and not match_filters(banner, filters): continue # Append the data if fout: helpers.write_banner(fout, banner) # Loop over all the fields and print the banner as a row for i, field in enumerate(fields): tmp = u'' value = get_banner_field(banner, field) if value: field_type = type(value) # If the field is an array then merge it together if field_type == list: tmp = u';'.join(value) elif field_type in [int, float]: tmp = u'{}'.format(value) else: tmp = escape_data(value) # Colorize certain fields if the user wants it if color: tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white')) # Add the field information to the row if i > 0: row += separator row += tmp click.echo(row)
def download(limit, skip, filename, query): """Download search results and save them in a compressed JSON file.""" key = get_api_key() # Create the query string out of the provided tuple query = ' '.join(query).strip() # Make sure the user didn't supply an empty string if query == '': raise click.ClickException('Empty search query') filename = filename.strip() if filename == '': raise click.ClickException('Empty filename') # Add the appropriate extension if it's not there atm if not filename.endswith('.json.gz'): filename += '.json.gz' # Perform the search api = shodan.Shodan(key) try: total = api.count(query)['total'] info = api.info() except Exception: raise click.ClickException( 'The Shodan API is unresponsive at the moment, please try again later.' ) # Print some summary information about the download request click.echo('Search query:\t\t\t%s' % query) click.echo('Total number of results:\t%s' % total) click.echo('Query credits left:\t\t%s' % info['unlocked_left']) click.echo('Output file:\t\t\t%s' % filename) if limit > total: limit = total # A limit of -1 means that we should download all the data if limit <= 0: limit = total # Adjust the total number of results we should expect to download if the user is skipping results if skip > 0: limit -= skip with helpers.open_file(filename, 'w') as fout: count = 0 try: cursor = api.search_cursor(query, minify=False, skip=skip) with click.progressbar(cursor, length=limit) as bar: for banner in bar: helpers.write_banner(fout, banner) count += 1 if count >= limit: break except Exception: pass # Let the user know we're done if count < limit: click.echo( click.style('Notice: fewer results were saved than requested', 'yellow')) click.echo( click.style( u'Saved {} results into file {}'.format(count, filename), 'green'))
def domain_info(domain, details, save, history, type): """View all available information for a domain""" key = get_api_key() api = shodan.Shodan(key) try: info = api.dns.domain_info(domain, history=history, type=type) except shodan.APIError as e: raise click.ClickException(e.value) # Grab the host information for any IP records that were returned hosts = {} if details: ips = [ record['value'] for record in info['data'] if record['type'] in ['A', 'AAAA'] ] ips = set(ips) fout = None if save: filename = u'{}-hosts.json.gz'.format(domain) fout = helpers.open_file(filename) for ip in ips: try: hosts[ip] = api.host(ip) # Store the banners if requested if fout: for banner in hosts[ip]['data']: if 'placeholder' not in banner: helpers.write_banner(fout, banner) except shodan.APIError: pass # Ignore any API lookup errors as this isn't critical information # Save the DNS data if save: filename = u'{}.json.gz'.format(domain) fout = helpers.open_file(filename) for record in info['data']: helpers.write_banner(fout, record) click.secho(info['domain'].upper(), fg='green') click.echo('') for record in info['data']: click.echo( u'{:32} {:14} {}'.format( click.style(record['subdomain'], fg='cyan'), click.style(record['type'], fg='yellow'), record['value']), nl=False, ) if record['value'] in hosts: host = hosts[record['value']] click.secho(u' Ports: {}'.format(', '.join( [str(port) for port in sorted(host['ports'])])), fg='blue', nl=False) click.echo('')
def alert_download(filename, alert_id): """Download all information for monitored networks/ IPs.""" key = get_api_key() api = shodan.Shodan(key) ips = set() networks = set() # Helper method to process batches of IPs def batch(iterable, size=1): iter_length = len(iterable) for ndx in range(0, iter_length, size): yield iterable[ndx:min(ndx + size, iter_length)] try: # Get the list of alerts for the user click.echo('Looking up alert information...') if alert_id: alerts = [api.alerts(aid=alert_id.strip())] else: alerts = api.alerts() click.echo('Compiling list of networks/ IPs to download...') for alert in alerts: for net in alert['filters']['ip']: if '/' in net: networks.add(net) else: ips.add(net) click.echo('Downloading...') with open_file(filename) as fout: # Check if the user is able to use batch IP lookups batch_size = 1 if len(ips) > 0: api_info = api.info() if api_info['plan'] in ['corp', 'stream-100']: batch_size = 100 # Convert it to a list so we can index into it ips = list(ips) # Grab all the IP information for ip in batch(ips, size=batch_size): try: click.echo(ip) results = api.host(ip) if not isinstance(results, list): results = [results] for host in results: for banner in host['data']: write_banner(fout, banner) except APIError: pass sleep( 1 ) # Slow down a bit to make sure we don't hit the rate limit # Grab all the network ranges for net in networks: try: counter = 0 click.echo(net) for banner in api.search_cursor('net:{}'.format(net)): write_banner(fout, banner) # Slow down a bit to make sure we don't hit the rate limit if counter % 100 == 0: sleep(1) counter += 1 except APIError: pass except shodan.APIError as e: raise click.ClickException(e.value) click.secho('Successfully downloaded results into: {}'.format(filename), fg='green')