def parse(color, fields, filters, filename, separator, filenames):
"""Extract information out of compressed JSON files."""
# Strip out any whitespace in the fields and turn them into an array
fields = [item.strip() for item in fields.split(',')]
if len(fields) == 0:
raise click.ClickException('Please define at least one property to show')
has_filters = len(filters) > 0
# Setup the output file handle
fout = None
if filename:
# If no filters were provided raise an error since it doesn't make much sense w/out them
if not has_filters:
raise click.ClickException('Output file specified without any filters. Need to use filters with this option.')
# Add the appropriate extension if it's not there atm
if not filename.endswith('.json.gz'):
filename += '.json.gz'
fout = helpers.open_file(filename)
for banner in helpers.iterate_files(filenames):
row = u''
# Validate the banner against any provided filters
if has_filters and not match_filters(banner, filters):
continue
# Append the data
if fout:
helpers.write_banner(fout, banner)
# Loop over all the fields and print the banner as a row
for i, field in enumerate(fields):
tmp = u''
value = get_banner_field(banner, field)
if value:
field_type = type(value)
# If the field is an array then merge it together
if field_type == list:
tmp = u';'.join(value)
elif field_type in [int, float]:
tmp = u'{}'.format(value)
else:
tmp = escape_data(value)
# Colorize certain fields if the user wants it
if color:
tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white'))
# Add the field information to the row
if i > 0:
row += separator
row += tmp
click.echo(row)
def parse(color, fields, filters, filename, separator, filenames):
"""Extract information out of compressed JSON files."""
# Strip out any whitespace in the fields and turn them into an array
fields = [item.strip() for item in fields.split(',')]
if len(fields) == 0:
raise click.ClickException('Please define at least one property to show')
has_filters = len(filters) > 0
# Setup the output file handle
fout = None
if filename:
# If no filters were provided raise an error since it doesn't make much sense w/out them
if not has_filters:
raise click.ClickException('Output file specified without any filters. Need to use filters with this option.')
# Add the appropriate extension if it's not there atm
if not filename.endswith('.json.gz'):
filename += '.json.gz'
fout = helpers.open_file(filename)
for banner in helpers.iterate_files(filenames):
row = u''
# Validate the banner against any provided filters
if has_filters and not match_filters(banner, filters):
continue
# Append the data
if fout:
helpers.write_banner(fout, banner)
# Loop over all the fields and print the banner as a row
for i, field in enumerate(fields):
tmp = u''
value = get_banner_field(banner, field)
if value:
field_type = type(value)
# If the field is an array then merge it together
if field_type == list:
tmp = u';'.join(value)
elif field_type in [int, float]:
tmp = u'{}'.format(value)
else:
tmp = escape_data(value)
# Colorize certain fields if the user wants it
if color:
tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white'))
# Add the field information to the row
if i > 0:
row += separator
row += tmp
click.echo(row)
def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries, asn, alert, compresslevel):
"""Stream data in real-time."""
# Setup the Shodan API
key = get_api_key()
api = shodan.Shodan(key)
# Temporarily change the baseurl
api.stream.base_url = streamer
# Strip out any whitespace in the fields and turn them into an array
fields = [item.strip() for item in fields.split(',')]
if len(fields) == 0:
raise click.ClickException('Please define at least one property to show')
# The user must choose "ports", "countries", "asn" or nothing - can't select multiple
# filtered streams at once.
stream_type = []
if ports:
stream_type.append('ports')
if countries:
stream_type.append('countries')
if asn:
stream_type.append('asn')
if alert:
stream_type.append('alert')
if len(stream_type) > 1:
raise click.ClickException('Please use --ports, --countries OR --asn. You cant subscribe to multiple filtered streams at once.')
stream_args = None
# Turn the list of ports into integers
if ports:
try:
stream_args = [int(item.strip()) for item in ports.split(',')]
except ValueError:
raise click.ClickException('Invalid list of ports')
if alert:
alert = alert.strip()
if alert.lower() != 'all':
stream_args = alert
if asn:
stream_args = asn.split(',')
if countries:
stream_args = countries.split(',')
# Flatten the list of stream types
# Possible values are:
# - all
# - asn
# - countries
# - ports
if len(stream_type) == 1:
stream_type = stream_type[0]
else:
stream_type = 'all'
# Decide which stream to subscribe to based on whether or not ports were selected
def _create_stream(name, args, timeout):
return {
'all': api.stream.banners(timeout=timeout),
'alert': api.stream.alert(args, timeout=timeout),
'asn': api.stream.asn(args, timeout=timeout),
'countries': api.stream.countries(args, timeout=timeout),
'ports': api.stream.ports(args, timeout=timeout),
}.get(name, 'all')
stream = _create_stream(stream_type, stream_args, timeout=timeout)
counter = 0
quit = False
last_time = timestr()
fout = None
if datadir:
fout = open_streaming_file(datadir, last_time, compresslevel)
while not quit:
try:
for banner in stream:
# Limit the number of results to output
if limit > 0:
counter += 1
if counter > limit:
quit = True
break
# Write the data to the file
if datadir:
cur_time = timestr()
if cur_time != last_time:
last_time = cur_time
fout.close()
fout = open_streaming_file(datadir, last_time)
helpers.write_banner(fout, banner)
# Print the banner information to stdout
if not quiet:
row = u''
# Loop over all the fields and print the banner as a row
for field in fields:
tmp = u''
value = get_banner_field(banner, field)
if value:
field_type = type(value)
# If the field is an array then merge it together
if field_type == list:
tmp = u';'.join(value)
elif field_type in [int, float]:
tmp = u'{}'.format(value)
else:
tmp = escape_data(value)
# Colorize certain fields if the user wants it
if color:
tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white'))
# Add the field information to the row
row += tmp
row += separator
click.echo(row)
except requests.exceptions.Timeout:
raise click.ClickException('Connection timed out')
except KeyboardInterrupt:
quit = True
except shodan.APIError as e:
raise click.ClickException(e.value)
except Exception:
# For other errors lets just wait a bit and try to reconnect again
time.sleep(1)
# Create a new stream object to subscribe to
stream = _create_stream(stream_type, stream_args, timeout=timeout)
def search(color, fields, limit, separator, query):
"""Search the Shodan database"""
key = get_api_key()
# Create the query string out of the provided tuple
query = ' '.join(query).strip()
# Make sure the user didn't supply an empty string
if query == '':
raise click.ClickException('Empty search query')
# For now we only allow up to 1000 results at a time
if limit > 1000:
raise click.ClickException('Too many results requested, maximum is 1,000')
# Strip out any whitespace in the fields and turn them into an array
fields = [item.strip() for item in fields.split(',')]
if len(fields) == 0:
raise click.ClickException('Please define at least one property to show')
# Perform the search
api = shodan.Shodan(key)
try:
results = api.search(query, limit=limit)
except shodan.APIError as e:
raise click.ClickException(e.value)
# Error out if no results were found
if results['total'] == 0:
raise click.ClickException('No search results found')
# We buffer the entire output so we can use click's pager functionality
output = u''
for banner in results['matches']:
row = u''
# Loop over all the fields and print the banner as a row
for field in fields:
tmp = u''
value = get_banner_field(banner, field)
if value:
field_type = type(value)
# If the field is an array then merge it together
if field_type == list:
tmp = u';'.join(value)
elif field_type in [int, float]:
tmp = u'{}'.format(value)
else:
tmp = escape_data(value)
# Colorize certain fields if the user wants it
if color:
tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white'))
# Add the field information to the row
row += tmp
row += separator
# click.echo(out + separator, nl=False)
output += row + u'\n'
# click.echo('')
click.echo_via_pager(output)
def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries, asn, alert, compresslevel):
"""Stream data in real-time."""
# Setup the Shodan API
key = get_api_key()
api = shodan.Shodan(key)
# Temporarily change the baseurl
api.stream.base_url = streamer
# Strip out any whitespace in the fields and turn them into an array
fields = [item.strip() for item in fields.split(',')]
if len(fields) == 0:
raise click.ClickException('Please define at least one property to show')
# The user must choose "ports", "countries", "asn" or nothing - can't select multiple
# filtered streams at once.
stream_type = []
if ports:
stream_type.append('ports')
if countries:
stream_type.append('countries')
if asn:
stream_type.append('asn')
if alert:
stream_type.append('alert')
if len(stream_type) > 1:
raise click.ClickException('Please use --ports, --countries OR --asn. You cant subscribe to multiple filtered streams at once.')
stream_args = None
# Turn the list of ports into integers
if ports:
try:
stream_args = [int(item.strip()) for item in ports.split(',')]
except ValueError:
raise click.ClickException('Invalid list of ports')
if alert:
alert = alert.strip()
if alert.lower() != 'all':
stream_args = alert
if asn:
stream_args = asn.split(',')
if countries:
stream_args = countries.split(',')
# Flatten the list of stream types
# Possible values are:
# - all
# - asn
# - countries
# - ports
if len(stream_type) == 1:
stream_type = stream_type[0]
else:
stream_type = 'all'
# Decide which stream to subscribe to based on whether or not ports were selected
def _create_stream(name, args, timeout):
return {
'all': api.stream.banners(timeout=timeout),
'alert': api.stream.alert(args, timeout=timeout),
'asn': api.stream.asn(args, timeout=timeout),
'countries': api.stream.countries(args, timeout=timeout),
'ports': api.stream.ports(args, timeout=timeout),
}.get(name, 'all')
stream = _create_stream(stream_type, stream_args, timeout=timeout)
counter = 0
quit = False
last_time = timestr()
fout = None
if datadir:
fout = open_streaming_file(datadir, last_time, compresslevel)
while not quit:
try:
for banner in stream:
# Limit the number of results to output
if limit > 0:
counter += 1
if counter > limit:
quit = True
break
# Write the data to the file
if datadir:
cur_time = timestr()
if cur_time != last_time:
last_time = cur_time
fout.close()
fout = open_streaming_file(datadir, last_time)
helpers.write_banner(fout, banner)
# Print the banner information to stdout
if not quiet:
row = u''
# Loop over all the fields and print the banner as a row
for field in fields:
tmp = u''
value = get_banner_field(banner, field)
if value:
field_type = type(value)
# If the field is an array then merge it together
if field_type == list:
tmp = u';'.join(value)
elif field_type in [int, float]:
tmp = u'{}'.format(value)
else:
tmp = escape_data(value)
# Colorize certain fields if the user wants it
if color:
tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white'))
# Add the field information to the row
row += tmp
row += separator
click.echo(row)
except requests.exceptions.Timeout:
raise click.ClickException('Connection timed out')
except KeyboardInterrupt:
quit = True
except shodan.APIError as e:
raise click.ClickException(e.value)
except Exception:
# For other errors lets just wait a bit and try to reconnect again
time.sleep(1)
# Create a new stream object to subscribe to
stream = _create_stream(stream_type, stream_args, timeout=timeout)
def search(color, fields, limit, separator, query):
"""Search the Shodan database"""
key = get_api_key()
# Create the query string out of the provided tuple
query = ' '.join(query).strip()
# Make sure the user didn't supply an empty string
if query == '':
raise click.ClickException('Empty search query')
# For now we only allow up to 1000 results at a time
if limit > 1000:
raise click.ClickException('Too many results requested, maximum is 1,000')
# Strip out any whitespace in the fields and turn them into an array
fields = [item.strip() for item in fields.split(',')]
if len(fields) == 0:
raise click.ClickException('Please define at least one property to show')
# Perform the search
api = shodan.Shodan(key)
try:
results = api.search(query, limit=limit)
except shodan.APIError as e:
raise click.ClickException(e.value)
# Error out if no results were found
if results['total'] == 0:
raise click.ClickException('No search results found')
# We buffer the entire output so we can use click's pager functionality
output = u''
for banner in results['matches']:
row = u''
# Loop over all the fields and print the banner as a row
for field in fields:
tmp = u''
value = get_banner_field(banner, field)
if value:
field_type = type(value)
# If the field is an array then merge it together
if field_type == list:
tmp = u';'.join(value)
elif field_type in [int, float]:
tmp = u'{}'.format(value)
else:
tmp = escape_data(value)
# Colorize certain fields if the user wants it
if color:
tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white'))
# Add the field information to the row
row += tmp
row += separator
# click.echo(out + separator, nl=False)
output += row + u'\n'
# click.echo('')
click.echo_via_pager(output)
