def to_file(query, str_path_output_file, should_convert,
should_add_institutions):
"""Makes Censys Export request with given query, converts results and writes to output file
:param query: Strings which presents Censys SQL queries
:param str_path_output_file: String which points to existing output file
:param should_convert: Boolean if results should be converted
:param should_add_institutions: boolean if an institution field should be added when converting
"""
c = new_api_obj('SQL_EXPORT')
print("Executing query: " + query)
# Start new Job
res = c.new_job(query, flatten=False)
job_id = res["job_id"]
result = c.check_job_loop(job_id)
if result['status'] == 'success':
temp_file = increment_until_new_file("temp")
nr_of_files_counter = 0
total_results = 0
paths = result['download_paths']
for path in paths:
nr_of_files_counter += 1
print("Retrieving file " + str(nr_of_files_counter) + " of " +
str(len(paths)) + "...")
urllib.request.urlretrieve(path, temp_file)
print("Processing results...")
with open(str_path_output_file, 'a') as output_file:
for result in open(temp_file):
result_json = dict_clean_empty(json.loads(result))
output_file.write(json.dumps(result_json) + '\n')
total_results += 1
os.remove(temp_file)
print("Done.")
print(
str(total_results) + ' total results written in ',
str_path_output_file)
if should_convert:
institutions = None
if should_add_institutions:
institutions = get_institutions()
convert_file(str_path_output_file, 'censys', institutions)
else:
print('Censys job failed.' + '\n' + str(result))
from base import ask_input_file
from base import ask_input_directory
from base import increment_until_new_file
from base import create_output_directory
from base import get_option_from_user
from base import ask_continue
import json
import os
print('---IpInfo json converter---')
choice = get_option_from_user('File input or directory input?(f/d)',
{'f', 'd'})
if choice is 'f':
input_file = ask_input_file('outputfiles/ipinfo/')
str_path_output_file = increment_until_new_file(
'converted_outputfiles/' +
os.path.splitext(os.path.basename(str(input_file)))[0] + '-converted' +
os.path.splitext(str(input_file))[1])
with open(str_path_output_file, 'a') as output_file:
for str_banner in input_file.open():
banner = dict_clean_empty(json.loads(str_banner))
to_es_convert(banner)
output_file.write(json.dumps(banner) + '\n')
print('Converted ' + str(input_file.as_posix()) + ' to ' +
str_path_output_file)
elif choice is 'd':
input_directory = ask_input_directory()
files_to_convert = []
for file in os.listdir(input_directory):
if file.endswith(".json"):
files_to_convert.append(file)
print('These files will be converted: ' + str(files_to_convert))