def crawl(arguments):
if arguments.remote is not None:
call(['scp', '-r', '.', '{}:crawler'.format(arguments.remote)])
if arguments.minimal:
minimal_option = '--minimal'
else:
minimal_option = ''
if arguments.parse_log:
parse_log_option = '--parse_log'
else:
parse_log_option = ''
call(['ssh', '-t', arguments.remote, 'python3 -B crawler/crawler.py crawl {} {} crawler/output.json'.format(minimal_option, parse_log_option)])
call(['scp', '{}:crawler/output.json'.format(arguments.remote), arguments.output_file_name])
call(['ssh', arguments.remote, 'rm -rf crawler'])
sys.exit()
if arguments.parse_log:
service_retrieval_class = PodManagerLogParser
else:
service_retrieval_class = LeasesParser
service_descriptions = service_retrieval_class.retrieve_service_descriptions()
data = Data()
for service_description in service_descriptions:
bot = Bot(service_description)
service = bot.crawl()
data.add_service(service)
json_data_representation = DataToJsonConverter.convert(data, arguments.minimal)
with open(arguments.output_file_name, 'w') as output_file:
JsonUtils.dump(json_data_representation, output_file, indent=4)
