def main():
possible_file_types = ['csv', 'json', 'crits']
parser = argparse.ArgumentParser()
parser.add_argument('--version', action='version', version='%(prog)s (version {0})'.format(__version__))
parser.add_argument('-t', '--type', dest='file_type', choices=possible_file_types, default=possible_file_types[0],
help="Specify output file type.")
parser.add_argument('-f', '--file', default=None, help="Specify output file. Defaults to harvest.FILETYPE")
parser.add_argument('--output-dir', dest='output_dir', default='',
help="Specify output direction. Default to current directory")
parser.add_argument('-d', '--delete', help="Delete intermediate files", action="store_true")
parser.add_argument('-e', '--enrich', help="Enrich data", action="store_true")
parser.add_argument('--tiq-test', help="Output in tiq-test format", action="store_true")
args = parser.parse_args()
if len(args.output_dir):
if not os.path.isdir(args.output_dir):
parser.print_usage()
sys.exit('Output directory "{0}" does not exist.'.format(args.output_dir))
def filepath(filename):
return os.path.join(args.output_dir, filename)
if args.file:
out_filepath = filepath(args.file)
else:
out_filepath = filepath('harvest.' + args.file_type)
harvest_filepath = filepath('harvest.json')
crop_filepath = filepath('crop.json')
enrich_filepath = filepath('enrich.json')
enriched_filepath = filepath('enriched.' + args.file_type)
reap(harvest_filepath)
thresh(harvest_filepath, crop_filepath)
bale(crop_filepath, out_filepath, args.file_type, True)
if args.enrich or args.tiq_test:
winnow(crop_filepath, crop_filepath, enrich_filepath)
bale(enrich_filepath, enriched_filepath, args.file_type, False)
if args.tiq_test:
tiq_output(crop_filepath, enrich_filepath)
if args.delete:
# be careful with this when we support a JSON output type
os.remove(harvest_filepath)
os.remove(crop_filepath)
os.remove(enrich_filepath)
parser.add_argument('--tiq-test', help="Output in tiq-test format", action="store_true")
args = parser.parse_args()
possible_types = ['csv', 'CSV']
if not args.type:
out_type = 'csv'
elif args.type not in possible_types:
sys.exit('Invalid file type specified. Possible types are: %s' % possible_types)
else:
out_type = args.type
if args.file:
out_file = args.file
else:
out_file = 'harvest.'+out_type
reap('harvest.json')
thresh('harvest.json', 'crop.json')
if args.enrich:
winnow('crop.json', 'crop.json', 'enrich.json')
bale('crop.json', out_file, out_type)
if args.tiq_test:
tiq_output('crop.json', 'enrich.json')
if args.delete:
# be careful with this when we support a JSON output type
os.remove('harvest.json')
os.remove('crop.json')
if not args.type:
out_type = 'csv'
elif args.type.lower() not in possible_types:
sys.exit('Invalid file type specified. Possible types are: %s' %
possible_types)
else:
out_type = args.type.lower()
if args.file:
out_file = args.file
else:
out_file = 'harvest.' + out_type
reap('harvest.json')
thresh('harvest.json', 'crop.json')
bale('crop.json', out_file, out_type, True)
if args.enrich or args.tiq_test:
winnow('crop.json', 'crop.json', 'enrich.json')
bale('enrich.json', 'enriched.' + out_type, out_type, False)
if args.tiq_test:
tiq_output('crop.json', 'enrich.json')
if args.delete:
# be careful with this when we support a JSON output type
os.remove('harvest.json')
os.remove('crop.json')
os.remove('enrich.json')
possible_types = ['csv', 'json','crits', 'cef' ]
if not args.type:
out_type = 'csv'
elif args.type.lower() not in possible_types:
sys.exit('Invalid file type specified. Possible types are: %s' % possible_types)
else:
out_type = args.type.lower()
if args.file:
out_file = args.file
else:
out_file = 'harvest.'+out_type
reap('harvest.json')
thresh('harvest.json', 'crop.json')
bale('crop.json', out_file, out_type, True)
if args.enrich or args.tiq_test:
winnow('crop.json', 'crop.json', 'enrich.json')
bale('enrich.json', 'enriched.'+out_type, out_type, False)
if args.tiq_test:
tiq_output('crop.json', 'enrich.json')
if args.delete:
# be careful with this when we support a JSON output type
os.remove('harvest.json')
os.remove('crop.json')
os.remove('enrich.json')