Example #1
0
def main():
    possible_file_types = ['csv', 'json', 'crits']

    parser = argparse.ArgumentParser()
    parser.add_argument('--version', action='version', version='%(prog)s (version {0})'.format(__version__))
    parser.add_argument('-t', '--type', dest='file_type', choices=possible_file_types, default=possible_file_types[0],
                        help="Specify output file type.")
    parser.add_argument('-f', '--file', default=None, help="Specify output file. Defaults to harvest.FILETYPE")
    parser.add_argument('--output-dir', dest='output_dir', default='',
                        help="Specify output direction. Default to current directory")
    parser.add_argument('-d', '--delete', help="Delete intermediate files", action="store_true")
    parser.add_argument('-e', '--enrich', help="Enrich data", action="store_true")
    parser.add_argument('--tiq-test', help="Output in tiq-test format", action="store_true")
    args = parser.parse_args()

    if len(args.output_dir):
        if not os.path.isdir(args.output_dir):
            parser.print_usage()
            sys.exit('Output directory "{0}" does not exist.'.format(args.output_dir))

    def filepath(filename):
        return os.path.join(args.output_dir, filename)

    if args.file:
        out_filepath = filepath(args.file)
    else:
        out_filepath = filepath('harvest.' + args.file_type)

    harvest_filepath = filepath('harvest.json')
    crop_filepath = filepath('crop.json')
    enrich_filepath = filepath('enrich.json')
    enriched_filepath = filepath('enriched.' + args.file_type)

    reap(harvest_filepath)
    thresh(harvest_filepath, crop_filepath)
    bale(crop_filepath, out_filepath, args.file_type, True)

    if args.enrich or args.tiq_test:
        winnow(crop_filepath, crop_filepath, enrich_filepath)
        bale(enrich_filepath, enriched_filepath, args.file_type, False)

    if args.tiq_test:
        tiq_output(crop_filepath, enrich_filepath)

    if args.delete:
        # be careful with this when we support a JSON output type
        os.remove(harvest_filepath)
        os.remove(crop_filepath)
        os.remove(enrich_filepath)
Example #2
0
parser.add_argument('--tiq-test', help="Output in tiq-test format", action="store_true")
args = parser.parse_args()

possible_types = ['csv', 'CSV']

if not args.type:
    out_type = 'csv'
elif args.type not in possible_types:
    sys.exit('Invalid file type specified. Possible types are: %s' % possible_types)
else:
    out_type = args.type

if args.file:
    out_file = args.file
else:
    out_file = 'harvest.'+out_type

reap('harvest.json')
thresh('harvest.json', 'crop.json')
if args.enrich:
    winnow('crop.json', 'crop.json', 'enrich.json')
bale('crop.json', out_file, out_type)

if args.tiq_test:
    tiq_output('crop.json', 'enrich.json')

if args.delete:
    # be careful with this when we support a JSON output type
    os.remove('harvest.json')
    os.remove('crop.json')
Example #3
0
    out_type = 'csv'
elif args.type.lower() not in possible_types:
    sys.exit('Invalid file type specified. Possible types are: %s' % possible_types)
else:
    out_type = args.type.lower()

if args.file:
    out_file = args.file
else:
    out_file = 'harvest.'+out_type

base_path = os.path.dirname(__file__) + '/'
bp = base_path

reap(bp + 'harvest.json')
thresh(bp + 'harvest.json', bp + 'crop.json')
bale(bp + 'crop.json', out_file, out_type, True)

if args.enrich or args.tiq_test:
    winnow(bp + 'crop.json', bp + 'crop.json', bp +'enrich.json')
    bale(bp + 'enrich.json', bp + 'enriched.'+out_type, out_type, False)

if args.tiq_test:
    tiq_output(bp + 'crop.json', bp + 'enrich.json')

if args.delete:
    # be careful with this when we support a JSON output type
    os.remove(bp + 'harvest.json')
    os.remove(bp + 'crop.json')
    os.remove(bp + 'enrich.json')