def main(args): pargs = parse_args(args) logging_setup(pargs.verbose, get_local_path(pargs.dataset_name,'result','result.log')) print '[STARTED]' local_extract(pargs.dataset_name, pargs.extractor, pargs.timeout, pargs.retry_failed, pargs.skip_existing) print '[DONE]'
def _verify_args(args): # verify arguments provoded by argparse and # return the path to the output directory # printing arguments print 'dataset type: %s' % args.dataset_type print 'dataset name: %s' % args.dataset_name #validate dataset name if not check_local_path(args.dataset_name): print 'error: this dataset does not exist' sys.exit(-1) # validate path argument if args.path and not os.path.exists(args.path): print 'error: path does not exist' sys.exit(-1) output_dir = args.path or get_local_path(args.dataset_name) print 'output directory: %s' % output_dir return output_dir
def __init__(self, output_dir, dataset_name): self.dataset_name = dataset_name self._dataset_dir = get_local_path(dataset_name) self._output_dir = output_dir self.meta_data_list = [] # list to be serialized