def main(args):
pargs = parse_args(args)
logging_setup(pargs.verbose, get_local_path(pargs.dataset_name,'result','result.log'))
print '[STARTED]'
local_extract(pargs.dataset_name, pargs.extractor,
pargs.timeout, pargs.retry_failed, pargs.skip_existing)
print '[DONE]'
def _verify_args(args):
# verify arguments provoded by argparse and
# return the path to the output directory
# printing arguments
print 'dataset type: %s' % args.dataset_type
print 'dataset name: %s' % args.dataset_name
#validate dataset name
if not check_local_path(args.dataset_name):
print 'error: this dataset does not exist'
sys.exit(-1)
# validate path argument
if args.path and not os.path.exists(args.path):
print 'error: path does not exist'
sys.exit(-1)
output_dir = args.path or get_local_path(args.dataset_name)
print 'output directory: %s' % output_dir
return output_dir
def _verify_args(args):
# verify arguments provoded by argparse and
# return the path to the output directory
# printing arguments
print 'dataset type: %s' % args.dataset_type
print 'dataset name: %s' % args.dataset_name
#validate dataset name
if not check_local_path(args.dataset_name):
print 'error: this dataset does not exist'
sys.exit(-1)
# validate path argument
if args.path and not os.path.exists(args.path):
print 'error: path does not exist'
sys.exit(-1)
output_dir = args.path or get_local_path(args.dataset_name)
print 'output directory: %s' % output_dir
return output_dir
def __init__(self, output_dir, dataset_name):
self.dataset_name = dataset_name
self._dataset_dir = get_local_path(dataset_name)
self._output_dir = output_dir
self.meta_data_list = [] # list to be serialized
def __init__(self, output_dir, dataset_name):
self.dataset_name = dataset_name
self._dataset_dir = get_local_path(dataset_name)
self._output_dir = output_dir
self.meta_data_list = [] # list to be serialized
