parser.add_argument("dataset", help="Dataset", choices=DATASETS)
parser.add_argument("tree", help="file to transform", type=argparse.FileType('r'))
parser.add_argument("--region", help="Region to connect to", default="us-east-1")
parser.add_argument("-c", "--create", help="create table in database", action="store_true")
parser.add_argument("-f", "--flush", help="flush database.", action="store_true")
parser.add_argument("-d", "--dryrun", help="Process data, but don't insert into DB", action="store_true")
parser.add_argument("-v", "--verbose", action="store_true")
args = parser.parse_args()
if args.region == "localhost":
client = boto3.resource('dynamodb', endpoint_url="http://localhost:8000")
else:
client = boto3.resource('dynamodb')
t = Table(client, args.dataset)
if args.flush:
logging.info("Deleting table: " + t.table_name)
if not args.dryrun:
t.delete()
if args.create:
logging.info("Creating table: " + t.table_name)
if not args.dryrun:
t.create(write=2000)
entries = 0
start = time.time()
parser = TreeFile(args.tree)
parser.add_argument("-c", "--create", help="create table in database", action="store_true")
parser.add_argument("-f", "--flush", help="flush database.", action="store_true")
parser.add_argument("-d", "--dryrun", help="Process data, but don't insert into DB", action="store_true")
parser.add_argument("-v", "--verbose", action="store_true")
args = parser.parse_args()
table_name = args.dataset + "_beta"
logging.warning("Appending _beta to table name to comply with Babel expectations. " + table_name)
if args.region == "localhost":
client = boto3.resource('dynamodb', endpoint_url="http://localhost:8000")
else:
client = boto3.resource('dynamodb')
t = Table(client, table_name)
if args.flush:
logging.info("Deleting table: " + t.table_name)
if not args.dryrun:
t.delete()
if args.create:
logging.info("Creating table: " + t.table_name)
if not args.dryrun:
t.create(write=2000)
b = Benchmark()
if args.rec_type:
if args.rec_type == "classic":
make_classic(args, b, t)
