def cli_teardown(argv):
"""Command-line interface for tearing down the remote environment."""
parser = argparse.ArgumentParser(
prog="bazel_bf teardown",
description="Tear down the remote environment entirely.")
parser.add_argument("--force", action='store_true')
args = parser.parse_args(argv)
lambda_config = config.read_config()
if not args.force:
print "Configuration is: " + json.dumps(
lambda_config, indent=2, sort_keys=True)
sys.stdout.write("Confirm tearing down the remote environment? [yes/No] ")
choice = raw_input().lower()
if choice == "yes":
print "Proceeding..."
else:
raise CommandLineException("Abort!")
(next_lambda_config, err) = setup.teardown(lambda_config)
config.write_config(next_lambda_config)
if err:
raise CommandLineException(
"Errors were encountered during tear down: " +
"some resources might not have been properly deleted")
files = {'file': (batch_name, json.dumps(items), 'application/json'), 'context': context}
logging.info(f"POST: sending '{batch_name}' to {url}")
with session1.post(url, files=files, stream=True) as r:
if r.status_code >= 300:
raise Exception(f"Failed {batch_name} with status {r.status_code}:\n\n{r.json()}")
results_file_name = f'{batch_name}-results.json'
logging.info(f"Extracting '{results_file_name}' into the 'results' folder.")
parser = StreamingFormDataParser(headers=r.headers)
masked_batch = ValueTarget()
parser.register('file', masked_batch)
parser.register('results', FileTarget(f'results/{results_file_name}'))
for chunk in r.iter_content():
parser.data_received(chunk)
masked_batch = json.loads(masked_batch.value)
# The batch writer will automatically handle buffering and sending items in batches.
# In addition, the batch writer will also automatically handle any unprocessed items and resend them as needed.
with target_table.batch_writer() as batch:
for item in masked_batch:
batch.put_item(Item=item)
start_key = response.get('LastEvaluatedKey', None)
done = start_key is None
scan_kwargs['ExclusiveStartKey'] = start_key
if not done:
response = table.scan(**scan_kwargs)
items = response.get('Items', [])
finally:
teardown(session1)
filename = askopenfilename()
if not filename:
break
basename = ntpath.basename(filename)
process_files = [(filename, 'masked')]
for file_name, masked_folder in process_files:
files = {'file': open(file_name, 'rb'), 'context': context}
logging.info(f"POST: sending '{file_name}' to {url}")
with s.post(url, files=files, stream=True) as r:
if r.status_code >= 300:
logging.info(
f"Failed with status {r.status_code}:\n\n{r.json()}"
)
break
logging.info(
f"Extracting 'masked_{basename}' and 'masked_{basename}_results.json' into {masked_folder}."
)
parser = StreamingFormDataParser(headers=r.headers)
parser.register(
'file',
FileTarget(f'{masked_folder}/masked_{basename}'))
parser.register(
'results',
FileTarget(
f'{masked_folder}/masked_{basename}_results.json'))
for chunk in r.iter_content(4096):
parser.data_received(chunk)
finally:
teardown(s)
lines = args.lines
iterations = args.iterations
test_folder = 'test-files'
file_name = f'test-{lines}.csv'
file_path = f'{test_folder}/{file_name}'
os.makedirs(test_folder, exist_ok=True)
if not os.path.exists(file_path):
logging.info(f'Creating {file_name}...')
with open(file_path, 'w') as f:
test = 'this is a test'
char_num = 1000 - len(test) - len(os.linesep)
chars_per_col = 10
num_cols = int(char_num / chars_per_col)
remaining_padding = 'x' * (char_num % chars_per_col)
line = ','.join(['x' * (chars_per_col - 1) for i in range(num_cols)] + [f'{remaining_padding}this is a test'])
for i in range(lines):
f.write(line)
f.write(os.linesep)
logging.info(f'Created {file_name}.')
with requests.Session() as session:
try:
setup(session)
context = json.dumps({
"fileSearchContextName": file_search_context_name,
"fileMaskContextName": file_mask_context_name
})
utils.benchmark_search_mask(session, file_path, context,
lines, 'text/csv', iterations)
finally:
teardown(session)