def generate_for_each(macro_prefix, header_guard_prefix, supported_size,
nested_loops_count, output_dir):
'''
Generate FOR_EACH macros, one per required nested level. Each for loop
will support a VA_ARGS_length up to the specified `supported_size`.
'''
lines = utils.generate_header_begin(macro_prefix, header_guard_prefix,
'for_each')
lines.append('')
# Generate #includes for CAT, DEFER, INC macros
lines.append('#include "{}arg.hpp"'.format(macro_prefix.lower()))
lines.append('#include "{}cat.hpp"'.format(macro_prefix.lower()))
lines.append('#include "{}inc.hpp"'.format(macro_prefix.lower()))
lines.append('')
# Generate multiple for each macros for nested loops
for x in range(nested_loops_count):
# Generate macro which invokes the correct iteration
lines += generate_for_each_dispatch_macro(macro_prefix, x)
lines.append('')
# Generate the ITERN macros.
for i in range(supported_size, 1, -1):
lines += generate_for_each_itern_macro(macro_prefix, x, i)
# Generate the ITER1 macro.
lines += generate_for_each_iter1_macro(macro_prefix, x)
lines.append('')
lines.append(utils.generate_header_end())
utils.write_lines(
utils.get_output_file_name(macro_prefix, 'for_each', output_dir),
lines)
def generate_cat(macro_prefix, header_prefix, supported_size, output_dir):
'''
Generate CAT macro which concatenate its two arguments.
'''
lines = utils.generate_header_begin(macro_prefix, header_prefix, 'cat')
lines.append('')
lines += utils.get_cat_lines(macro_prefix, '', supported_size)
lines.append('')
lines.append(utils.generate_header_end())
utils.write_lines(
utils.get_output_file_name(macro_prefix, 'cat', output_dir), lines)
def generate_arg(macro_prefix, header_guard_prefix, supported_size,
output_dir):
'''
Generate the ARG_LENGTH macro.
'''
# Seed lines with header guard.
lines = utils.generate_header_begin(macro_prefix, header_guard_prefix,
'arg')
lines.append('')
# Generate ARG_NTH, where N = `supported_size` + 1, giving back the Nth
# argument in the variable arguments.
args_list = ['_{}'.format(i) for i in range(1, supported_size + 1)]
args_str = ', '.join(args_list)
arg_nth = ARG_NTH_TEMPLATE.format(macro_prefix.upper(), supported_size + 1,
args_str)
lines.append(arg_nth)
lines.append('')
# Generate ARG_LENGTH for getting the length of the variable arguments.
# Works for lists from 1 to `supported_size`.
lengths_list = ['{}'.format(i) for i in range(supported_size, 0, -1)]
lengths_str = ', '.join(lengths_list)
arg_length = ARG_LENGTH_TEMPLATE.format(macro_prefix.upper(),
supported_size + 1, lengths_str)
lines.append(arg_length)
lines.append('')
# Generate macro for getting the second of the variable arguments.
lines.append(ARG_2ND_TEMPLATE.format(macro_prefix))
lines.append('')
# Generate CAT for use in ARG macros.
lines += utils.get_cat_lines(macro_prefix, 'ARG_', 2)
lines.append('')
# Generate ARG_IS_SINGLE for checking if the variable arguments is of size
# 1.
lines.append(ARG_IS_SINGLE_TEMPLATE.format(macro_prefix.upper()))
lines.append('')
# Generate ARG_SINGULAR for checking if de-paranthesized argument is
# a list of size 1.
lines.append(ARG_IS_SINGULAR_TEMPLATE.format(macro_prefix.upper()))
lines.append('')
lines.append(utils.generate_header_end())
utils.write_lines(
utils.get_output_file_name(macro_prefix, 'arg', output_dir), lines)
def generate_inc(macro_prefix, header_guard_prefix, supported_size,
output_dir):
'''
Generate INC macros which gives a number one higher than the invoked INC.
'''
lines = utils.generate_header_begin(macro_prefix, header_guard_prefix,
'inc')
lines.append('')
lines.append('// INC_N gives back N+1')
# Generate INC_N macros, which gives back N + 1.
inc_macros = [
INC_TEMPLATE.format(macro_prefix.upper(), i, i + 1)
for i in range(supported_size)
]
lines += inc_macros
lines.append('')
lines.append(utils.generate_header_end())
utils.write_lines(
utils.get_output_file_name(macro_prefix, 'inc', output_dir), lines)
def test_load_json(self):
file_name = utils.get_output_file_name()
loaded_data = utils.load_json(file_name, verbose=True)
self.assertGreater(len(loaded_data), 0)
def test_get_output_file_name(self):
file_name = utils.get_output_file_name()
self.assertGreater(len(file_name), 0)
def main(args):
# process hyperparameters
args = vars(args)
dataset = args['dataset']
dataset_path = args['dataset_path']
lm_model = args['lm']
device = args['device']
encryption_method = args["encrypt"]
use_cached_encryption_results = (encryption_method == "cached")
steganography_method = args["encode"]
precision = args["precision"]
temp = args["temp"]
topK = args["topK"]
block_size = args["block_size"]
nucleus = args["nucleus"]
delta = args["delta"]
if delta:
nucleus = 2**(-1.0 * delta)
print("Loading large LM to GPU, please wait for a few seconds...")
enc, model, device = get_model(model_name=lm_model, device_id=device)
# load plaintext dataset
if dataset != "random":
with open(f"{dataset_path}/plaintext.txt", "r") as fin:
plaintexts = [
line.strip() for line in fin.readlines() if line.strip() != ""
]
print(f"Encoding {len(plaintexts)} plaintexts")
bin2words, words2bin = get_bins(len(enc.encoder), block_size)
args["bin2words"] = bin2words
args["words2bin"] = words2bin
# encryption
print(f"Encryption Algorithm: {encryption_method}")
if use_cached_encryption_results:
print("Load existing encrypted messages")
encryption_infos = []
messages = []
with open(f"{dataset_path}/message_bits.txt", "r") as fin:
for line in fin:
line = line.strip()
if line:
messages.append(eval(line))
else:
encryption_infos = []
encryption_context = ""
messages = []
for plaintext in tqdm(plaintexts, desc="encrypting"):
message, info = plaintext2bits(plaintext, encryption_context,
model, enc, lm_model,
encryption_method, device)
messages.append(message)
encryption_infos.append(info)
with open(f"{dataset_path}/message_bits.txt", "w") as fout:
for message in messages:
fout.write(str(message))
fout.write("\n")
# steganography encoding
encoding_infos = []
encoding_context = "Washington received his initial military training and command with the Virginia Regiment during the French and Indian War. He was later elected to the Virginia House of Burgesses and was named a delegate to the Continental Congress, where he was appointed Commanding General of the nation's Continental Army. Washington led American forces, allied with France, in the defeat of the British at Yorktown. Once victory for the United States was in hand in 1783, Washington resigned his commission."
covertexts = []
print(f"Steganography Encoding Algorithm: {steganography_method}")
start = time.time()
for message in tqdm(messages[:100], desc="encoding"):
covertext, info = bits2covertext(message,
encoding_context,
model,
enc,
lm_model,
steganography_method,
device,
bin2words=bin2words,
words2bin=words2bin,
precision=precision,
temp=temp,
topK=topK,
block_size=block_size,
nucleus=nucleus)
covertexts.append(covertext)
encoding_infos.append(info)
end = time.time()
efficiency = (end - start) / 100
print(f"Use {efficiency} per example")
results = {
"encrpytion_infos": encryption_infos,
"encoding_infos": encoding_infos,
"covertexts": covertexts
}
output_name = get_output_file_name(args)
with open(output_name, "w") as fout:
json.dump(results,
fout,
indent=4,
sort_keys=True,
separators=(',', ': '))