def load_or_gen_list_dict(rec_f, rec):
cls_idx_dict_f = os.path.splitext(rec_f)[0] + '.json'
idx_cls_lst_f = os.path.splitext(rec_f)[0] + '-lst.json'
if os.path.exists(cls_idx_dict_f) and os.path.exists(idx_cls_lst_f):
idx_cls_lst = load_json(idx_cls_lst_f)
cls_idx_dict = load_json(cls_idx_dict_f)
idx_cls_lst = [int(item) for item in idx_cls_lst]
keys = list(cls_idx_dict.keys())
for k in keys:
cls_idx_dict[int(k)] = cls_idx_dict.pop(k)
return idx_cls_lst, cls_idx_dict
else:
idx_cls_lst = []
for idx in rec.idx.keys():
record = rec.read_idx(idx)
h, _ = recordio.unpack(record)
idx_cls_lst.append([idx, int(h.label)])
cls_idx_dict = {}
for idx, y in idx_cls_lst:
if y in cls_idx_dict:
cls_idx_dict[y].append(idx)
else:
cls_idx_dict[y] = [idx]
idx_cls_lst = [int(l) for _, l in idx_cls_lst]
save_json(idx_cls_lst, idx_cls_lst_f)
save_json(cls_idx_dict, cls_idx_dict_f)
return idx_cls_lst, cls_idx_dict
def test_compress_and_decode_full_report(self):
"""
Ensures that a typical, complete report is compressed properly.
"""
report = load_json(fetch_file(resource_uri("reports/DAOBug.json")))
hexstring = self.compress_report(report)
decoded_report = self.decode_report(hexstring)
expected_report = load_json(
fetch_file(resource_uri("reports/DAOBugDecompressed.json")))
self.__compare_json(decoded_report, expected_report)
def compare_json(self,
audit_file,
report_file_path,
json_loaded=False,
ignore_id=False):
if not json_loaded:
actual_json = load_json(audit_file)
else:
actual_json = audit_file
expected_json = load_json(fetch_file(resource_uri(report_file_path)))
if ignore_id:
expected_json['request_id'] = actual_json['request_id']
diff = DeepDiff(
actual_json,
expected_json,
exclude_paths={
"root['contract_uri']",
"root['version']",
# There is no keystore used for testing. Accounts
# are dynamic and therefore cannot be compared
"root['auditor']",
"root['requestor']",
# Path is different depending on whether running inside Docker
"root['timestamp']",
"root['start_time']",
"root['end_time']",
"root['analyzers_reports'][0]['analyzer']['command']",
"root['analyzers_reports'][0]['coverages'][0]['file']",
"root['analyzers_reports'][0]['potential_vulnerabilities'][0]['file']",
"root['analyzers_reports'][0]['start_time']",
"root['analyzers_reports'][0]['end_time']",
"root['analyzers_reports'][1]['analyzer']['command']",
"root['analyzers_reports'][1]['coverages'][0]['file']",
"root['analyzers_reports'][1]['potential_vulnerabilities'][0]['file']",
"root['analyzers_reports'][1]['start_time']",
"root['analyzers_reports'][1]['end_time']",
"root['analyzers_reports'][2]['analyzer']['command']",
"root['analyzers_reports'][2]['coverages'][0]['file']",
"root['analyzers_reports'][2]['potential_vulnerabilities'][0]['file']",
"root['analyzers_reports'][2]['start_time']",
"root['analyzers_reports'][2]['end_time']",
# Once scripts are either executed or skipped. The traces at position 1 differ.
"root['analyzers_reports'][0]['trace']",
"root['analyzers_reports'][1]['trace']",
"root['analyzers_reports'][2]['trace']"
})
pprint(diff)
self.assertEqual(diff, {})
self.assertEqual(ntpath.basename(actual_json['contract_uri']),
ntpath.basename(expected_json['contract_uri']))
def main(args):
if args.delete:
ans = raw_input("Are you sure (y/[n])? ")
if ans == 'y':
print es.delete_index(args.index)
else:
print "Aborted."
return
page_mapping = io.load_json(config.ELASTICSEARCH_PAGE_MAPPING_FILE)
sentence_mapping = io.load_json(config.ELASTICSEARCH_SENTENCE_MAPPING_FILE)
print es.create_index(args.index)
print es.put_page_mapping(page_mapping)
print es.put_sentence_mapping(sentence_mapping)
def load_or_gen_dict(rec_f, rec):
cls_idx_dict_f = os.path.splitext(rec_f)[0] + '.json'
if os.path.exists(cls_idx_dict_f):
cls_idx_dict = load_json(cls_idx_dict_f)
keys = list(cls_idx_dict.keys())
for k in keys:
cls_idx_dict[int(k)] = cls_idx_dict.pop(k)
return cls_idx_dict
else:
idx_cls_lst = []
for idx in rec.idx.keys():
record = rec.read_idx(idx)
h, _ = recordio.unpack(record)
idx_cls_lst.append([idx, h.label])
cls_idx_dict = {}
for idx, y in idx_cls_lst:
y = int(y)
if y in cls_idx_dict:
cls_idx_dict[y].append(idx)
else:
cls_idx_dict[y] = [idx]
save_json(cls_idx_dict, os.path.splitext(rec_f)[0] + '.json')
return cls_idx_dict
def from_json(cls, json_file):
try:
w2i = cls(io.load_json(json_file))
except json.decoder.JSONDecodeError:
words = io.load_txt(json_file)
w2i = cls({w: i for i, w in enumerate(words)})
return w2i
def create_visda_datasets(self, train_tforms, eval_tforms):
"""
Create visda datasets
:param train_tforms: training transformers
:param eval_tforms: evaluation transformers
:return:
trs_set: training source set
trt_set: training target set
tes_set: testing source set
tet_set: testing target set
"""
# Read config
cfg = load_json(self.args.cfg)
trs = cfg['SRC']['TRAIN']
trt = cfg['TGT']['TRAIN']
tes = cfg['SRC']['TRAIN']
tet = cfg['TGT']['TEST']
self.label_dict = cfg['Label']
trs_set = DomainRecDataset(trs,
trt,
tforms=train_tforms,
tformt=train_tforms,
ratio=self.args.ratio)
tes_set = DomainRecDataset(tes, tforms=eval_tforms)
tet_set = DomainRecDataset(tet, tforms=eval_tforms)
return trs_set, tes_set, tet_set
def load_training_state(self, load_dir):
state = load_json(os.path.join(load_dir, "training_state.json"))
self.steps_since_lr_change = state["steps_since_lr_change"]
self.best_self_model_step = state["best_self_model_step"]
self.best_rule_model_step = state["best_rule_model_step"]
self.lr_tracker = state["learning_rate"]
self.current_step = state["current_step"]
def create_office_datasets(self, train_tforms, eval_tforms):
"""
Create office datasets
:param train_tforms: training transformers
:param eval_tforms: evaluation transformers
:return:
trs_set: training source set
trt_set: training target set
tes_set: testing source set
tet_set: testing target set
"""
cfg = load_json(self.args.cfg)
cfg = split_office_train_test(cfg, 1, self.args.seed)
trs = cfg[self.args.src.upper()]['SRC-TR']
trt = cfg[self.args.tgt.upper()]['TGT-TR']
tes = cfg[self.args.src.upper()]['TGT-TE']
tet = cfg[self.args.tgt.upper()]['TGT-TE']
trs_set = DomainFolderDataset(trs,
trt,
tforms=train_tforms,
tformt=train_tforms,
ratio=self.args.ratio)
tes_set = DomainFolderDataset(tes, tforms=eval_tforms)
tet_set = DomainFolderDataset(tet, tforms=eval_tforms)
return trs_set, tes_set, tet_set
def main():
params = load_json(INPUT_SETTINGS)
params[1] = params["1"]
params[2] = params["2"]
start_time_1 = time.time()
gameplay = get_gameplay(params)
play_game(gameplay, params)
start_time_2 = time.time()
wins = []
for _ in tqdm(range(N_GAMES)):
winner = play_game(gameplay, params)
wins.append(winner)
end_time = time.time()
time_elapsed_1 = end_time - start_time_1
time_elapsed_2 = end_time - start_time_2
compilation_time = time_elapsed_1 - time_elapsed_2
time_per_game = time_elapsed_2 / float(N_GAMES)
print(f"Time taken for {N_GAMES} games with compilation time: {time_elapsed_1}")
print(f"Time taken for {N_GAMES} games without compilation time: {time_elapsed_2}")
print(f"Compilation time: {compilation_time}")
print(f"Seconds per game: {time_per_game}")
wins = np.array(wins)
p1_win_rate = np.sum(wins == 1) / float(N_GAMES)
p2_win_rate = np.sum(wins == 2) / float(N_GAMES)
print(f"Player 1 win rate: {p1_win_rate}, player 2 win rate: {p2_win_rate}")
def get_answers_from_json(cls, json_file, extra_param):
question_json = io.load_json(json_file)
answers = []
for question in question_json['questions']:
if len(question['answer'].split()) > 1:
raise Exception('Answer is not a single word')
answers.append(question['answer'])
return answers
def _init_sentences(self, global_hook=None):
# sentences 最后存储着{'1': 对象, '2': 对象, '3': 对象} 数据最后都在这个字典里了
sentences = {}
sentences_cfg = load_json(self.domain_cfg_path)
for cfg in sentences_cfg.items():
s_id = cfg[0]
sentences[s_id] = Sentence(self.domain_name, cfg, global_hook)
return sentences
def test_encode_decode_idempotence(self):
"""
Ensures that encode(decode(report)) == encode(decode(encode(decode(report))))
"""
report = load_json(fetch_file(resource_uri("reports/DAOBug.json")))
decoded_report = self.decode_report(self.compress_report(report))
twice_decoded_report = self.decode_report(
self.compress_report(decoded_report))
self.__compare_json(decoded_report, twice_decoded_report)
def load_office_cfg(self):
cfg = load_json(self.args.cfg)
cfg = split_office_train_test(cfg, 1, self.args.seed)
trs = cfg[self.args.src.upper()]['SRC-TR']
trt = cfg[self.args.tgt.upper()]['TGT-TR']
tes = cfg[self.args.src.upper()]['TGT-TE']
tet = cfg[self.args.tgt.upper()]['TGT-TE']
return trs, trt, tes, tet
def load_visda_cfg(self):
cfg = load_json(self.args.cfg)
trs = cfg['SRC']['TRAIN']
trt = cfg['TGT']['TRAIN']
tes = cfg['SRC']['TRAIN']
tet = cfg['TGT']['TEST']
self.label_dict = cfg['Label']
return trs, trt, tes, tet
def load_digits_cfg(self):
cfg = load_json(self.args.cfg)
cfg = split_digits_train_test(cfg, self.args.src.upper(),
self.args.tgt.upper(), 1, self.args.seed)
trs = cfg[self.args.src.upper()]['TR']
trt = cfg[self.args.tgt.upper()]['TR']
tes = cfg[self.args.src.upper()]['TE']
tet = cfg[self.args.tgt.upper()]['TE']
return trs, trt, tes, tet
def create_contract(self, web3_client, contract_abi_uri, contract_address):
"""
Creates the audit contract from ABI.
"""
abi_file = io_utils.fetch_file(contract_abi_uri)
abi_json = io_utils.load_json(abi_file)
return web3_client.eth.contract(
address=contract_address,
abi=abi_json,
)
def execute(self):
self.game_params = load_json(INPUT_SETTINGS)
self.game_params[1] = self.game_params["1"]
self.game_params[2] = self.game_params["2"]
self.controller = Controller(self.game_params)
self.controller.run()
while self.show_screen:
for event in pg.event.get():
if event.type == pg.QUIT:
self.show_screen = False
self.cleanup()
def __assert_all_analyzers(self, request_id):
"""
Asserts that all configured analyzers were executed and are included in the report.
"""
row = self.__fetch_audit_from_db(request_id)
audit_uri = row['audit_uri']
audit_file = fetch_file(audit_uri)
actual_json = load_json(audit_file)
executed_analyzers = [x['analyzer']['name'] for x in actual_json['analyzers_reports']]
for analyzer in self.__config.analyzers_config:
name, conf = list(analyzer.items())[0]
self.assertTrue(name in executed_analyzers)
def generate_csv_per_scene_twostep(src_scene):
print(src_scene)
fov_h, fov_v = set_vis_fov(radius)
cached_csv_lines_list = []
for i in range(len(label_list)):
cached_csv_lines = [["depth_path", "vismap_twostep", "label"]]
cached_csv_lines_list.append(cached_csv_lines)
src_scene_label_path = os.path.join(src_dataset_folder, src_scene, "motionlabeltwostep.json")
if os.path.isfile(src_scene_label_path):
skip_csv = True
csv_file_path_lists = []
for i in range(len(label_list)):
csv_file_path = os.path.join(dst_dataset_folder, "{}_{}.csv".format(src_scene, label_list[i]))
print(csv_file_path)
csv_file_path_lists.append(csv_file_path)
skip_csv = skip_csv and os.path.exists(csv_file_path)
if skip_csv and (not enforce_regeneration):
print("csv files of scene {} exists. Skip".format(src_scene))
else:
print("produce csv files of scene {}".format(src_scene))
src_scene_depth_path = os.path.join(src_dataset_folder, src_scene, "depth")
src_scene_vismap_path = os.path.join(src_dataset_folder, src_scene, "{}_{}".format(config["visibility_folder"], str(int(radius*100))))
# load the json data
label_raw_data = io.load_json(src_scene_label_path)
radius_key = str(radius)
# iterate over each pose
for pose_id_key in label_raw_data[radius_key]:
depth_path = os.path.join(src_scene_depth_path, "depth{}.png".format(pose_id_key))
# iterate over each status
for status_key in label_raw_data[radius_key][pose_id_key]:
status = float(status_key)
selected_label_list = label_raw_data[radius_key][pose_id_key][status_key]["selected_label"]
# iterate over each label for each combinations of neighbouring poses achieving the status
for i, label in enumerate(selected_label_list):
cached_csv_line = []
cached_csv_line.append(depth_path)
vis_map_prefix = os.path.join(src_scene_vismap_path, "pose_{}_recons_{:d}_H{:d}_V{:d}".format(pose_id_key,int(status * 100), int(
round(fov_h / np.pi * 180)), int(
round(fov_v / np.pi * 180))))
vis_map_path = os.path.join(src_scene_vismap_path, "{}_comb_{:d}.png".format(vis_map_prefix, i))
cached_csv_line.append(vis_map_path)
cached_csv_line.append(label)
cached_csv_lines_list[int(label)].append(cached_csv_line)
for i in range(len(label_list)):
c_io.save_csv(csv_file_path_lists[i], cached_csv_lines_list[i])
else:
print("motion label is not available yet")
def __init__(self, image_root_path, question_json_path, question_vocab_path, answer_vocab_path,
transform, target_transform=None, data_nums='all', question_per_image=None):
super(ClevrDataset, self).__init__()
self.image_root_path = image_root_path
if isinstance(question_json_path, str):
self.question_json = io.load_json(question_json_path)
else:
self.question_json = question_json_path
self.dataset_info = self.question_json['info']
self.question_vocab = Vocab.from_json(question_vocab_path)
self.answer_vocab = Vocab.from_json(answer_vocab_path)
self.transform = transform
self.target_transform = target_transform
self.annotations = ClevrDataset.build_annotation(self.question_json, data_nums, question_per_image)
def load_or_gen_list(rec_f, rec):
idx_cls_lst_f = os.path.splitext(rec_f)[0] + '-lst.json'
if os.path.exists(idx_cls_lst_f):
idx_cls_lst = load_json(idx_cls_lst_f)
idx_cls_lst = [int(l) for l in idx_cls_lst]
return idx_cls_lst
else:
idx_cls_lst = []
for idx in rec.idx.keys():
record = rec.read_idx(idx)
h, _ = recordio.unpack(record)
idx_cls_lst.append(int(h.label))
save_json(idx_cls_lst, idx_cls_lst_f)
return idx_cls_lst
def test_load_json_for_existent_file(self):
json_obj = {
'key1': 123,
'key2': 456,
}
fd, tmp = mkstemp(text=True)
with os.fdopen(fd, "w") as json_file:
json.dump(json_obj, json_file)
json_file.close()
found = load_json(tmp)
expected = json_obj
self.assertEqual(found, expected)
def generate_csv_per_scene(src_scene):
print(src_scene)
cached_csv_lines_list = []
for i in range(len(label_list)):
cached_csv_lines = [["depth_path", "label"]]
cached_csv_lines_list.append(cached_csv_lines)
src_scene_label_path = os.path.join(src_dataset_folder, src_scene,
"motionlabeltwostep.json")
if os.path.isfile(src_scene_label_path):
skip_csv = True
csv_file_path_lists = []
for i in range(len(label_list)):
csv_file_path = os.path.join(
dst_dataset_folder, "{}_{}.csv".format(src_scene,
label_list[i]))
csv_file_path_lists.append(csv_file_path)
skip_csv = skip_csv and os.path.exists(csv_file_path)
if skip_csv and (not enforce_regeneration):
print("csv files of scene {} exists. Skip".format(src_scene))
else:
print("produce csv files of scene {}".format(src_scene))
src_scene_depth_path = os.path.join(src_dataset_folder, src_scene,
"depth")
src_scene_label_path = os.path.join(src_dataset_folder, src_scene,
"motionlabeltwostep.json")
# load the json data
label_raw_data = io.load_json(src_scene_label_path)
neigh_range_key = str(neigh_range)
# for each item in label
# data_result[radius_key][pose_id_key][status_key]["selected_label"]
for pose_id_key in label_raw_data[neigh_range_key]:
depth_path = os.path.join(src_scene_depth_path,
"depth{}.png".format(pose_id_key))
selected_label_list = label_raw_data[neigh_range_key][
pose_id_key]["0"]["selected_label"]
for i, label in enumerate(selected_label_list):
cached_csv_line = []
cached_csv_line.append(depth_path)
cached_csv_line.append(label)
cached_csv_lines_list[int(label)].append(cached_csv_line)
for i in range(len(label_list)):
c_io.save_csv(csv_file_path_lists[i], cached_csv_lines_list[i])
else:
print("motion label is not available yet")
def analyse_data_list_coverage(rooms, methods, dome_data_result,
vox_num_option):
coverage_array_dict = {}
coverage_std_array_dict = {}
mean_coverage_array_dict = {}
std_coverage_array_dict = {}
for room in rooms:
file_path = os.path.join(
proj_path, result_folder,
"{}_{}_{}.json".format(dataset_name, room, vox_num_option))
raw_data = io_local.load_json(file_path)
coverage_array_dict_room = {}
for start_index in raw_data:
for method in raw_data[start_index]:
temp_array = np.asarray(
raw_data[start_index][method]["volume_size"]) / float(
dome_data_result[dataset_name][room])
temp_array[temp_array > 1] = 1.0
if method in coverage_array_dict:
coverage_array_dict[method] = np.vstack(
(coverage_array_dict[method], temp_array))
else:
coverage_array_dict[method] = temp_array
if method in coverage_array_dict_room:
coverage_array_dict_room[method] = np.vstack(
(coverage_array_dict_room[method], temp_array))
else:
coverage_array_dict_room[method] = temp_array
# compute the std for each room, the final one will be averaged by rooms
for method in methods:
temp_std_array = np.std(coverage_array_dict_room[method], axis=0)
if method in coverage_std_array_dict:
coverage_std_array_dict[method] = np.vstack(
(coverage_std_array_dict[method], temp_std_array))
else:
coverage_std_array_dict[method] = temp_std_array
for method in methods:
mean_coverage_array_dict[method] = np.mean(coverage_array_dict[method],
axis=0)
std_coverage_array_dict[method] = np.mean(
coverage_std_array_dict[method], axis=0)
return mean_coverage_array_dict, std_coverage_array_dict
def test_create_set_from_compressed_report(self):
# Tests whether vulnerability sets for compressed reports match those from
# their corresponding uncompressed ones.
for report in os.listdir(fetch_file(resource_uri("reports/"))):
uncompressed_report = load_json(
fetch_file(resource_uri("reports/DAOBug.json")))
expected_set = VulnerabilitiesSet.from_uncompressed_report(
uncompressed_report)
request_id = uncompressed_report['request_id']
encoder = ReportEncoder()
compressed_report = encoder.compress_report(
uncompressed_report, request_id)
decompressed_report = encoder.decode_report(
compressed_report, request_id)
found_set = VulnerabilitiesSet.from_uncompressed_report(
decompressed_report)
self.assertEquals(expected_set, found_set)
def fetch_config(inject_contract=False):
# create config from file, the contract is not provided and will be injected separately
config_file_uri = resource_uri("test_config.yaml")
config = ConfigFactory.create_from_file(config_file_uri,
os.getenv("QSP_ENV",
default="dev"),
validate_contract_settings=False)
if inject_contract:
contract_source_uri = "./tests/resources/QuantstampAuditMock.sol"
contract_metadata_uri = "./tests/resources/QuantstampAudit-metadata.json"
audit_contract_metadata = load_json(fetch_file(contract_metadata_uri))
audit_contract_name = get(audit_contract_metadata, '/contractName')
addr, contract = __load_audit_contract_from_src(
config.web3_client, contract_source_uri, audit_contract_name,
config.account)
config._Config__audit_contract_address = addr
config._Config__audit_contract = contract
config_utils = ConfigUtils(config.node_version)
config_utils.check_configuration_settings(config)
return config
def analyse_data_list_computation(rooms, methods, key, vox_num_option):
computation_array_dict = {}
mean_computation_array_dict = {}
std_computation_array_dict = {}
# data_result[start_index_key][method]["volume_size"] = volume_sizes
for room in rooms:
file_path = os.path.join(
proj_path, result_folder,
"{}_{}_{}.json".format(dataset_name, room, vox_num_option))
raw_data = io_local.load_json(file_path)
for start_index in raw_data:
for method in raw_data[start_index]:
temp_array = np.asarray(raw_data[start_index][method][key])
if method in computation_array_dict:
computation_array_dict[method] = np.vstack(
(computation_array_dict[method], temp_array))
else:
computation_array_dict[method] = temp_array
for method in methods:
mean_computation_array_dict[method] = np.mean(
computation_array_dict[method], axis=0)
std_computation_array_dict[method] = np.std(
computation_array_dict[method], axis=0)
return mean_computation_array_dict, std_computation_array_dict
def parse_cfg(self):
return collections.OrderedDict(
load_json(os.path.join(self.stat_cfg_path, 'stat.json')))
def __load_report(self, report_file_path):
return load_json(fetch_file(resource_uri(report_file_path)))
