def render_actions(renders: list):
"""
For every render directory, we render the actions
"""
good_renders = []
bad_renders = []
for recording_name, render_path in tqdm.tqdm(renders):
if E(J(render_path, 'network.npy')):
if E(J(render_path, GOOD_MARKER_NAME)):
good_renders.append((recording_name, render_path))
else:
bad_renders.append((recording_name, render_path))
else:
try:
recording = get_recording_archive(recording_name)
def extract(fname):
return recording.extract(fname, render_path)
# Extract actions
assert str(ACTION_FILE) in [
str(x) for x in recording.namelist()
]
# Extract it if it doesnt exist
action_mcbr = extract(ACTION_FILE)
# Check that it's not-empty.
assert not os.stat(action_mcbr).st_size == 0
# Run the actual parse action and make sure that its actually of length 0.
p = subprocess.Popen([
"python3", "parse_action.py",
os.path.abspath(action_mcbr)
],
cwd='action_rendering')
returncode = (p.wait())
assert returncode == 0
good_renders.append((recording_name, render_path))
except AssertionError as e:
_, _, tb = sys.exc_info()
traceback.print_tb(tb) # Fixed format
touch(J(render_path, BAD_MARKER_NAME))
remove(J(render_path, GOOD_MARKER_NAME))
bad_renders.append((recording_name, render_path))
return good_renders, bad_renders
def render_metadata(renders: list):
"""
Unpacks the metadata of a recording and checks its validity.
"""
good_renders = []
bad_renders = []
for recording_name, render_path in tqdm.tqdm(renders):
if E(render_path):
# Check if metadata has already been extracted.
# if (E(J(render_path, GOOD_MARKER_NAME)) or
# E(J(render_path, BAD_MARKER_NAME))):
# # If it has been computed see if it is valid
# # or not.
# if E(J(render_path, GOOD_MARKER_NAME)):
# good_renders.append((recording_name, render_path))
# else:
# bad_renders.append((recording_name, render_path))
# else:
# BAH check metadata each time
if True:
try:
recording = get_recording_archive(recording_name)
def extract(fname):
return recording.extract(fname, render_path)
# If everything is good extract the metadata.
for mfile in METADATA_FILES:
assert str(mfile) in [
str(x) for x in recording.namelist()
]
if not E(J(render_path, mfile)):
extract(mfile)
# check that stream_meta_data is good
with open(J(render_path, 'metaData.json'), 'r') as f:
# print(render_path)
jbos = json.load(f)
# assert (ile["duration"] > 60000 or jbos["duration"] == 0)
assert (jbos["duration"] > 300000)
# go through and check if we got the experiments.
try:
with open(J(render_path, 'markers.json'), 'r') as f:
markers = json.load(f)
has_any_exps = False
for marker in markers:
exp_metadata = marker['value']['metadata'][
'expMetadata']
for exp in RENDER_ONLY_EXPERIMENTS:
has_any_exps = (
exp in exp_metadata) or has_any_exps
assert has_any_exps
except (KeyError, FileNotFoundError):
raise AssertionError("Couldn't open metadata json.")
# check that stream_meta_data is good
with open(J(render_path, 'stream_meta_data.json'),
'r') as f:
jbos = json.load(f)
assert jbos["has_EOF"]
assert not jbos["miss_seq_num"]
touch(J(render_path, GOOD_MARKER_NAME))
remove(J(render_path, BAD_MARKER_NAME))
good_renders.append((recording_name, render_path))
except (json.decoder.JSONDecodeError, AssertionError):
_, _, tb = sys.exc_info()
traceback.print_tb(tb) # Fixed format
# Mark that this is a bad file.
touch(J(render_path, BAD_MARKER_NAME))
remove(J(render_path, GOOD_MARKER_NAME))
bad_renders.append((recording_name, render_path))
return good_renders, bad_renders
def add(self, other):
touch(os.path.join(self.file_name, other))