def test_linkauth_rw_pb():
"""
Write LINKAUTH PBs to disk and back.
:return:
"""
slar = _make_linkauth_records()
""" Write out protocol buffers """
w_pb_fs(testdir, slar)
""" Read in protocol buffers """
r_pb_fs(os.path.join(testdir, slar.get_filename()),
hyperframe.S3LinkAuthRecord)
def test_link_rw_pb():
"""
Write LINK PBs to disk and back.
:return:
"""
file_link, s3_link = _make_link_records()
""" Write out protocol buffers """
w_pb_fs(testdir, file_link)
w_pb_fs(testdir, s3_link)
""" Read in protocol buffers """
r_pb_fs(os.path.join(testdir, file_link.get_filename()),
hyperframe.FileLinkRecord)
r_pb_fs(os.path.join(testdir, s3_link.get_filename()),
hyperframe.S3LinkRecord)
def test_hframe_rw_pb():
"""
Write HyperFrame PBs to disk and back.
:return:
"""
hf1 = _make_hframe_record('inner_record')
hf2 = _make_hframe_record('outer_record', hframes=[hf1, ])
""" Write out protocol buffers """
w_pb_fs(testdir, hf2)
for fr in hf2.get_frames(None, testing_dir=testdir):
w_pb_fs(testdir, fr)
""" Read in protocol buffers """
hf2_read = r_pb_fs(os.path.join(testdir, hf2.get_filename()), hyperframe.HyperFrameRecord)
validate_hframe_record(hf2_read)
def rebuild_db(self):
"""
For this context, read in all pb's and rebuild tables.
All state is immutable.
1.) Read in all HFrame PBs
2.) Ensure that each HFrame PB is consistent -- it was completely written
to disk.
3.) A.) If yes, try to insert into db if it doesn't already exist.
B.) If not consistent and not in db, leave it. Could be concurrent add.
C.) If not consistent and in db as 'valid', mark db entry as 'invalid'
dbck does the opposite process. It will read the DB and see
Returns:
num errors (int):
"""
hframes = {}
frames = {}
auths = {}
pb_types = [('*_hframe.pb', hyperframe.HyperFrameRecord, hframes),
('*_frame.pb', hyperframe.FrameRecord, frames),
('*_auth.pb', hyperframe.LinkAuthBase, auths)]
# Make all the tables first.
for glb, rcd_type, store in pb_types:
rcd_type.create_table(self.local_engine)
for uuid_dir in os.listdir(self.get_object_dir()):
for glb, rcd_type, store in pb_types:
files = glob.glob(
os.path.join(os.path.join(self.get_object_dir(), uuid_dir),
glb))
for f in files:
# hyperframes, frames, and links all have uuid fields
rcd = hyperframe.r_pb_fs(f, rcd_type)
store[rcd.pb.uuid] = rcd
for hfr in hframes.itervalues():
if DataContext._validate_hframe(hfr, frames, auths):
# looks like a good hyperframe
# print "Writing out HFR {} {}".format(hfr.pb.human_name, hfr.pb.uuid)
hyperframe.w_pb_db(hfr, self.local_engine)
for str_tuple in hfr.pb.frames:
fr_uuid = str_tuple.v
# The frame pb doesn't store the hfr_uuid, but the db
# does. Since we are reading from disk, we need to
# set it back into the FrameRecord.
frames[fr_uuid].hframe_uuid = hfr.pb.uuid
hyperframe.w_pb_db(frames[fr_uuid], self.local_engine)
else:
# invalid hyperframe, if present in db as valid, mark invalid
# Try to read it in
hfr_from_db_list = hyperframe.select_hfr_db(self.local_engine,
uuid=hfr.pb.uuid)
assert (len(hfr_from_db_list) == 0
or len(hfr_from_db_list) == 1)
if len(hfr_from_db_list) == 1:
hfr_from_db = hfr_from_db_list[0]
if hfr_from_db.state == hyperframe.RecordState.valid:
# If it is valid, and we know it isn't, mark invalid
hyperframe.update_hfr_db(
self.local_engine,
hyperframe.RecordState.invalid,
uuid=hfr.pb.uuid)