def _write_hframe_local(self, hfr):
"""
Args:
hfr:
Returns:
"""
# Write DB HyperFrame
result = hyperframe.w_pb_db(hfr, self.local_engine)
# Write FS Frames
for fr in hfr.get_frames(self):
hyperframe.w_pb_fs(
os.path.join(self.get_object_dir(), hfr.pb.uuid), fr)
# Write FS HyperFrame
hyperframe.w_pb_fs(os.path.join(self.get_object_dir(), hfr.pb.uuid),
hfr)
# Write DB Frames
for fr in hfr.get_frames(self):
hyperframe.w_pb_db(fr, self.local_engine)
self.prune_uncommitted_history(hfr.pb.human_name)
return result
def test_linkauth_rw_db():
"""
Create a pb buffer
write to db
read from db
:return:
"""
global engine_g
""" Create table """
hyperframe.LinkAuthBase.create_table(engine_g)
""" Create some PB records """
slar = _make_linkauth_records()
""" Write out PBs as rows """
slar_hash = hashlib.md5(slar.pb.SerializeToString()).hexdigest()
w_pb_db(slar, engine_g)
""" Read in PBs as rows"""
link_auth_results = r_pb_db(hyperframe.LinkAuthBase, engine_g)
slar_hash2 = None
for x in link_auth_results:
if x.pb.WhichOneof('auth') == 's3_auth':
slar_hash2 = hashlib.md5(x.pb.SerializeToString()).hexdigest()
assert (slar_hash == slar_hash2)
def test_hframe_rw_db():
"""
Create a pb buffer
write to db
read from db
:return:
"""
global engine_g
""" Create table """
hyperframe.HyperFrameRecord.create_table(engine_g)
""" Create some PB records """
hf1 = _make_hframe_record('inner_record')
hf2 = _make_hframe_record('outer_record', hframes=[
hf1,
])
""" Write out PBs as rows """
hf_hash = hashlib.md5(hf2.pb.SerializeToString()).hexdigest()
w_pb_db(hf2, engine_g)
""" Read in PBs as rows"""
hf_results = r_pb_db(hyperframe.HyperFrameRecord, engine_g)
hf_hash2 = None
for x in hf_results:
hf_hash2 = hashlib.md5(x.pb.SerializeToString()).hexdigest()
assert (hf_hash == hf_hash2)
def write_hframe_db_only(self, hfr):
"""
Quick hack to write an HFR pb into the db from DisdatFS
Args:
hfr:
Returns:
"""
hyperframe.w_pb_db(hfr, self.local_engine)
# Write DB Frames
for fr in hfr.get_frames(self):
hyperframe.w_pb_db(fr, self.local_engine)
def atomic_update_hframe(self, hfr):
"""
Given an HFR that has new meta information, such as tags, update the version on disk atomically,
then make an update to the data base
Note: This has only been spec'd to work when we update tags. If you're making any other changes to the
original HyperFrameRecord, you will need to review this code.
TODO: This is not acid wrt to the database. We need to make a transaction for this update.
At least try / catch the remove before we update the file.
Args:
hfr:
Returns:
result object
"""
# 1.) Delete DB record
hyperframe.delete_hfr_db(self.local_engine, uuid=hfr.pb.uuid)
# 2.) Write FS HyperFrame PB to a sister file and then move to original file.
hyperframe.w_pb_fs(os.path.join(self.get_object_dir(), hfr.pb.uuid),
hfr,
atomic=True)
# 3.) Write DB HyperFrame and tags
result = hyperframe.w_pb_db(hfr, self.local_engine)
return result
def test_link_rw_db():
"""
Create a pb buffer
write to db
read from db
:return:
"""
global engine_g
""" Create table """
hyperframe.LinkBase.create_table(engine_g)
""" Create some PB records """
local_link, s3_link, db_link = _make_link_records()
""" Write out PBs as rows """
local_hash = w_pb_db(local_link, engine_g)
s3_hash = w_pb_db(s3_link, engine_g)
db_hash = w_pb_db(db_link, engine_g)
""" Read in PBs as rows"""
link_results = r_pb_db(hyperframe.LinkBase, engine_g)
local_hash2 = None
s3_hash2 = None
db_hash2 = None
for x in link_results:
if x.pb.WhichOneof('link') == 'local':
local_hash2 = hashlib.md5(local_link.pb.SerializeToString()).hexdigest()
if x.pb.WhichOneof('link') == 's3':
s3_hash2 = hashlib.md5(s3_link.pb.SerializeToString()).hexdigest()
if x.pb.WhichOneof('link') == 'database':
db_hash2 = hashlib.md5(db_link.pb.SerializeToString()).hexdigest()
assert (local_hash == local_hash2)
assert (s3_hash == s3_hash2)
assert (db_hash == db_hash2)
def rebuild_db(self):
"""
For this context, read in all pb's and rebuild tables.
All state is immutable.
1.) Read in all HFrame PBs
2.) Ensure that each HFrame PB is consistent -- it was completely written
to disk.
3.) A.) If yes, try to insert into db if it doesn't already exist.
B.) If not consistent and not in db, leave it. Could be concurrent add.
C.) If not consistent and in db as 'valid', mark db entry as 'invalid'
dbck does the opposite process. It will read the DB and see
Returns:
num errors (int):
"""
hframes = {}
frames = {}
auths = {}
pb_types = [('*_hframe.pb', hyperframe.HyperFrameRecord, hframes),
('*_frame.pb', hyperframe.FrameRecord, frames),
('*_auth.pb', hyperframe.LinkAuthBase, auths)]
# Make all the tables first.
for glb, rcd_type, store in pb_types:
rcd_type.create_table(self.local_engine)
for uuid_dir in os.listdir(self.get_object_dir()):
for glb, rcd_type, store in pb_types:
files = glob.glob(
os.path.join(os.path.join(self.get_object_dir(), uuid_dir),
glb))
for f in files:
# hyperframes, frames, and links all have uuid fields
rcd = hyperframe.r_pb_fs(f, rcd_type)
store[rcd.pb.uuid] = rcd
for hfr in hframes.itervalues():
if DataContext._validate_hframe(hfr, frames, auths):
# looks like a good hyperframe
# print "Writing out HFR {} {}".format(hfr.pb.human_name, hfr.pb.uuid)
hyperframe.w_pb_db(hfr, self.local_engine)
for str_tuple in hfr.pb.frames:
fr_uuid = str_tuple.v
# The frame pb doesn't store the hfr_uuid, but the db
# does. Since we are reading from disk, we need to
# set it back into the FrameRecord.
frames[fr_uuid].hframe_uuid = hfr.pb.uuid
hyperframe.w_pb_db(frames[fr_uuid], self.local_engine)
else:
# invalid hyperframe, if present in db as valid, mark invalid
# Try to read it in
hfr_from_db_list = hyperframe.select_hfr_db(self.local_engine,
uuid=hfr.pb.uuid)
assert (len(hfr_from_db_list) == 0
or len(hfr_from_db_list) == 1)
if len(hfr_from_db_list) == 1:
hfr_from_db = hfr_from_db_list[0]
if hfr_from_db.state == hyperframe.RecordState.valid:
# If it is valid, and we know it isn't, mark invalid
hyperframe.update_hfr_db(
self.local_engine,
hyperframe.RecordState.invalid,
uuid=hfr.pb.uuid)
