def _write_hframe_local(self, hfr): """ Args: hfr: Returns: """ # Write DB HyperFrame result = hyperframe.w_pb_db(hfr, self.local_engine) # Write FS Frames for fr in hfr.get_frames(self): hyperframe.w_pb_fs( os.path.join(self.get_object_dir(), hfr.pb.uuid), fr) # Write FS HyperFrame hyperframe.w_pb_fs(os.path.join(self.get_object_dir(), hfr.pb.uuid), hfr) # Write DB Frames for fr in hfr.get_frames(self): hyperframe.w_pb_db(fr, self.local_engine) self.prune_uncommitted_history(hfr.pb.human_name) return result
def test_linkauth_rw_db(): """ Create a pb buffer write to db read from db :return: """ global engine_g """ Create table """ hyperframe.LinkAuthBase.create_table(engine_g) """ Create some PB records """ slar = _make_linkauth_records() """ Write out PBs as rows """ slar_hash = hashlib.md5(slar.pb.SerializeToString()).hexdigest() w_pb_db(slar, engine_g) """ Read in PBs as rows""" link_auth_results = r_pb_db(hyperframe.LinkAuthBase, engine_g) slar_hash2 = None for x in link_auth_results: if x.pb.WhichOneof('auth') == 's3_auth': slar_hash2 = hashlib.md5(x.pb.SerializeToString()).hexdigest() assert (slar_hash == slar_hash2)
def test_hframe_rw_db(): """ Create a pb buffer write to db read from db :return: """ global engine_g """ Create table """ hyperframe.HyperFrameRecord.create_table(engine_g) """ Create some PB records """ hf1 = _make_hframe_record('inner_record') hf2 = _make_hframe_record('outer_record', hframes=[ hf1, ]) """ Write out PBs as rows """ hf_hash = hashlib.md5(hf2.pb.SerializeToString()).hexdigest() w_pb_db(hf2, engine_g) """ Read in PBs as rows""" hf_results = r_pb_db(hyperframe.HyperFrameRecord, engine_g) hf_hash2 = None for x in hf_results: hf_hash2 = hashlib.md5(x.pb.SerializeToString()).hexdigest() assert (hf_hash == hf_hash2)
def write_hframe_db_only(self, hfr): """ Quick hack to write an HFR pb into the db from DisdatFS Args: hfr: Returns: """ hyperframe.w_pb_db(hfr, self.local_engine) # Write DB Frames for fr in hfr.get_frames(self): hyperframe.w_pb_db(fr, self.local_engine)
def atomic_update_hframe(self, hfr): """ Given an HFR that has new meta information, such as tags, update the version on disk atomically, then make an update to the data base Note: This has only been spec'd to work when we update tags. If you're making any other changes to the original HyperFrameRecord, you will need to review this code. TODO: This is not acid wrt to the database. We need to make a transaction for this update. At least try / catch the remove before we update the file. Args: hfr: Returns: result object """ # 1.) Delete DB record hyperframe.delete_hfr_db(self.local_engine, uuid=hfr.pb.uuid) # 2.) Write FS HyperFrame PB to a sister file and then move to original file. hyperframe.w_pb_fs(os.path.join(self.get_object_dir(), hfr.pb.uuid), hfr, atomic=True) # 3.) Write DB HyperFrame and tags result = hyperframe.w_pb_db(hfr, self.local_engine) return result
def test_link_rw_db(): """ Create a pb buffer write to db read from db :return: """ global engine_g """ Create table """ hyperframe.LinkBase.create_table(engine_g) """ Create some PB records """ local_link, s3_link, db_link = _make_link_records() """ Write out PBs as rows """ local_hash = w_pb_db(local_link, engine_g) s3_hash = w_pb_db(s3_link, engine_g) db_hash = w_pb_db(db_link, engine_g) """ Read in PBs as rows""" link_results = r_pb_db(hyperframe.LinkBase, engine_g) local_hash2 = None s3_hash2 = None db_hash2 = None for x in link_results: if x.pb.WhichOneof('link') == 'local': local_hash2 = hashlib.md5(local_link.pb.SerializeToString()).hexdigest() if x.pb.WhichOneof('link') == 's3': s3_hash2 = hashlib.md5(s3_link.pb.SerializeToString()).hexdigest() if x.pb.WhichOneof('link') == 'database': db_hash2 = hashlib.md5(db_link.pb.SerializeToString()).hexdigest() assert (local_hash == local_hash2) assert (s3_hash == s3_hash2) assert (db_hash == db_hash2)
def rebuild_db(self): """ For this context, read in all pb's and rebuild tables. All state is immutable. 1.) Read in all HFrame PBs 2.) Ensure that each HFrame PB is consistent -- it was completely written to disk. 3.) A.) If yes, try to insert into db if it doesn't already exist. B.) If not consistent and not in db, leave it. Could be concurrent add. C.) If not consistent and in db as 'valid', mark db entry as 'invalid' dbck does the opposite process. It will read the DB and see Returns: num errors (int): """ hframes = {} frames = {} auths = {} pb_types = [('*_hframe.pb', hyperframe.HyperFrameRecord, hframes), ('*_frame.pb', hyperframe.FrameRecord, frames), ('*_auth.pb', hyperframe.LinkAuthBase, auths)] # Make all the tables first. for glb, rcd_type, store in pb_types: rcd_type.create_table(self.local_engine) for uuid_dir in os.listdir(self.get_object_dir()): for glb, rcd_type, store in pb_types: files = glob.glob( os.path.join(os.path.join(self.get_object_dir(), uuid_dir), glb)) for f in files: # hyperframes, frames, and links all have uuid fields rcd = hyperframe.r_pb_fs(f, rcd_type) store[rcd.pb.uuid] = rcd for hfr in hframes.itervalues(): if DataContext._validate_hframe(hfr, frames, auths): # looks like a good hyperframe # print "Writing out HFR {} {}".format(hfr.pb.human_name, hfr.pb.uuid) hyperframe.w_pb_db(hfr, self.local_engine) for str_tuple in hfr.pb.frames: fr_uuid = str_tuple.v # The frame pb doesn't store the hfr_uuid, but the db # does. Since we are reading from disk, we need to # set it back into the FrameRecord. frames[fr_uuid].hframe_uuid = hfr.pb.uuid hyperframe.w_pb_db(frames[fr_uuid], self.local_engine) else: # invalid hyperframe, if present in db as valid, mark invalid # Try to read it in hfr_from_db_list = hyperframe.select_hfr_db(self.local_engine, uuid=hfr.pb.uuid) assert (len(hfr_from_db_list) == 0 or len(hfr_from_db_list) == 1) if len(hfr_from_db_list) == 1: hfr_from_db = hfr_from_db_list[0] if hfr_from_db.state == hyperframe.RecordState.valid: # If it is valid, and we know it isn't, mark invalid hyperframe.update_hfr_db( self.local_engine, hyperframe.RecordState.invalid, uuid=hfr.pb.uuid)