def state_from_tr_bson(cls, doc): r""" >>> from common.typed_uuids import DatasetUUID, MessageUUID, PeerUUID >>> # No optional arguments >>> Transaction.state_from_tr_bson({ ... 'src': UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'), ... 'state': {'dataset_uuid': ... UUID('cf9a54b1-1239-48de-9b25-d7fa927db125')}, ... 'uuid': UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'), ... 'dst': UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'), ... 'type': 'BACKUP', ... 'ts': datetime(2012, 9, 26, 14, 29, 48, 877434) ... }) # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE BackupTransactionState_Node(tr_start_time=....datetime(2012, 9, 26, 14, 29, 48, 877434), tr_uuid=MessageUUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'), tr_src_uuid=PeerUUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'), tr_dst_uuid=PeerUUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'), dataset_uuid=DatasetUUID('cf9a54b1-1239-48de-9b25-d7fa927db125')) >>> # All optional arguments >>> Transaction.state_from_tr_bson({ ... 'src': UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'), ... 'state': {'dataset_uuid': ... UUID('cf9a54b1-1239-48de-9b25-d7fa927db125')}, ... 'uuid': UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'), ... 'parent': UUID('a096cf37-dc4f-4d0e-9488-da50443c8c40'), ... 'dst': UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'), ... 'wake_up_ts': datetime(2012, 10, 3, 12, 31, 8, 279807), ... 'type': 'BACKUP', ... 'ts': datetime(2012, 9, 26, 14, 29, 48, 877434) ... }) # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE BackupTransactionState_Node(tr_start_time=....datetime(2012, 9, 26, 14, 29, 48, 877434), tr_uuid=MessageUUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'), tr_src_uuid=PeerUUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'), tr_dst_uuid=PeerUUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'), dataset_uuid=DatasetUUID('cf9a54b1-1239-48de-9b25-d7fa927db125')) @type: NodeTransactionState """ assert cls.validate_schema(doc), repr(doc) state_class = \ transaction_states.TRANSACTION_STATE_CLASS_BY_NAME[doc['type']] assert state_class.validate_schema(doc['state']), repr(doc) return state_class.from_bson(doc['state'])( tr_start_time=doc['ts'], tr_uuid=TransactionUUID.safe_cast_uuid(doc['uuid']), tr_src_uuid=PeerUUID.safe_cast_uuid(doc['src']), tr_dst_uuid=PeerUUID.safe_cast_uuid(doc['dst']))
def _prepare_chunk_maps_from_docstore(chunk_map): r""" Convert a chunk map (like, in C{self.chunks_to_replicate}) or in C{self.chunks_to_restore} from the form stored in the FastDB to the proper classes. >>> from uuid import UUID >>> from bson.binary import Binary >>> cls = ReceiveChunksTransactionState_Node >>> cls._prepare_chunk_maps_from_docstore({ ... 'e96a073b3cd049a6b14a1fb04c221a9c': ... [{'crc32': 710928501, 'maxsize_code': 1, ... 'hash': Binary('abcdabcd' * 8, 0), ... 'uuid':UUID('5b237ceb-300d-4c88-b4c0-6331cb14b5b4'), ... 'size': 73819}], ... '233ad9c2268f4506ab0f4c71461c5d88': ... [{'crc32': 134052443, 'maxsize_code': 1, ... 'hash': Binary('abcdefgh' * 8, 0), ... 'uuid':UUID('5b237ceb-300d-4c88-b4c0-6331cb14b5b4'), ... 'size': 2097152}, ... {'crc32': 2120017837, 'maxsize_code': 0, ... 'hash': Binary('01234567' * 8, 0), ... 'uuid':UUID('940f0711-52d7-42fb-bf4c-818580f432dc'), ... 'size': 143941}, ... {'crc32': 3704113112, 'maxsize_code': 1, ... 'hash': Binary('76543210' * 8, 0), ... 'uuid':UUID('a5b605f2-6ea5-49f3-8658-d217b7e8e784'), ... 'size': 2097151}] ... }) # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE {PeerUUID('e96a073b-3cd0-49a6-b14a-1fb04c221a9c'): [models.types.ChunkInfo(uuid=ChunkUUID('5b237ceb-...331cb14b5b4'), maxsize_code=1, hash=unhexlify('616263646...36461626364'), size=73819, crc32=0x2A5FE875)], PeerUUID('233ad9c2-268f-4506-ab0f-4c71461c5d88'): [models.types.ChunkInfo(uuid=ChunkUUID('5b237ceb-...331cb14b5b4'), maxsize_code=1, hash=unhexlify('616263646...36465666768'), size=2097152, crc32=0x07FD7A5B), models.types.ChunkInfo(uuid=ChunkUUID('940f0711-...18580f432dc'), maxsize_code=0, hash=unhexlify('303132333...23334353637'), size=143941, crc32=0x7E5CE7AD), models.types.ChunkInfo(uuid=ChunkUUID('a5b605f2-...217b7e8e784'), maxsize_code=1, hash=unhexlify('373635343...53433323130'), size=2097151, crc32=0xDCC847D8)]} @type chunk_map: dict @rtype: dict """ return {PeerUUID(host_uuid_str): [model_ChunkInfo.from_bson(chunk_doc)() for chunk_doc in chunk_docs] for host_uuid_str, chunk_docs in chunk_map.iteritems()}
def from_bson(cls, doc): r""" >>> from common.typed_uuids import DatasetUUID, MessageUUID, PeerUUID >>> # No optional arguments >>> Transaction.from_bson({ ... 'src': UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'), ... 'state': {'dataset_uuid': ... UUID('cf9a54b1-1239-48de-9b25-d7fa927db125')}, ... 'uuid': UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'), ... 'dst': UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'), ... 'type': 'BACKUP', ... 'ts': datetime(2012, 9, 26, 14, 29, 48, 877434) ... })() # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE Transaction(_id=None, type_='BACKUP', uuid=MessageUUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'), src=PeerUUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'), dst=PeerUUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'), ts=datetime.datetime(2012, 9, 26, 14, 29, 48, 877434), state=BackupTransactionState_Node(tr_start_time=....datetime(2012, 9, 26, 14, 29, 48, 877434), tr_uuid=MessageUUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'), tr_src_uuid=PeerUUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'), tr_dst_uuid=PeerUUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'), dataset_uuid=DatasetUUID('cf9a54b1-...-d7fa927db125'))) >>> # All optional arguments >>> Transaction.from_bson({ ... 'src': UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'), ... 'state': {'dataset_uuid': ... UUID('cf9a54b1-1239-48de-9b25-d7fa927db125')}, ... 'uuid': UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'), ... 'parent': UUID('a096cf37-dc4f-4d0e-9488-da50443c8c40'), ... 'dst': UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'), ... 'wake_up_ts': datetime(2012, 10, 3, 12, 31, 8, 279807), ... 'type': 'BACKUP', ... 'ts': datetime(2012, 9, 26, 14, 29, 48, 877434) ... })() # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE Transaction(_id=None, type_='BACKUP', uuid=MessageUUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'), src=PeerUUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'), dst=PeerUUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'), ts=datetime.datetime(2012, 9, 26, 14, 29, 48, 877434), state=Backup...State_Node(tr_start_time=....datetime(2012, 9, 26, 14, 29, 48, 877434), tr_uuid=MessageUUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'), tr_src_uuid=PeerUUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'), tr_dst_uuid=PeerUUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'), dataset_uuid=DatasetUUID('cf9a54b1-1239-...-d7fa927db125')), parent=UUID('a096cf37-dc4f-4d0e-9488-da50443c8c40'), wake_up_ts=datetime.datetime(2012, 10, 3, 12, 31, 8, 279807)) @rtype: Transaction """ assert cls.validate_schema(doc), repr(doc) state = cls.state_from_tr_bson(doc) return partial( super(Transaction, cls).from_bson(doc), type_=state.tr_type, uuid=TransactionUUID.safe_cast_uuid(state.tr_uuid), src=PeerUUID.safe_cast_uuid(state.tr_src_uuid), dst=PeerUUID.safe_cast_uuid(state.tr_dst_uuid), ts=state.tr_start_time, state=state, # Optional parent=doc.get('parent'), wake_up_ts=doc.get('wake_up_ts'))
def from_bson(cls, doc): r""" >>> from datetime import datetime >>> from uuid import UUID >>> from bson.binary import Binary >>> tr_start_time = datetime(2012, 9, 26, 14, 29, 48, 877434) >>> tr_uuid = UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2') >>> tr_src_uuid = UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24') >>> tr_dst_uuid=UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d') >>> SendChunksTransactionState_Node.from_bson({ ... 'chunks_map': { ... '233ad9c2268f4506ab0f4c71461c5d88': ... {'chunks': ... [{'crc32': 710928501, 'maxsize_code': 1, ... 'hash': Binary('abcdabcd' * 8, 0), ... 'uuid': ... UUID('0a7064b3-bef6-45c0-9e82-e9f9a40dfcf3'), ... 'size': 73819}], ... 'urls': ['https://192.168.1.2:1234', ... 'https://127.0.0.1:1234']}, ... 'e96a073b3cd049a6b14a1fb04c221a9c': ... {'chunks': ... [{'crc32': 134052443, 'maxsize_code': 1, ... 'hash': Binary('abcdefgh' * 8, 0), ... 'uuid': ... UUID('5b237ceb-300d-4c88-b4c0-6331cb14b5b4'), ... 'size': 2097152}, ... {'crc32': 2120017837, 'maxsize_code': 0, ... 'hash': Binary('01234567' * 8, 0), ... 'uuid': ... UUID('940f0711-52d7-42fb-bf4c-818580f432dc'), ... 'size': 143941}, ... {'crc32': 3704113112, 'maxsize_code': 1, ... 'hash': Binary('76543210' * 8, 0), ... 'uuid': ... UUID('a5b605f2-6ea5-49f3-8658-d217b7e8e784'), ... 'size': 2097151}], ... 'urls': ['https://192.168.2.3:4242']} ... } ... })(tr_start_time=tr_start_time, ... tr_uuid=tr_uuid, ... tr_src_uuid=tr_src_uuid, ... tr_dst_uuid=tr_dst_uuid ... ) # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE SendCh...State_Node(chunks_map={Host(uuid=PeerUUID('e96a073b...1a9c'), urls=['https://192...3:4242']): [models.t...ChunkInfo(uuid=ChunkUUID('5b237ceb-...31cb14b5b4'), maxsize_code=1, hash=unhexlify('616263646...6465666768'), size=2097152, crc32=0x07FD7A5B), models.t...ChunkInfo(uuid=ChunkUUID('940f0711-...8580f432dc'), maxsize_code=0, hash=unhexlify('303132333...3334353637'), size=143941, crc32=0x7E5CE7AD), models.t...ChunkInfo(uuid=ChunkUUID('a5b605f2-...17b7e8e784'), maxsize_code=1, hash=unhexlify('373635343...3433323130'), size=2097151, crc32=0xDCC847D8)], Host(uuid=PeerUUID('233ad9c2-268f-4506-ab0f-4c71461c5d88'), urls=['https://192.168.1.2:1234', 'https://127.0.0.1:1234']): [models.t...ChunkInfo(uuid=ChunkUUID('0a7064b3-...f9a40dfcf3'), maxsize_code=1, hash=unhexlify('616263646...6461626364'), size=73819, crc32=0x2A5FE875)]}) """ assert cls.validate_schema(doc), repr(doc) chunks_map_preprocessed = \ {Host(uuid=PeerUUID(host_uuid_str), urls=per_chunk_data['urls']): [model_ChunkInfo.from_bson(doc)() for doc in per_chunk_data['chunks']] for host_uuid_str, per_chunk_data in doc['chunks_map'].iteritems()} return partial( super(SendChunksTransactionState_Node, cls).from_bson(doc), # Mandatory chunks_map=chunks_map_preprocessed)
def restore_dataset_to_lacking_hosts(self, me, host, ds_uuid): """ Given a dataset (its UUID), restore it to every host which lacks it (if it is a sync dataset.). @param me: my node @type me: Node @param host: host which just completed the backup and is going to restore the data, or C{None} if not applicable. @type host: Host, NoneType @type ds_uuid: DatasetUUID """ logger.debug('Restoring DS %s to all lacking hosts (if needed), ' 'except %r', ds_uuid, host) with db.RDB() as rdbw: # If the dataset is non-syncing, _host_uuids_to_restore # will be empty. host_uuid = host.uuid if host else None _host_uuids_to_restore = \ TrustedQueries.TrustedDatasets \ .get_host_uuids_lacking_sync_dataset( ds_uuid, rdbw=rdbw) # But, if we have a host that definitely just completed the backup, # we can filter it out. # At the same step, we eagerly evaluate it, so we can get out # of RDB wrapper. host_uuids_to_restore = list(_host_uuids_to_restore) \ if host_uuid is None \ else [u for u in _host_uuids_to_restore if u != host_uuid] if host_uuids_to_restore: logger.debug('Will probably restore dataset %s to %r', ds_uuid, host_uuids_to_restore) for restore_host_uuid in host_uuids_to_restore: restore_host_uuid = \ PeerUUID.safe_cast_uuid(restore_host_uuid) if self.__known_hosts.is_peer_alive(restore_host_uuid): logger.debug('Restoring dataset %s to host %r', ds_uuid, restore_host_uuid) r_tr = self.__app.tr_manager.create_new_transaction( name='RESTORE', src=me, dst=self.__known_hosts[restore_host_uuid], parent=None, # RESTORE-specific ds_uuid=ds_uuid, # None means "all files" file_paths_for_basedirs=None, wr_uuid=None) else: logger.debug("Could've restored dataset %s to host %r, " 'but the host is not alive', ds_uuid, restore_host_uuid) else: logger.debug('Dataset %s is likely not auto-syncable', ds_uuid)
def from_bson(cls, doc): r""" >>> from common.typed_uuids import DatasetUUID, MessageUUID, PeerUUID >>> # No optional arguments >>> Transaction.from_bson({ ... 'src': UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'), ... 'state': {'dataset_uuid': ... UUID('cf9a54b1-1239-48de-9b25-d7fa927db125')}, ... 'uuid': UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'), ... 'dst': UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'), ... 'type': 'BACKUP', ... 'ts': datetime(2012, 9, 26, 14, 29, 48, 877434) ... })() # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE Transaction(_id=None, type_='BACKUP', uuid=MessageUUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'), src=PeerUUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'), dst=PeerUUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'), ts=datetime.datetime(2012, 9, 26, 14, 29, 48, 877434), state=BackupTransactionState_Node(tr_start_time=....datetime(2012, 9, 26, 14, 29, 48, 877434), tr_uuid=MessageUUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'), tr_src_uuid=PeerUUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'), tr_dst_uuid=PeerUUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'), dataset_uuid=DatasetUUID('cf9a54b1-...-d7fa927db125'))) >>> # All optional arguments >>> Transaction.from_bson({ ... 'src': UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'), ... 'state': {'dataset_uuid': ... UUID('cf9a54b1-1239-48de-9b25-d7fa927db125')}, ... 'uuid': UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'), ... 'parent': UUID('a096cf37-dc4f-4d0e-9488-da50443c8c40'), ... 'dst': UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'), ... 'wake_up_ts': datetime(2012, 10, 3, 12, 31, 8, 279807), ... 'type': 'BACKUP', ... 'ts': datetime(2012, 9, 26, 14, 29, 48, 877434) ... })() # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE Transaction(_id=None, type_='BACKUP', uuid=MessageUUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'), src=PeerUUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'), dst=PeerUUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'), ts=datetime.datetime(2012, 9, 26, 14, 29, 48, 877434), state=Backup...State_Node(tr_start_time=....datetime(2012, 9, 26, 14, 29, 48, 877434), tr_uuid=MessageUUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'), tr_src_uuid=PeerUUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'), tr_dst_uuid=PeerUUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'), dataset_uuid=DatasetUUID('cf9a54b1-1239-...-d7fa927db125')), parent=UUID('a096cf37-dc4f-4d0e-9488-da50443c8c40'), wake_up_ts=datetime.datetime(2012, 10, 3, 12, 31, 8, 279807)) @rtype: Transaction """ assert cls.validate_schema(doc), repr(doc) state = cls.state_from_tr_bson(doc) return partial(super(Transaction, cls).from_bson(doc), type_=state.tr_type, uuid=TransactionUUID.safe_cast_uuid(state.tr_uuid), src=PeerUUID.safe_cast_uuid(state.tr_src_uuid), dst=PeerUUID.safe_cast_uuid(state.tr_dst_uuid), ts=state.tr_start_time, state=state, # Optional parent=doc.get('parent'), wake_up_ts=doc.get('wake_up_ts'))