def state_from_tr_bson(cls, doc):
r"""
>>> from common.typed_uuids import DatasetUUID, MessageUUID, PeerUUID
>>> # No optional arguments
>>> Transaction.state_from_tr_bson({
... 'src': UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
... 'state': {'dataset_uuid':
... UUID('cf9a54b1-1239-48de-9b25-d7fa927db125')},
... 'uuid': UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
... 'dst': UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'),
... 'type': 'BACKUP',
... 'ts': datetime(2012, 9, 26, 14, 29, 48, 877434)
... }) # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE
BackupTransactionState_Node(tr_start_time=....datetime(2012, 9, 26,
14, 29, 48,
877434),
tr_uuid=MessageUUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
tr_src_uuid=PeerUUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
tr_dst_uuid=PeerUUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'),
dataset_uuid=DatasetUUID('cf9a54b1-1239-48de-9b25-d7fa927db125'))
>>> # All optional arguments
>>> Transaction.state_from_tr_bson({
... 'src': UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
... 'state': {'dataset_uuid':
... UUID('cf9a54b1-1239-48de-9b25-d7fa927db125')},
... 'uuid': UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
... 'parent': UUID('a096cf37-dc4f-4d0e-9488-da50443c8c40'),
... 'dst': UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'),
... 'wake_up_ts': datetime(2012, 10, 3, 12, 31, 8, 279807),
... 'type': 'BACKUP',
... 'ts': datetime(2012, 9, 26, 14, 29, 48, 877434)
... }) # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE
BackupTransactionState_Node(tr_start_time=....datetime(2012, 9, 26,
14, 29, 48,
877434),
tr_uuid=MessageUUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
tr_src_uuid=PeerUUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
tr_dst_uuid=PeerUUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'),
dataset_uuid=DatasetUUID('cf9a54b1-1239-48de-9b25-d7fa927db125'))
@type: NodeTransactionState
"""
assert cls.validate_schema(doc), repr(doc)
state_class = \
transaction_states.TRANSACTION_STATE_CLASS_BY_NAME[doc['type']]
assert state_class.validate_schema(doc['state']), repr(doc)
return state_class.from_bson(doc['state'])(
tr_start_time=doc['ts'],
tr_uuid=TransactionUUID.safe_cast_uuid(doc['uuid']),
tr_src_uuid=PeerUUID.safe_cast_uuid(doc['src']),
tr_dst_uuid=PeerUUID.safe_cast_uuid(doc['dst']))
def state_from_tr_bson(cls, doc):
r"""
>>> from common.typed_uuids import DatasetUUID, MessageUUID, PeerUUID
>>> # No optional arguments
>>> Transaction.state_from_tr_bson({
... 'src': UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
... 'state': {'dataset_uuid':
... UUID('cf9a54b1-1239-48de-9b25-d7fa927db125')},
... 'uuid': UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
... 'dst': UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'),
... 'type': 'BACKUP',
... 'ts': datetime(2012, 9, 26, 14, 29, 48, 877434)
... }) # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE
BackupTransactionState_Node(tr_start_time=....datetime(2012, 9, 26,
14, 29, 48,
877434),
tr_uuid=MessageUUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
tr_src_uuid=PeerUUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
tr_dst_uuid=PeerUUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'),
dataset_uuid=DatasetUUID('cf9a54b1-1239-48de-9b25-d7fa927db125'))
>>> # All optional arguments
>>> Transaction.state_from_tr_bson({
... 'src': UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
... 'state': {'dataset_uuid':
... UUID('cf9a54b1-1239-48de-9b25-d7fa927db125')},
... 'uuid': UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
... 'parent': UUID('a096cf37-dc4f-4d0e-9488-da50443c8c40'),
... 'dst': UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'),
... 'wake_up_ts': datetime(2012, 10, 3, 12, 31, 8, 279807),
... 'type': 'BACKUP',
... 'ts': datetime(2012, 9, 26, 14, 29, 48, 877434)
... }) # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE
BackupTransactionState_Node(tr_start_time=....datetime(2012, 9, 26,
14, 29, 48,
877434),
tr_uuid=MessageUUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
tr_src_uuid=PeerUUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
tr_dst_uuid=PeerUUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'),
dataset_uuid=DatasetUUID('cf9a54b1-1239-48de-9b25-d7fa927db125'))
@type: NodeTransactionState
"""
assert cls.validate_schema(doc), repr(doc)
state_class = \
transaction_states.TRANSACTION_STATE_CLASS_BY_NAME[doc['type']]
assert state_class.validate_schema(doc['state']), repr(doc)
return state_class.from_bson(doc['state'])(
tr_start_time=doc['ts'],
tr_uuid=TransactionUUID.safe_cast_uuid(doc['uuid']),
tr_src_uuid=PeerUUID.safe_cast_uuid(doc['src']),
tr_dst_uuid=PeerUUID.safe_cast_uuid(doc['dst']))
def _prepare_chunk_maps_from_docstore(chunk_map):
r"""
Convert a chunk map (like, in C{self.chunks_to_replicate}) or
in C{self.chunks_to_restore} from the form stored in the FastDB
to the proper classes.
>>> from uuid import UUID
>>> from bson.binary import Binary
>>> cls = ReceiveChunksTransactionState_Node
>>> cls._prepare_chunk_maps_from_docstore({
... 'e96a073b3cd049a6b14a1fb04c221a9c':
... [{'crc32': 710928501, 'maxsize_code': 1,
... 'hash': Binary('abcdabcd' * 8, 0),
... 'uuid':UUID('5b237ceb-300d-4c88-b4c0-6331cb14b5b4'),
... 'size': 73819}],
... '233ad9c2268f4506ab0f4c71461c5d88':
... [{'crc32': 134052443, 'maxsize_code': 1,
... 'hash': Binary('abcdefgh' * 8, 0),
... 'uuid':UUID('5b237ceb-300d-4c88-b4c0-6331cb14b5b4'),
... 'size': 2097152},
... {'crc32': 2120017837, 'maxsize_code': 0,
... 'hash': Binary('01234567' * 8, 0),
... 'uuid':UUID('940f0711-52d7-42fb-bf4c-818580f432dc'),
... 'size': 143941},
... {'crc32': 3704113112, 'maxsize_code': 1,
... 'hash': Binary('76543210' * 8, 0),
... 'uuid':UUID('a5b605f2-6ea5-49f3-8658-d217b7e8e784'),
... 'size': 2097151}]
... }) # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE
{PeerUUID('e96a073b-3cd0-49a6-b14a-1fb04c221a9c'):
[models.types.ChunkInfo(uuid=ChunkUUID('5b237ceb-...331cb14b5b4'),
maxsize_code=1,
hash=unhexlify('616263646...36461626364'),
size=73819, crc32=0x2A5FE875)],
PeerUUID('233ad9c2-268f-4506-ab0f-4c71461c5d88'):
[models.types.ChunkInfo(uuid=ChunkUUID('5b237ceb-...331cb14b5b4'),
maxsize_code=1,
hash=unhexlify('616263646...36465666768'),
size=2097152, crc32=0x07FD7A5B),
models.types.ChunkInfo(uuid=ChunkUUID('940f0711-...18580f432dc'),
maxsize_code=0,
hash=unhexlify('303132333...23334353637'),
size=143941, crc32=0x7E5CE7AD),
models.types.ChunkInfo(uuid=ChunkUUID('a5b605f2-...217b7e8e784'),
maxsize_code=1,
hash=unhexlify('373635343...53433323130'),
size=2097151, crc32=0xDCC847D8)]}
@type chunk_map: dict
@rtype: dict
"""
return {PeerUUID(host_uuid_str):
[model_ChunkInfo.from_bson(chunk_doc)()
for chunk_doc in chunk_docs]
for host_uuid_str, chunk_docs in chunk_map.iteritems()}
def from_bson(cls, doc):
r"""
>>> from common.typed_uuids import DatasetUUID, MessageUUID, PeerUUID
>>> # No optional arguments
>>> Transaction.from_bson({
... 'src': UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
... 'state': {'dataset_uuid':
... UUID('cf9a54b1-1239-48de-9b25-d7fa927db125')},
... 'uuid': UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
... 'dst': UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'),
... 'type': 'BACKUP',
... 'ts': datetime(2012, 9, 26, 14, 29, 48, 877434)
... })() # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE
Transaction(_id=None,
type_='BACKUP',
uuid=MessageUUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
src=PeerUUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
dst=PeerUUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'),
ts=datetime.datetime(2012, 9, 26, 14, 29, 48, 877434),
state=BackupTransactionState_Node(tr_start_time=....datetime(2012,
9, 26, 14, 29, 48, 877434),
tr_uuid=MessageUUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
tr_src_uuid=PeerUUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
tr_dst_uuid=PeerUUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'),
dataset_uuid=DatasetUUID('cf9a54b1-...-d7fa927db125')))
>>> # All optional arguments
>>> Transaction.from_bson({
... 'src': UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
... 'state': {'dataset_uuid':
... UUID('cf9a54b1-1239-48de-9b25-d7fa927db125')},
... 'uuid': UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
... 'parent': UUID('a096cf37-dc4f-4d0e-9488-da50443c8c40'),
... 'dst': UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'),
... 'wake_up_ts': datetime(2012, 10, 3, 12, 31, 8, 279807),
... 'type': 'BACKUP',
... 'ts': datetime(2012, 9, 26, 14, 29, 48, 877434)
... })() # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE
Transaction(_id=None,
type_='BACKUP',
uuid=MessageUUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
src=PeerUUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
dst=PeerUUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'),
ts=datetime.datetime(2012, 9, 26, 14, 29, 48, 877434),
state=Backup...State_Node(tr_start_time=....datetime(2012, 9, 26,
14, 29, 48,
877434),
tr_uuid=MessageUUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
tr_src_uuid=PeerUUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
tr_dst_uuid=PeerUUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'),
dataset_uuid=DatasetUUID('cf9a54b1-1239-...-d7fa927db125')),
parent=UUID('a096cf37-dc4f-4d0e-9488-da50443c8c40'),
wake_up_ts=datetime.datetime(2012, 10, 3, 12, 31, 8, 279807))
@rtype: Transaction
"""
assert cls.validate_schema(doc), repr(doc)
state = cls.state_from_tr_bson(doc)
return partial(
super(Transaction, cls).from_bson(doc),
type_=state.tr_type,
uuid=TransactionUUID.safe_cast_uuid(state.tr_uuid),
src=PeerUUID.safe_cast_uuid(state.tr_src_uuid),
dst=PeerUUID.safe_cast_uuid(state.tr_dst_uuid),
ts=state.tr_start_time,
state=state,
# Optional
parent=doc.get('parent'),
wake_up_ts=doc.get('wake_up_ts'))
def from_bson(cls, doc):
r"""
>>> from datetime import datetime
>>> from uuid import UUID
>>> from bson.binary import Binary
>>> tr_start_time = datetime(2012, 9, 26, 14, 29, 48, 877434)
>>> tr_uuid = UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2')
>>> tr_src_uuid = UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24')
>>> tr_dst_uuid=UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d')
>>> SendChunksTransactionState_Node.from_bson({
... 'chunks_map': {
... '233ad9c2268f4506ab0f4c71461c5d88':
... {'chunks':
... [{'crc32': 710928501, 'maxsize_code': 1,
... 'hash': Binary('abcdabcd' * 8, 0),
... 'uuid':
... UUID('0a7064b3-bef6-45c0-9e82-e9f9a40dfcf3'),
... 'size': 73819}],
... 'urls': ['https://192.168.1.2:1234',
... 'https://127.0.0.1:1234']},
... 'e96a073b3cd049a6b14a1fb04c221a9c':
... {'chunks':
... [{'crc32': 134052443, 'maxsize_code': 1,
... 'hash': Binary('abcdefgh' * 8, 0),
... 'uuid':
... UUID('5b237ceb-300d-4c88-b4c0-6331cb14b5b4'),
... 'size': 2097152},
... {'crc32': 2120017837, 'maxsize_code': 0,
... 'hash': Binary('01234567' * 8, 0),
... 'uuid':
... UUID('940f0711-52d7-42fb-bf4c-818580f432dc'),
... 'size': 143941},
... {'crc32': 3704113112, 'maxsize_code': 1,
... 'hash': Binary('76543210' * 8, 0),
... 'uuid':
... UUID('a5b605f2-6ea5-49f3-8658-d217b7e8e784'),
... 'size': 2097151}],
... 'urls': ['https://192.168.2.3:4242']}
... }
... })(tr_start_time=tr_start_time,
... tr_uuid=tr_uuid,
... tr_src_uuid=tr_src_uuid,
... tr_dst_uuid=tr_dst_uuid
... ) # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE
SendCh...State_Node(chunks_map={Host(uuid=PeerUUID('e96a073b...1a9c'),
urls=['https://192...3:4242']):
[models.t...ChunkInfo(uuid=ChunkUUID('5b237ceb-...31cb14b5b4'),
maxsize_code=1,
hash=unhexlify('616263646...6465666768'),
size=2097152, crc32=0x07FD7A5B),
models.t...ChunkInfo(uuid=ChunkUUID('940f0711-...8580f432dc'),
maxsize_code=0,
hash=unhexlify('303132333...3334353637'),
size=143941, crc32=0x7E5CE7AD),
models.t...ChunkInfo(uuid=ChunkUUID('a5b605f2-...17b7e8e784'),
maxsize_code=1,
hash=unhexlify('373635343...3433323130'),
size=2097151, crc32=0xDCC847D8)],
Host(uuid=PeerUUID('233ad9c2-268f-4506-ab0f-4c71461c5d88'),
urls=['https://192.168.1.2:1234', 'https://127.0.0.1:1234']):
[models.t...ChunkInfo(uuid=ChunkUUID('0a7064b3-...f9a40dfcf3'),
maxsize_code=1,
hash=unhexlify('616263646...6461626364'),
size=73819, crc32=0x2A5FE875)]})
"""
assert cls.validate_schema(doc), repr(doc)
chunks_map_preprocessed = \
{Host(uuid=PeerUUID(host_uuid_str),
urls=per_chunk_data['urls']):
[model_ChunkInfo.from_bson(doc)()
for doc in per_chunk_data['chunks']]
for host_uuid_str, per_chunk_data
in doc['chunks_map'].iteritems()}
return partial(
super(SendChunksTransactionState_Node, cls).from_bson(doc),
# Mandatory
chunks_map=chunks_map_preprocessed)
def restore_dataset_to_lacking_hosts(self, me, host, ds_uuid):
"""
Given a dataset (its UUID), restore it to every host which lacks it
(if it is a sync dataset.).
@param me: my node
@type me: Node
@param host: host which just completed the backup and is going
to restore the data, or C{None} if not applicable.
@type host: Host, NoneType
@type ds_uuid: DatasetUUID
"""
logger.debug('Restoring DS %s to all lacking hosts (if needed), '
'except %r',
ds_uuid, host)
with db.RDB() as rdbw:
# If the dataset is non-syncing, _host_uuids_to_restore
# will be empty.
host_uuid = host.uuid if host else None
_host_uuids_to_restore = \
TrustedQueries.TrustedDatasets \
.get_host_uuids_lacking_sync_dataset(
ds_uuid, rdbw=rdbw)
# But, if we have a host that definitely just completed the backup,
# we can filter it out.
# At the same step, we eagerly evaluate it, so we can get out
# of RDB wrapper.
host_uuids_to_restore = list(_host_uuids_to_restore) \
if host_uuid is None \
else [u for u in _host_uuids_to_restore
if u != host_uuid]
if host_uuids_to_restore:
logger.debug('Will probably restore dataset %s to %r',
ds_uuid, host_uuids_to_restore)
for restore_host_uuid in host_uuids_to_restore:
restore_host_uuid = \
PeerUUID.safe_cast_uuid(restore_host_uuid)
if self.__known_hosts.is_peer_alive(restore_host_uuid):
logger.debug('Restoring dataset %s to host %r',
ds_uuid, restore_host_uuid)
r_tr = self.__app.tr_manager.create_new_transaction(
name='RESTORE',
src=me,
dst=self.__known_hosts[restore_host_uuid],
parent=None,
# RESTORE-specific
ds_uuid=ds_uuid,
# None means "all files"
file_paths_for_basedirs=None,
wr_uuid=None)
else:
logger.debug("Could've restored dataset %s to host %r, "
'but the host is not alive',
ds_uuid, restore_host_uuid)
else:
logger.debug('Dataset %s is likely not auto-syncable', ds_uuid)
def from_bson(cls, doc):
r"""
>>> from common.typed_uuids import DatasetUUID, MessageUUID, PeerUUID
>>> # No optional arguments
>>> Transaction.from_bson({
... 'src': UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
... 'state': {'dataset_uuid':
... UUID('cf9a54b1-1239-48de-9b25-d7fa927db125')},
... 'uuid': UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
... 'dst': UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'),
... 'type': 'BACKUP',
... 'ts': datetime(2012, 9, 26, 14, 29, 48, 877434)
... })() # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE
Transaction(_id=None,
type_='BACKUP',
uuid=MessageUUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
src=PeerUUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
dst=PeerUUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'),
ts=datetime.datetime(2012, 9, 26, 14, 29, 48, 877434),
state=BackupTransactionState_Node(tr_start_time=....datetime(2012,
9, 26, 14, 29, 48, 877434),
tr_uuid=MessageUUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
tr_src_uuid=PeerUUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
tr_dst_uuid=PeerUUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'),
dataset_uuid=DatasetUUID('cf9a54b1-...-d7fa927db125')))
>>> # All optional arguments
>>> Transaction.from_bson({
... 'src': UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
... 'state': {'dataset_uuid':
... UUID('cf9a54b1-1239-48de-9b25-d7fa927db125')},
... 'uuid': UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
... 'parent': UUID('a096cf37-dc4f-4d0e-9488-da50443c8c40'),
... 'dst': UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'),
... 'wake_up_ts': datetime(2012, 10, 3, 12, 31, 8, 279807),
... 'type': 'BACKUP',
... 'ts': datetime(2012, 9, 26, 14, 29, 48, 877434)
... })() # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE
Transaction(_id=None,
type_='BACKUP',
uuid=MessageUUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
src=PeerUUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
dst=PeerUUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'),
ts=datetime.datetime(2012, 9, 26, 14, 29, 48, 877434),
state=Backup...State_Node(tr_start_time=....datetime(2012, 9, 26,
14, 29, 48,
877434),
tr_uuid=MessageUUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
tr_src_uuid=PeerUUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
tr_dst_uuid=PeerUUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'),
dataset_uuid=DatasetUUID('cf9a54b1-1239-...-d7fa927db125')),
parent=UUID('a096cf37-dc4f-4d0e-9488-da50443c8c40'),
wake_up_ts=datetime.datetime(2012, 10, 3, 12, 31, 8, 279807))
@rtype: Transaction
"""
assert cls.validate_schema(doc), repr(doc)
state = cls.state_from_tr_bson(doc)
return partial(super(Transaction, cls).from_bson(doc),
type_=state.tr_type,
uuid=TransactionUUID.safe_cast_uuid(state.tr_uuid),
src=PeerUUID.safe_cast_uuid(state.tr_src_uuid),
dst=PeerUUID.safe_cast_uuid(state.tr_dst_uuid),
ts=state.tr_start_time,
state=state,
# Optional
parent=doc.get('parent'),
wake_up_ts=doc.get('wake_up_ts'))