def from_bson(cls, doc): r""" >>> from datetime import datetime >>> BackupTransactionState_Node.from_bson({ ... 'dataset_uuid': UUID('cf9a54b1-1239-48de-9b25-d7fa927db125') ... })(tr_start_time=datetime(2012, 9, 26, 14, 29, 48, 877434), ... tr_uuid=UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'), ... tr_src_uuid=UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'), ... tr_dst_uuid=UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d') ... ) # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE BackupTransactionState_Node(tr_start_time=....datetime(2012, 9, 26, 14, 29, 48, 877434), tr_uuid=UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'), tr_src_uuid=UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'), tr_dst_uuid=UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'), dataset_uuid=DatasetUUID('cf9a54b1-1239-48de-9b25-d7fa927db125')) """ assert cls.validate_schema(doc), repr(doc) return partial(super(BackupTransactionState_Node, cls).from_bson(doc), # Mandatory dataset_uuid=DatasetUUID.safe_cast_uuid( doc['dataset_uuid']), # Optional ack_result_code=doc.get('ack_result_code'))
def from_bson(cls, doc): r""" >>> from datetime import datetime >>> BackupTransactionState_Node.from_bson({ ... 'dataset_uuid': UUID('cf9a54b1-1239-48de-9b25-d7fa927db125') ... })(tr_start_time=datetime(2012, 9, 26, 14, 29, 48, 877434), ... tr_uuid=UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'), ... tr_src_uuid=UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'), ... tr_dst_uuid=UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d') ... ) # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE BackupTransactionState_Node(tr_start_time=....datetime(2012, 9, 26, 14, 29, 48, 877434), tr_uuid=UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'), tr_src_uuid=UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'), tr_dst_uuid=UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'), dataset_uuid=DatasetUUID('cf9a54b1-1239-48de-9b25-d7fa927db125')) """ assert cls.validate_schema(doc), repr(doc) return partial( super(BackupTransactionState_Node, cls).from_bson(doc), # Mandatory dataset_uuid=DatasetUUID.safe_cast_uuid(doc['dataset_uuid']), # Optional ack_result_code=doc.get('ack_result_code'))
def _init_from_body(self, body): r""" >>> from uuid import UUID >>> from common.inhabitants import Host, Node >>> host = Host(uuid=UUID('00000000-7606-420c-8a98-a6c8728ac98d')) >>> node = Node(uuid=UUID('11111111-79b4-49e0-b72e-8c4ce6a2aed9')) >>> msg0 = BackupMessage( ... src=node, ... dst=host, ... uuid=UUID('4ac2536a-4a1e-4b08-ad4e-11bd675fdf15') ... ) >>> BackupMessage._init_from_body._without_bunzip2._without_json_loads( ... msg0, {}) Traceback (most recent call last): ... Exception: Unparseable backup message body: {} >>> msg1 = BackupMessage( ... src=node, ... dst=host, ... uuid=UUID('baebd0f2-fc58-417b-97ee-08a892cd5a8f'), ... status_code=1 ... ) >>> BackupMessage._init_from_body._without_bunzip2._without_json_loads( ... msg1, {'dataset_uuid': '60fb966767ae4e3c9c86edac180dd4e6'}) >>> msg1 # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE BackupMessage(src=Node(uuid=PeerUUID('11111111-...-8c4ce6a2aed9')), dst=Host(uuid=PeerUUID('00000000-7606-420c-8a98-a6c8728ac98d')), uuid=MessageUUID('baebd0f2-fc58-417b-97ee-08a892cd5a8f'), status_code=1 [ dataset_uuid=DatasetUUID('60fb9667-67ae-4e3c-9c86-edac180dd4e6') ]) """ # N2H if 'dataset_uuid' in body: self.dataset_uuid = DatasetUUID(body['dataset_uuid']) else: raise Exception('Unparseable backup message body: {!r}' .format(body))
def __backup_some_phys_files(self, base_dir, files, ugroup, __do_start_backup=True): r"""Given some files, create a new dataset and start to backup them. >>> # ugroup = UserGroup( >>> # uuid=UserGroupUUID('00000000-bbbb-0000-0000-000000000001'), >>> # name='AlphA', >>> # private=True, >>> # enc_key='\x01\xe6\x13\xdab)\xd2n\xd6\xafTH\x03h\x02\x12' >>> # '\x17D\x1a\xeb\x8b6\xc0\x9b\xa6\x7f\xcc\x06N\xcf' >>> # '\x8b\xcd' >>> # ) >>> # __backup_some_phys_files( >>> # base_dir='u'/home/john/FreeBrie', >>> # files=[ >>> # LocalPhysicalFileStateRel( >>> # rel_dir='', >>> # rel_file=u'f1.mp3', >>> # size=13829879, >>> # time_changed=datetime(2012, 11, 5, 12,12,41,904430)), >>> # LocalPhysicalFileStateRel( >>> # rel_dir='', >>> # rel_file=u'f2.avi', >>> # size=3522710, >>> # time_changed=datetime(2012, 11, 5, 12,12,41,988433)), >>> # LocalPhysicalFileStateRel( >>> # rel_dir=u'a/b', >>> # rel_file=u'bbb', >>> # size=4, >>> # time_changed=datetime(2012, 10, 11, 15 33 42 19808)), >>> # LocalPhysicalFileStateRel( >>> # rel_dir=u'a/b/c', >>> # rel_file=u'ccc', >>> # size=4, >>> # time_changed=datetime(2012, 10, 11, 15 33 41 979807)) >>> # ], >>> # ugroup=ugroup) @todo: complete the unit test, which is half-done! @param base_dir: the directory being backed up. @type base_dir: basestring @param files: the iterable over the files which should be backed up. Contains C{LocalPhysicalFileStateRel} objects. The caller should ensure that C{files} is non-empty! @type files: col.Iterable @type ugroup: UserGroup @return: the created dataset (if succeeded). @rtype: DatasetOnPhysicalFiles, NoneType """ logger.debug('__backup_some_phys_files(%r, %r)', base_dir, ugroup) # Group files by rel_dir; then ignore base_dir, # keep only rel_dir, rel_file, size and time_changed files_grouped_by_rel_dir = \ ((RelVirtualFile(rel_dir=f.rel_dir, filename=f.rel_file, # If we can read real stat, read it; # otherwise we'll emulate it with fake_stat stat=coalesce(os_ex.safe_stat( # real stat os.path.join(base_dir, f.rel_path)), os_ex.fake_stat( # deleted file st_mode=None, atime=f.time_changed, mtime=f.time_changed, ctime=f.time_changed, size=None)), stat_getter=lambda f=f: os_ex.safe_stat( os.path.join(base_dir, f.rel_path)), file_getter=lambda f=f: open(os.path.join(base_dir, f.rel_path), 'rb')) for f in per_rel_dir) for rel_dir, per_rel_dir in sorted_groupby(files, attrgetter('rel_dir'))) # Example: # files_grouped_by_rel_dir = [ # [ # RelVirtualFile(...), # RelVirtualFile(...), # RelVirtualFile(...) # ], # [ # RelVirtualFile(...), # [ # RelVirtualFile(...) # ] # ] _path_map = {base_dir: {'ifiles': files_grouped_by_rel_dir, 'stat': os_ex.safe_stat(base_dir)}} ds_uuid = DatasetUUID.safe_cast_uuid(gen_uuid()) ds = self.select_paths_for_backup(ds_name='', ds_uuid=ds_uuid, ugroup_uuid=ugroup.uuid, sync=True, paths_map=_path_map) if ds is not None and __do_start_backup: self.start_backup(ds_uuid) return ds
def __backup_some_phys_files(self, base_dir, files, ugroup, __do_start_backup=True): r"""Given some files, create a new dataset and start to backup them. >>> # ugroup = UserGroup( >>> # uuid=UserGroupUUID('00000000-bbbb-0000-0000-000000000001'), >>> # name='AlphA', >>> # private=True, >>> # enc_key='\x01\xe6\x13\xdab)\xd2n\xd6\xafTH\x03h\x02\x12' >>> # '\x17D\x1a\xeb\x8b6\xc0\x9b\xa6\x7f\xcc\x06N\xcf' >>> # '\x8b\xcd' >>> # ) >>> # __backup_some_phys_files( >>> # base_dir='u'/home/john/FreeBrie', >>> # files=[ >>> # LocalPhysicalFileStateRel( >>> # rel_dir='', >>> # rel_file=u'f1.mp3', >>> # size=13829879, >>> # time_changed=datetime(2012, 11, 5, 12,12,41,904430)), >>> # LocalPhysicalFileStateRel( >>> # rel_dir='', >>> # rel_file=u'f2.avi', >>> # size=3522710, >>> # time_changed=datetime(2012, 11, 5, 12,12,41,988433)), >>> # LocalPhysicalFileStateRel( >>> # rel_dir=u'a/b', >>> # rel_file=u'bbb', >>> # size=4, >>> # time_changed=datetime(2012, 10, 11, 15 33 42 19808)), >>> # LocalPhysicalFileStateRel( >>> # rel_dir=u'a/b/c', >>> # rel_file=u'ccc', >>> # size=4, >>> # time_changed=datetime(2012, 10, 11, 15 33 41 979807)) >>> # ], >>> # ugroup=ugroup) @todo: complete the unit test, which is half-done! @param base_dir: the directory being backed up. @type base_dir: basestring @param files: the iterable over the files which should be backed up. Contains C{LocalPhysicalFileStateRel} objects. The caller should ensure that C{files} is non-empty! @type files: col.Iterable @type ugroup: UserGroup @return: the created dataset (if succeeded). @rtype: DatasetOnPhysicalFiles, NoneType """ logger.debug('__backup_some_phys_files(%r, %r)', base_dir, ugroup) # Group files by rel_dir; then ignore base_dir, # keep only rel_dir, rel_file, size and time_changed files_grouped_by_rel_dir = \ ((RelVirtualFile(rel_dir=f.rel_dir, filename=f.rel_file, # If we can read real stat, read it; # otherwise we'll emulate it with fake_stat stat=coalesce(os_ex.safe_stat( # real stat os.path.join(base_dir, f.rel_path)), os_ex.fake_stat( # deleted file st_mode=None, atime=f.time_changed, mtime=f.time_changed, ctime=f.time_changed, size=None)), stat_getter=lambda f=f: os_ex.safe_stat( os.path.join(base_dir, f.rel_path)), file_getter=lambda f=f: open(os.path.join(base_dir, f.rel_path), 'rb')) for f in per_rel_dir) for rel_dir, per_rel_dir in sorted_groupby(files, attrgetter('rel_dir'))) # Example: # files_grouped_by_rel_dir = [ # [ # RelVirtualFile(...), # RelVirtualFile(...), # RelVirtualFile(...) # ], # [ # RelVirtualFile(...), # [ # RelVirtualFile(...) # ] # ] _path_map = { base_dir: { 'ifiles': files_grouped_by_rel_dir, 'stat': os_ex.safe_stat(base_dir) } } ds_uuid = DatasetUUID.safe_cast_uuid(gen_uuid()) ds = self.select_paths_for_backup(ds_name='', ds_uuid=ds_uuid, ugroup_uuid=ugroup.uuid, sync=True, paths_map=_path_map) if ds is not None and __do_start_backup: self.start_backup(ds_uuid) return ds
def __restore_files(self): """Internal procedure which actually restores the files. @todo: the Fingerprint calculation should be turned into "file is read by blocks and then repacked into 16KiB segments"; then recalculation of the fingerprint in case of FP mismatch won't be needed. """ _message = self.message my_host = self.manager.app.host feature_set = self.manager.app.feature_set ds = _message.dataset wr_uuid = _message.wr_uuid ugroup = _message.ugroup restore_directory = self.__get_restore_directory() assert _message.sync == (wr_uuid is None) == (ds.uuid is not None), \ (_message.sync, wr_uuid, ds) base_dir_id = None # will be used later if restore_directory is None: logger.error('Do not know the restore directory') else: logger.debug('Going to restore dataset %r for %r to %r', ds, ugroup, restore_directory) if not os.path.exists(restore_directory): os.makedirs(restore_directory) group_key = ugroup.enc_key if feature_set.per_group_encryption \ else None cryptographer = Cryptographer(group_key=group_key, key_generator=None) is_whole_dataset_restored = _message.sync logger.debug('Restoring %s files from dataset: %r', 'all' if is_whole_dataset_restored else 'selected', coalesce(ds, 'N/A')) # TODO: use the "delete time" from the LocalPhysicalFileState! _now = datetime.utcnow() # If we are syncing-in the whole dataset, we should write it # into the DB as a whole. The files/file_locals will be bound to it # so that after restore, we'll know on this Host that these states # are fully synced to the cloud already (in fact, they came # from the cloud). if _message.sync: # Let's hack into the files and substitute the base_dir. # TODO: do it better! for f in _message.files.iterkeys(): f.base_dir = restore_directory # Write the whole dataset to the DB _small_files = _message.files.keys() # not iterkeys(0 for now! _dirs = {restore_directory: (_small_files, [])} # Given the information in the inbound message about # the whole dataset, store this dataset in the DB. dataset = DatasetWithDirectories( name=ds.name, sync=ds.sync, directories=_dirs, # TODO: transport real data # from the node uuid=DatasetUUID.safe_cast_uuid(ds.uuid), ugroup_uuid=UserGroupUUID.safe_cast_uuid( ugroup.uuid), time_started=ds.time_started, time_completed=_now) with db.RDB() as rdbw: # Do we already have the dataset? _ds_in_progress = \ HostQueries.HostDatasets.get_my_ds_in_progress( host_uuid=my_host.uuid, ds_uuid=dataset.uuid, rdbw=rdbw) if _ds_in_progress is None: # We don't have it, insert. dummy_ds_uuid = \ HostQueries.HostDatasets.create_dataset_for_backup( my_host.uuid, dataset, rdbw) assert dummy_ds_uuid == dataset.uuid, \ (dummy_ds_uuid, dataset.uuid) base_dir_id = \ HostQueries.HostFiles.add_or_get_base_directory( restore_directory, ugroup.uuid, rdbw) error_in_any_file_occured = False # # Finally, loop over the files and restore each one # for file_, file_blocks in _message.files.iteritems(): self.__restore_op_for_path(file_, file_blocks, is_whole_dataset_restored, base_dir_id, restore_directory, cryptographer, ds) # Loop over the files completed if is_whole_dataset_restored: logger.debug('Restoring %r completed, there were %s issues.', ds, 'some' if error_in_any_file_occured else 'no') if not error_in_any_file_occured: with db.RDB() as rdbw: logger.debug('Updating %r at host %s...', ds, my_host.uuid) ds_to_finish = \ Queries.Datasets.get_dataset_by_uuid(ds.uuid, my_host.uuid, rdbw) ds_to_finish.time_completed = datetime.utcnow() logger.debug('Updating %r as completed', dataset) # Mark the current dataset as completed # only after the response from the node is received. Queries.Datasets.update_dataset(my_host.uuid, ds_to_finish, rdbw) # Everything seems ok to this moment with self.open_state(for_update=True) as state: state.ack_result_code = RestoreMessage.ResultCodes.OK
def __restore_files(self): """Internal procedure which actually restores the files. @todo: the Fingerprint calculation should be turned into "file is read by blocks and then repacked into 16KiB segments"; then recalculation of the fingerprint in case of FP mismatch won't be needed. """ _message = self.message my_host = self.manager.app.host feature_set = self.manager.app.feature_set ds = _message.dataset wr_uuid = _message.wr_uuid ugroup = _message.ugroup restore_directory = self.__get_restore_directory() assert _message.sync == (wr_uuid is None) == (ds.uuid is not None), \ (_message.sync, wr_uuid, ds) base_dir_id = None # will be used later if restore_directory is None: logger.error('Do not know the restore directory') else: logger.debug('Going to restore dataset %r for %r to %r', ds, ugroup, restore_directory) if not os.path.exists(restore_directory): os.makedirs(restore_directory) group_key = ugroup.enc_key if feature_set.per_group_encryption \ else None cryptographer = Cryptographer(group_key=group_key, key_generator=None) is_whole_dataset_restored = _message.sync logger.debug('Restoring %s files from dataset: %r', 'all' if is_whole_dataset_restored else 'selected', coalesce(ds, 'N/A')) # TODO: use the "delete time" from the LocalPhysicalFileState! _now = datetime.utcnow() # If we are syncing-in the whole dataset, we should write it # into the DB as a whole. The files/file_locals will be bound to it # so that after restore, we'll know on this Host that these states # are fully synced to the cloud already (in fact, they came # from the cloud). if _message.sync: # Let's hack into the files and substitute the base_dir. # TODO: do it better! for f in _message.files.iterkeys(): f.base_dir = restore_directory # Write the whole dataset to the DB _small_files = _message.files.keys() # not iterkeys(0 for now! _dirs = {restore_directory: (_small_files, [])} # Given the information in the inbound message about # the whole dataset, store this dataset in the DB. dataset = DatasetWithDirectories( name=ds.name, sync=ds.sync, directories=_dirs, # TODO: transport real data # from the node uuid=DatasetUUID.safe_cast_uuid(ds.uuid), ugroup_uuid=UserGroupUUID.safe_cast_uuid(ugroup.uuid), time_started=ds.time_started, time_completed=_now) with db.RDB() as rdbw: # Do we already have the dataset? _ds_in_progress = \ HostQueries.HostDatasets.get_my_ds_in_progress( host_uuid=my_host.uuid, ds_uuid=dataset.uuid, rdbw=rdbw) if _ds_in_progress is None: # We don't have it, insert. dummy_ds_uuid = \ HostQueries.HostDatasets.create_dataset_for_backup( my_host.uuid, dataset, rdbw) assert dummy_ds_uuid == dataset.uuid, \ (dummy_ds_uuid, dataset.uuid) base_dir_id = \ HostQueries.HostFiles.add_or_get_base_directory( restore_directory, ugroup.uuid, rdbw) error_in_any_file_occured = False # # Finally, loop over the files and restore each one # for file_, file_blocks in _message.files.iteritems(): self.__restore_op_for_path(file_, file_blocks, is_whole_dataset_restored, base_dir_id, restore_directory, cryptographer, ds) # Loop over the files completed if is_whole_dataset_restored: logger.debug('Restoring %r completed, there were %s issues.', ds, 'some' if error_in_any_file_occured else 'no') if not error_in_any_file_occured: with db.RDB() as rdbw: logger.debug('Updating %r at host %s...', ds, my_host.uuid) ds_to_finish = \ Queries.Datasets.get_dataset_by_uuid(ds.uuid, my_host.uuid, rdbw) ds_to_finish.time_completed = datetime.utcnow() logger.debug('Updating %r as completed', dataset) # Mark the current dataset as completed # only after the response from the node is received. Queries.Datasets.update_dataset( my_host.uuid, ds_to_finish, rdbw) # Everything seems ok to this moment with self.open_state(for_update=True) as state: state.ack_result_code = RestoreMessage.ResultCodes.OK
def create_directory(base_dir, rel_path, ds_name, group_uuid, sync, cryptographer, rdbw, ospath=posixpath): """Create new directory in the cloud. @note: all the paths should be in POSIX format. @param base_dir: the base directory path (in the dataset) where to upload the file. @type base_dir: basestring @param rel_path: the name of directory which should be created. @type rel_path: basestring @param group_uuid: the UUID of the user group, for which the file should be bound. @type group_uuid: UserGroupUUID @param sync: whether the created dataset should be considered a "sync dataset". @type sync: bool @param rdbw: RelDB wrapper. @type rdbw: DatabaseWrapperSQLAlchemy @return: the UUID of newly created dataset. @rtype: DatasetUUID """ upload_time = datetime.utcnow() # For each FileToUpload, create fake stat dir_fake_stat = fake_stat(isdir=True, atime=upload_time, mtime=upload_time, ctime=upload_time, size=None) # Turn the original FileToUpload's to RelVirtualFile's _vfile = RelVirtualFile( rel_dir=ospath.dirname(rel_path), filename=ospath.basename(rel_path), stat=dir_fake_stat, stat_getter=lambda dir_fake_stat=dir_fake_stat: dir_fake_stat, ) # isinstance(ftu, FileToUpload) # Group RelVirtualFile's by rel_dir # _files_grouped_by_rel_dir = \ # ((rvf for rvf in per_rel_dir) # for rel_dir, per_rel_dir # in sorted_groupby(_vfiles, attrgetter('rel_dir'))) paths_map = { base_dir: { "ifiles": [[_vfile]], "stat": fake_stat(isdir=True, atime=upload_time, mtime=upload_time, ctime=upload_time), } } ds_uuid = DatasetUUID.safe_cast_uuid(gen_uuid()) dataset = DatasetOnVirtualFiles.from_paths( ds_name, ds_uuid, group_uuid, sync, paths_map, upload_time, cryptographer ) assert dataset is not None # Use group_uuid as host_uuid fake_host_uuid = None dummy_ds_uuid = Queries.Datasets.create_dataset_for_backup(fake_host_uuid, dataset, rdbw) dataset.time_completed = datetime.utcnow() # That's all, folks! Queries.Datasets.update_dataset(fake_host_uuid, dataset, rdbw) return ds_uuid
) # consists_of(_vfiles, RelVirtualFile) # Group RelVirtualFile's by rel_dir _files_grouped_by_rel_dir = ( (rvf for rvf in per_rel_dir) for rel_dir, per_rel_dir in sorted_groupby(_vfiles, attrgetter("rel_dir")) ) paths_map = { base_dir: { "ifiles": _files_grouped_by_rel_dir, "stat": fake_stat(isdir=True, atime=upload_time, mtime=upload_time, ctime=upload_time), } } ds_uuid = DatasetUUID.safe_cast_uuid(gen_uuid()) dataset = DatasetOnVirtualFiles.from_paths( ds_name, ds_uuid, group_uuid, sync, paths_map, upload_time, cryptographer ) assert dataset is not None thosts = list(TrustedQueries.HostAtNode.get_all_trusted_hosts(for_storage=True, rdbw=rdbw)) logger.debug("Uploading dataset %r,... like, to %r", dataset, thosts) # Use group_uuid as host_uuid fake_host_uuid = None dummy_ds_uuid = Queries.Datasets.create_dataset_for_backup(fake_host_uuid, dataset, rdbw)
def create_directory(base_dir, rel_path, ds_name, group_uuid, sync, cryptographer, rdbw, ospath=posixpath): """Create new directory in the cloud. @note: all the paths should be in POSIX format. @param base_dir: the base directory path (in the dataset) where to upload the file. @type base_dir: basestring @param rel_path: the name of directory which should be created. @type rel_path: basestring @param group_uuid: the UUID of the user group, for which the file should be bound. @type group_uuid: UserGroupUUID @param sync: whether the created dataset should be considered a "sync dataset". @type sync: bool @param rdbw: RelDB wrapper. @type rdbw: DatabaseWrapperSQLAlchemy @return: the UUID of newly created dataset. @rtype: DatasetUUID """ upload_time = datetime.utcnow() # For each FileToUpload, create fake stat dir_fake_stat = fake_stat(isdir=True, atime=upload_time, mtime=upload_time, ctime=upload_time, size=None) # Turn the original FileToUpload's to RelVirtualFile's _vfile = RelVirtualFile( rel_dir=ospath.dirname(rel_path), filename=ospath.basename(rel_path), stat=dir_fake_stat, stat_getter=lambda dir_fake_stat=dir_fake_stat: dir_fake_stat) # isinstance(ftu, FileToUpload) # Group RelVirtualFile's by rel_dir # _files_grouped_by_rel_dir = \ # ((rvf for rvf in per_rel_dir) # for rel_dir, per_rel_dir # in sorted_groupby(_vfiles, attrgetter('rel_dir'))) paths_map = { base_dir: { 'ifiles': [[_vfile]], 'stat': fake_stat(isdir=True, atime=upload_time, mtime=upload_time, ctime=upload_time) } } ds_uuid = DatasetUUID.safe_cast_uuid(gen_uuid()) dataset = DatasetOnVirtualFiles.from_paths(ds_name, ds_uuid, group_uuid, sync, paths_map, upload_time, cryptographer) assert dataset is not None # Use group_uuid as host_uuid fake_host_uuid = None dummy_ds_uuid = Queries.Datasets.create_dataset_for_backup( fake_host_uuid, dataset, rdbw) dataset.time_completed = datetime.utcnow() # That's all, folks! Queries.Datasets.update_dataset(fake_host_uuid, dataset, rdbw) return ds_uuid
for rel_dir, per_rel_dir in sorted_groupby(_vfiles, attrgetter('rel_dir'))) paths_map = { base_dir: { 'ifiles': _files_grouped_by_rel_dir, 'stat': fake_stat(isdir=True, atime=upload_time, mtime=upload_time, ctime=upload_time) } } ds_uuid = DatasetUUID.safe_cast_uuid(gen_uuid()) dataset = DatasetOnVirtualFiles.from_paths(ds_name, ds_uuid, group_uuid, sync, paths_map, upload_time, cryptographer) assert dataset is not None thosts = list( TrustedQueries.HostAtNode.get_all_trusted_hosts(for_storage=True, rdbw=rdbw)) logger.debug('Uploading dataset %r,... like, to %r', dataset, thosts) # Use group_uuid as host_uuid fake_host_uuid = None