def create_new_host_for_user(self, username): """ Implements the @abstractmethod from C{AbstractUsersEnabledPeerBookMixin}. @rtype: Host """ uuid_created = False while not uuid_created: host_uuid = HostUUID.safe_cast_uuid(gen_uuid()) # Ensure it doesn't start with 0x00 byte uuid_created = (host_uuid.bytes[0] != '\x00') host = Host(uuid=host_uuid, urls=[], # Host-specific name=str(host_uuid), user=self.get_user_by_name(username)) with self._rdbw_factory() as rdbw, self.__fdbw_factory() as fdbw: dw = DataWrapper(rdbw=rdbw, fdbw=fdbw, bdbw=None) DataQueries.Inhabitants.add_host(username=username, hostname=str(host_uuid), host_uuid=host_uuid, trusted_host_caps=None, dw=dw) return host
def add_trusted_host(arguments): if len(arguments) < 3: cli_error('You must pass at least the user name, password digest ' 'and the host UUID!') else: username = str(arguments.popleft()) digest = str(arguments.popleft()) host_uuid = try_parse_uuid(arguments.popleft()) node_map = proceed_with_node() if node_map is not None: # Finally, add user NodeApp.add_user(UserGroupUUID.safe_cast_uuid(gen_uuid()), username=str(username), digest=digest, is_trusted=True) _for_storage = True _for_restore = False NodeApp.add_host(username=username, hostname='Trusted: {}'.format(host_uuid), host_uuid=host_uuid, trusted_host_tuple=(_for_storage, _for_restore)) NodeApp.change_user(username, digest) print(u'Added Trusted Host {!r}'.format(username))
def create_new_host_for_user(self, username): """ Implements the @abstractmethod from C{AbstractUsersEnabledPeerBookMixin}. @rtype: Host """ uuid_created = False while not uuid_created: host_uuid = HostUUID.safe_cast_uuid(gen_uuid()) # Ensure it doesn't start with 0x00 byte uuid_created = (host_uuid.bytes[0] != '\x00') host = Host( uuid=host_uuid, urls=[], # Host-specific name=str(host_uuid), user=self.get_user_by_name(username)) with self._rdbw_factory() as rdbw, self.__fdbw_factory() as fdbw: dw = DataWrapper(rdbw=rdbw, fdbw=fdbw, bdbw=None) DataQueries.Inhabitants.add_host(username=username, hostname=str(host_uuid), host_uuid=host_uuid, trusted_host_caps=None, dw=dw) return host
def __add_new_regular_user(username, arguments): if not arguments: cli_error('No digest string passed!') else: # Parse digest digest_str = arguments.popleft() try: digest = '{:040x}'.format(int(digest_str, 16)) except: cli_error('The argument \"{}\" is not a valid hexadecimal digest!' .format(digest_str)) if len(digest) != 40: cli_error('The digest argument should be a hexadecimal number ' 'up to 40 hexadigits long rather than {}!' .format(digest)) # Parse group UUID (optional) try: group_uuid = UUID(arguments[0]) # Parsed successfully arguments.popleft() except (IndexError, ValueError): group_uuid = gen_uuid() # Finally, add user NodeApp.add_user(UserGroupUUID.safe_cast_uuid(group_uuid), username=str(username), digest=digest, is_trusted=False) print(u'Added user "{}"'.format(username))
def greet_user(arguments): node_map = proceed_with_node() if node_map is not None: if not arguments: cli_error('No user name passed') elif len(arguments) < 1: cli_error('You must pass at least 1 arguments to this command: ' 'the name of the user.') else: username_str = arguments.popleft() with ds.FDB() as fdbw: FDBQueries.UserMessages.add_message( username=str(username_str), key='user_greeting {}'.format(gen_uuid()), body={ 'c': ('<p class="bold">{username}, welcome to our ' 'system</p>' '<p>This is a test message.</p>').format( username=username_str), 'en': ('<p class="bold">{username}, welcome to our ' 'system</p>' '<p>This is a test message.</p>').format( username=username_str), 'ru': (u'<p class="bold">Привет, {username}</p>' u'<p>Это тестовое сообщение.</p>').format( username=username_str) }, fdbw=fdbw) print(u'The greeting message has been sent to "{}"' .format(username_str))
def create_new_host_for_user(self, username): """ Implements the @abstractmethod from C{AbstractUsersEnabledPeerBookMixin}. @rtype: Host """ super(TrustedPeerBookInMemory, self).create_new_host_for_user(username) uuid_created = False while not uuid_created: host_uuid = gen_uuid() # Ensure it doesn't start with 0x00 byte uuid_created = (host_uuid.bytes[0] != '\x00') host = HostAtNode( uuid=host_uuid, urls=[], # Host-specific name=str(host_uuid), user=self.get_user_by_name(username), # HostAtNode-specific last_seen=None) self[host_uuid] = host return host
def add_group(arguments): node_map = proceed_with_node() if node_map is not None: if len(arguments) < 1: cli_error('You must pass at least 1 argument to this command: ' 'the name of the group.') else: groupname_str = arguments.popleft() # Parse group UUID (optional) try: group_uuid = UUID(arguments[0]) # Parsed successfully arguments.popleft() except (IndexError, ValueError): group_uuid = gen_uuid() user_group = UserGroup(uuid=UserGroupUUID.safe_cast_uuid( group_uuid), name=str(groupname_str), private=False, enc_key=gen_rand_key()) NodeApp.add_ugroup(user_group) print(u'Added group "{}" with UUID {}' .format(groupname_str, group_uuid))
def select_files(arguments): """ Generate digest for the username. """ if len(arguments) < 3: cli_error('The host UUID, the backup name, and at least one directory ' 'with the files should be passed!') else: my_uuid, ds_name = (try_parse_uuid(arguments.popleft()), arguments.popleft()) proceed_with_host_uuid_cli(my_uuid) paths = __get_all_remaining_args(arguments) if not paths: cli_error('No paths passed!') else: host_app = UHostApp(my_uuid, uhost_settings.detect_edition(), __create_chunk_storage()) ugroup_uuid = host_app.host.user.base_group.uuid path_map = {k: {'f+': ['all'], 'f-': [], 'stat': safe_stat(k)} for k in paths} host_app.select_paths_for_backup(ds_name=ds_name, ds_uuid=gen_uuid(), ugroup_uuid=ugroup_uuid, sync=False, paths_map=path_map)
def select_files(arguments): """ Generate digest for the username. """ if len(arguments) < 3: cli_error('The host UUID, the backup name, and at least one directory ' 'with the files should be passed!') else: my_uuid, ds_name = (try_parse_uuid(arguments.popleft()), arguments.popleft()) proceed_with_host_uuid_cli(my_uuid) paths = __get_all_remaining_args(arguments) if not paths: cli_error('No paths passed!') else: host_app = UHostApp(my_uuid, uhost_settings.detect_edition(), __create_chunk_storage()) ugroup_uuid = host_app.host.user.base_group.uuid path_map = { k: { 'f+': ['all'], 'f-': [], 'stat': safe_stat(k) } for k in paths } host_app.select_paths_for_backup(ds_name=ds_name, ds_uuid=gen_uuid(), ugroup_uuid=ugroup_uuid, sync=False, paths_map=path_map)
def _dummy_schedule_for_time(cls, dt): """ Create a dummy schedule item, with only the datetime field sensible, for various sorting purposes. @param dt: Datetime for the schedule item, assumed naive (but may be non-naive as well). @type dt: datetime """ naive_dt = dt if is_naive_dt(dt) \ else dt.astimezone(pytz.utc).replace(tzinfo=None) return cls(host_uuid=gen_uuid(), uuid=gen_uuid(), name='', period='once', next_backup_datetime=naive_dt, tz_info=pytz.utc, paths={})
def test_from_uuid(self): u = gen_uuid() u_as_a = TestChildA._from_uuid(u) u_as_aa = TestChildAA._from_uuid(u) u_as_b = TestChildB._from_uuid(u) u_as_bb = TestChildBB._from_uuid(u) self.assertEqual(all(i.hex == u.hex for i in (u_as_a, u_as_aa, u_as_b, u_as_bb)), True)
def __poll_restore_requests_in_thread(self): """Perform another iteration of polling the restore requests.""" assert not in_main_thread() poll_uuid = gen_uuid() logger.debug('Polling restore requests (%s)', poll_uuid) restore_request = True while restore_request is not None: with ds.FDB() as fdbw: restore_request = \ FDBQueries.RestoreRequests \ .atomic_start_oldest_restore_request(fdbw=fdbw) logger.debug('Poll (%s) returned %r', poll_uuid, restore_request) if restore_request is not None: # We've indeed have some restore request that needs processing. # Create new "virtual" dataset with all the data # to be restored. with db.RDB() as rdbw: new_ds_uuid = \ Queries.Datasets.restore_files_to_dataset_clone( restore_request.base_ds_uuid, restore_request.paths, restore_request.ts_start, rdbw) # Now we know the new dataset to be restored. # Btw, write it into the docstore. # Doesn't need to be atomic, as only a single node # may be processing it at a time. with ds.FDB() as fdbw: FDBQueries.RestoreRequests.set_ds_uuid( _id=restore_request._id, new_ds_uuid=new_ds_uuid, fdbw=fdbw) # After creating the dataset, let's restore it to all host # which are alive. _syncer = self.__server_process.app.syncer _syncer.restore_dataset_to_lacking_hosts( me=self.__server_process.me, host=None, ds_uuid=new_ds_uuid) logger.debug('Polling restore requests (%s) - done', poll_uuid)
def __create_some_dummy_chunks(self, how_many): """ Create some dummy chunks (in FS and in DB). Yields the UUIDs of dummy chunks after creation of each one. @param how_many: How many dummy chunks to create. @rtype: col.Iterable """ with self.__chunk_op_lock: logger.debug('Creating %i new dummy chunk(s)', how_many) # One by one, adding the dummy chunks for i in xrange(how_many): dummy_chunk_uuid = gen_uuid() HostQueries.HostChunks.create_dummy_chunk(dummy_chunk_uuid) fallocate(self.__get_chunk_file_path(dummy_chunk_uuid, is_dummy=True), 0x100000) yield dummy_chunk_uuid logger.debug('Created %i dummy chunk(s)', how_many)
def create_new_host_for_user(self, username): """ Implements the @abstractmethod from C{AbstractUsersEnabledPeerBookMixin}. @rtype: Host """ super(TrustedPeerBookInMemory, self).create_new_host_for_user(username) uuid_created = False while not uuid_created: host_uuid = gen_uuid() # Ensure it doesn't start with 0x00 byte uuid_created = (host_uuid.bytes[0] != '\x00') host = HostAtNode(uuid=host_uuid, urls=[], # Host-specific name=str(host_uuid), user=self.get_user_by_name(username), # HostAtNode-specific last_seen=None) self[host_uuid] = host return host
) # consists_of(_vfiles, RelVirtualFile) # Group RelVirtualFile's by rel_dir _files_grouped_by_rel_dir = ( (rvf for rvf in per_rel_dir) for rel_dir, per_rel_dir in sorted_groupby(_vfiles, attrgetter("rel_dir")) ) paths_map = { base_dir: { "ifiles": _files_grouped_by_rel_dir, "stat": fake_stat(isdir=True, atime=upload_time, mtime=upload_time, ctime=upload_time), } } ds_uuid = DatasetUUID.safe_cast_uuid(gen_uuid()) dataset = DatasetOnVirtualFiles.from_paths( ds_name, ds_uuid, group_uuid, sync, paths_map, upload_time, cryptographer ) assert dataset is not None thosts = list(TrustedQueries.HostAtNode.get_all_trusted_hosts(for_storage=True, rdbw=rdbw)) logger.debug("Uploading dataset %r,... like, to %r", dataset, thosts) # Use group_uuid as host_uuid fake_host_uuid = None dummy_ds_uuid = Queries.Datasets.create_dataset_for_backup(fake_host_uuid, dataset, rdbw)
for rel_dir, per_rel_dir in sorted_groupby(_vfiles, attrgetter('rel_dir'))) paths_map = { base_dir: { 'ifiles': _files_grouped_by_rel_dir, 'stat': fake_stat(isdir=True, atime=upload_time, mtime=upload_time, ctime=upload_time) } } ds_uuid = DatasetUUID.safe_cast_uuid(gen_uuid()) dataset = DatasetOnVirtualFiles.from_paths(ds_name, ds_uuid, group_uuid, sync, paths_map, upload_time, cryptographer) assert dataset is not None thosts = list( TrustedQueries.HostAtNode.get_all_trusted_hosts(for_storage=True, rdbw=rdbw)) logger.debug('Uploading dataset %r,... like, to %r', dataset, thosts) # Use group_uuid as host_uuid fake_host_uuid = None
def before_request(): g.request_id = gen_uuid()
def __backup_some_phys_files(self, base_dir, files, ugroup, __do_start_backup=True): r"""Given some files, create a new dataset and start to backup them. >>> # ugroup = UserGroup( >>> # uuid=UserGroupUUID('00000000-bbbb-0000-0000-000000000001'), >>> # name='AlphA', >>> # private=True, >>> # enc_key='\x01\xe6\x13\xdab)\xd2n\xd6\xafTH\x03h\x02\x12' >>> # '\x17D\x1a\xeb\x8b6\xc0\x9b\xa6\x7f\xcc\x06N\xcf' >>> # '\x8b\xcd' >>> # ) >>> # __backup_some_phys_files( >>> # base_dir='u'/home/john/FreeBrie', >>> # files=[ >>> # LocalPhysicalFileStateRel( >>> # rel_dir='', >>> # rel_file=u'f1.mp3', >>> # size=13829879, >>> # time_changed=datetime(2012, 11, 5, 12,12,41,904430)), >>> # LocalPhysicalFileStateRel( >>> # rel_dir='', >>> # rel_file=u'f2.avi', >>> # size=3522710, >>> # time_changed=datetime(2012, 11, 5, 12,12,41,988433)), >>> # LocalPhysicalFileStateRel( >>> # rel_dir=u'a/b', >>> # rel_file=u'bbb', >>> # size=4, >>> # time_changed=datetime(2012, 10, 11, 15 33 42 19808)), >>> # LocalPhysicalFileStateRel( >>> # rel_dir=u'a/b/c', >>> # rel_file=u'ccc', >>> # size=4, >>> # time_changed=datetime(2012, 10, 11, 15 33 41 979807)) >>> # ], >>> # ugroup=ugroup) @todo: complete the unit test, which is half-done! @param base_dir: the directory being backed up. @type base_dir: basestring @param files: the iterable over the files which should be backed up. Contains C{LocalPhysicalFileStateRel} objects. The caller should ensure that C{files} is non-empty! @type files: col.Iterable @type ugroup: UserGroup @return: the created dataset (if succeeded). @rtype: DatasetOnPhysicalFiles, NoneType """ logger.debug('__backup_some_phys_files(%r, %r)', base_dir, ugroup) # Group files by rel_dir; then ignore base_dir, # keep only rel_dir, rel_file, size and time_changed files_grouped_by_rel_dir = \ ((RelVirtualFile(rel_dir=f.rel_dir, filename=f.rel_file, # If we can read real stat, read it; # otherwise we'll emulate it with fake_stat stat=coalesce(os_ex.safe_stat( # real stat os.path.join(base_dir, f.rel_path)), os_ex.fake_stat( # deleted file st_mode=None, atime=f.time_changed, mtime=f.time_changed, ctime=f.time_changed, size=None)), stat_getter=lambda f=f: os_ex.safe_stat( os.path.join(base_dir, f.rel_path)), file_getter=lambda f=f: open(os.path.join(base_dir, f.rel_path), 'rb')) for f in per_rel_dir) for rel_dir, per_rel_dir in sorted_groupby(files, attrgetter('rel_dir'))) # Example: # files_grouped_by_rel_dir = [ # [ # RelVirtualFile(...), # RelVirtualFile(...), # RelVirtualFile(...) # ], # [ # RelVirtualFile(...), # [ # RelVirtualFile(...) # ] # ] _path_map = { base_dir: { 'ifiles': files_grouped_by_rel_dir, 'stat': os_ex.safe_stat(base_dir) } } ds_uuid = DatasetUUID.safe_cast_uuid(gen_uuid()) ds = self.select_paths_for_backup(ds_name='', ds_uuid=ds_uuid, ugroup_uuid=ugroup.uuid, sync=True, paths_map=_path_map) if ds is not None and __do_start_backup: self.start_backup(ds_uuid) return ds
def create_directory(base_dir, rel_path, ds_name, group_uuid, sync, cryptographer, rdbw, ospath=posixpath): """Create new directory in the cloud. @note: all the paths should be in POSIX format. @param base_dir: the base directory path (in the dataset) where to upload the file. @type base_dir: basestring @param rel_path: the name of directory which should be created. @type rel_path: basestring @param group_uuid: the UUID of the user group, for which the file should be bound. @type group_uuid: UserGroupUUID @param sync: whether the created dataset should be considered a "sync dataset". @type sync: bool @param rdbw: RelDB wrapper. @type rdbw: DatabaseWrapperSQLAlchemy @return: the UUID of newly created dataset. @rtype: DatasetUUID """ upload_time = datetime.utcnow() # For each FileToUpload, create fake stat dir_fake_stat = fake_stat(isdir=True, atime=upload_time, mtime=upload_time, ctime=upload_time, size=None) # Turn the original FileToUpload's to RelVirtualFile's _vfile = RelVirtualFile( rel_dir=ospath.dirname(rel_path), filename=ospath.basename(rel_path), stat=dir_fake_stat, stat_getter=lambda dir_fake_stat=dir_fake_stat: dir_fake_stat) # isinstance(ftu, FileToUpload) # Group RelVirtualFile's by rel_dir # _files_grouped_by_rel_dir = \ # ((rvf for rvf in per_rel_dir) # for rel_dir, per_rel_dir # in sorted_groupby(_vfiles, attrgetter('rel_dir'))) paths_map = { base_dir: { 'ifiles': [[_vfile]], 'stat': fake_stat(isdir=True, atime=upload_time, mtime=upload_time, ctime=upload_time) } } ds_uuid = DatasetUUID.safe_cast_uuid(gen_uuid()) dataset = DatasetOnVirtualFiles.from_paths(ds_name, ds_uuid, group_uuid, sync, paths_map, upload_time, cryptographer) assert dataset is not None # Use group_uuid as host_uuid fake_host_uuid = None dummy_ds_uuid = Queries.Datasets.create_dataset_for_backup( fake_host_uuid, dataset, rdbw) dataset.time_completed = datetime.utcnow() # That's all, folks! Queries.Datasets.update_dataset(fake_host_uuid, dataset, rdbw) return ds_uuid
print('For username {0}, the digest is {1}' .format(username, crypto.generate_digest( str(username), password, common_settings.HTTP_AUTH_REALM_NODE))) @cli_command( ('-dd', '--delete-dataset'), 'Delete a dataset from the node.\n' 'Arg. 1: base host UUID.\n' 'Arg. 2: dataset UUID.\n' 'Example:\n' ' {bin} {cmd} 00000000-1111-0000-0000-000000000001 ' + str(gen_uuid())) def delete_dataset(arguments): """ Delete the dataset from the Node. """ global _LOG_ACTIONS if len(arguments) < 2: cli_error('At least the host UUID and the dataset UUID ' 'should be passed!') else: my_uuid, ds_uuid = (try_parse_uuid(arguments.popleft()), try_parse_uuid(arguments.popleft())) print('Trying to delete the dataset {}'.format(ds_uuid))
def setUp(self): self.u = gen_uuid() self.child_a = TestChildA(self.u.hex) self.child_aa = TestChildAA(self.u.hex) self.child_b = TestChildB(self.u.hex)
'passed as the arguments!') else: username = arguments.popleft() password = _get_password_from_arguments(arguments) print('For username {0}, the digest is {1}'.format( username, crypto.generate_digest(str(username), password, common_settings.HTTP_AUTH_REALM_NODE))) @cli_command( ('-dd', '--delete-dataset'), 'Delete a dataset from the node.\n' 'Arg. 1: base host UUID.\n' 'Arg. 2: dataset UUID.\n' 'Example:\n' ' {bin} {cmd} 00000000-1111-0000-0000-000000000001 ' + str(gen_uuid())) def delete_dataset(arguments): """ Delete the dataset from the Node. """ global _LOG_ACTIONS if len(arguments) < 2: cli_error('At least the host UUID and the dataset UUID ' 'should be passed!') else: my_uuid, ds_uuid = (try_parse_uuid(arguments.popleft()), try_parse_uuid(arguments.popleft())) print('Trying to delete the dataset {}'.format(ds_uuid))
def __backup_some_phys_files(self, base_dir, files, ugroup, __do_start_backup=True): r"""Given some files, create a new dataset and start to backup them. >>> # ugroup = UserGroup( >>> # uuid=UserGroupUUID('00000000-bbbb-0000-0000-000000000001'), >>> # name='AlphA', >>> # private=True, >>> # enc_key='\x01\xe6\x13\xdab)\xd2n\xd6\xafTH\x03h\x02\x12' >>> # '\x17D\x1a\xeb\x8b6\xc0\x9b\xa6\x7f\xcc\x06N\xcf' >>> # '\x8b\xcd' >>> # ) >>> # __backup_some_phys_files( >>> # base_dir='u'/home/john/FreeBrie', >>> # files=[ >>> # LocalPhysicalFileStateRel( >>> # rel_dir='', >>> # rel_file=u'f1.mp3', >>> # size=13829879, >>> # time_changed=datetime(2012, 11, 5, 12,12,41,904430)), >>> # LocalPhysicalFileStateRel( >>> # rel_dir='', >>> # rel_file=u'f2.avi', >>> # size=3522710, >>> # time_changed=datetime(2012, 11, 5, 12,12,41,988433)), >>> # LocalPhysicalFileStateRel( >>> # rel_dir=u'a/b', >>> # rel_file=u'bbb', >>> # size=4, >>> # time_changed=datetime(2012, 10, 11, 15 33 42 19808)), >>> # LocalPhysicalFileStateRel( >>> # rel_dir=u'a/b/c', >>> # rel_file=u'ccc', >>> # size=4, >>> # time_changed=datetime(2012, 10, 11, 15 33 41 979807)) >>> # ], >>> # ugroup=ugroup) @todo: complete the unit test, which is half-done! @param base_dir: the directory being backed up. @type base_dir: basestring @param files: the iterable over the files which should be backed up. Contains C{LocalPhysicalFileStateRel} objects. The caller should ensure that C{files} is non-empty! @type files: col.Iterable @type ugroup: UserGroup @return: the created dataset (if succeeded). @rtype: DatasetOnPhysicalFiles, NoneType """ logger.debug('__backup_some_phys_files(%r, %r)', base_dir, ugroup) # Group files by rel_dir; then ignore base_dir, # keep only rel_dir, rel_file, size and time_changed files_grouped_by_rel_dir = \ ((RelVirtualFile(rel_dir=f.rel_dir, filename=f.rel_file, # If we can read real stat, read it; # otherwise we'll emulate it with fake_stat stat=coalesce(os_ex.safe_stat( # real stat os.path.join(base_dir, f.rel_path)), os_ex.fake_stat( # deleted file st_mode=None, atime=f.time_changed, mtime=f.time_changed, ctime=f.time_changed, size=None)), stat_getter=lambda f=f: os_ex.safe_stat( os.path.join(base_dir, f.rel_path)), file_getter=lambda f=f: open(os.path.join(base_dir, f.rel_path), 'rb')) for f in per_rel_dir) for rel_dir, per_rel_dir in sorted_groupby(files, attrgetter('rel_dir'))) # Example: # files_grouped_by_rel_dir = [ # [ # RelVirtualFile(...), # RelVirtualFile(...), # RelVirtualFile(...) # ], # [ # RelVirtualFile(...), # [ # RelVirtualFile(...) # ] # ] _path_map = {base_dir: {'ifiles': files_grouped_by_rel_dir, 'stat': os_ex.safe_stat(base_dir)}} ds_uuid = DatasetUUID.safe_cast_uuid(gen_uuid()) ds = self.select_paths_for_backup(ds_name='', ds_uuid=ds_uuid, ugroup_uuid=ugroup.uuid, sync=True, paths_map=_path_map) if ds is not None and __do_start_backup: self.start_backup(ds_uuid) return ds
def create_directory(base_dir, rel_path, ds_name, group_uuid, sync, cryptographer, rdbw, ospath=posixpath): """Create new directory in the cloud. @note: all the paths should be in POSIX format. @param base_dir: the base directory path (in the dataset) where to upload the file. @type base_dir: basestring @param rel_path: the name of directory which should be created. @type rel_path: basestring @param group_uuid: the UUID of the user group, for which the file should be bound. @type group_uuid: UserGroupUUID @param sync: whether the created dataset should be considered a "sync dataset". @type sync: bool @param rdbw: RelDB wrapper. @type rdbw: DatabaseWrapperSQLAlchemy @return: the UUID of newly created dataset. @rtype: DatasetUUID """ upload_time = datetime.utcnow() # For each FileToUpload, create fake stat dir_fake_stat = fake_stat(isdir=True, atime=upload_time, mtime=upload_time, ctime=upload_time, size=None) # Turn the original FileToUpload's to RelVirtualFile's _vfile = RelVirtualFile( rel_dir=ospath.dirname(rel_path), filename=ospath.basename(rel_path), stat=dir_fake_stat, stat_getter=lambda dir_fake_stat=dir_fake_stat: dir_fake_stat, ) # isinstance(ftu, FileToUpload) # Group RelVirtualFile's by rel_dir # _files_grouped_by_rel_dir = \ # ((rvf for rvf in per_rel_dir) # for rel_dir, per_rel_dir # in sorted_groupby(_vfiles, attrgetter('rel_dir'))) paths_map = { base_dir: { "ifiles": [[_vfile]], "stat": fake_stat(isdir=True, atime=upload_time, mtime=upload_time, ctime=upload_time), } } ds_uuid = DatasetUUID.safe_cast_uuid(gen_uuid()) dataset = DatasetOnVirtualFiles.from_paths( ds_name, ds_uuid, group_uuid, sync, paths_map, upload_time, cryptographer ) assert dataset is not None # Use group_uuid as host_uuid fake_host_uuid = None dummy_ds_uuid = Queries.Datasets.create_dataset_for_backup(fake_host_uuid, dataset, rdbw) dataset.time_completed = datetime.utcnow() # That's all, folks! Queries.Datasets.update_dataset(fake_host_uuid, dataset, rdbw) return ds_uuid