def from_bson(cls, doc):
r"""
>>> from datetime import datetime
>>> BackupTransactionState_Node.from_bson({
... 'dataset_uuid': UUID('cf9a54b1-1239-48de-9b25-d7fa927db125')
... })(tr_start_time=datetime(2012, 9, 26, 14, 29, 48, 877434),
... tr_uuid=UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
... tr_src_uuid=UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
... tr_dst_uuid=UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d')
... ) # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE
BackupTransactionState_Node(tr_start_time=....datetime(2012, 9, 26,
14, 29, 48,
877434),
tr_uuid=UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
tr_src_uuid=UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
tr_dst_uuid=UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'),
dataset_uuid=DatasetUUID('cf9a54b1-1239-48de-9b25-d7fa927db125'))
"""
assert cls.validate_schema(doc), repr(doc)
return partial(super(BackupTransactionState_Node, cls).from_bson(doc),
# Mandatory
dataset_uuid=DatasetUUID.safe_cast_uuid(
doc['dataset_uuid']),
# Optional
ack_result_code=doc.get('ack_result_code'))
def from_bson(cls, doc):
r"""
>>> from datetime import datetime
>>> BackupTransactionState_Node.from_bson({
... 'dataset_uuid': UUID('cf9a54b1-1239-48de-9b25-d7fa927db125')
... })(tr_start_time=datetime(2012, 9, 26, 14, 29, 48, 877434),
... tr_uuid=UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
... tr_src_uuid=UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
... tr_dst_uuid=UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d')
... ) # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE
BackupTransactionState_Node(tr_start_time=....datetime(2012, 9, 26,
14, 29, 48,
877434),
tr_uuid=UUID('1a82a181-741d-4a64-86e5-77a7dd000ba2'),
tr_src_uuid=UUID('fa87ebfd-d498-4ba6-9f04-a933e4512b24'),
tr_dst_uuid=UUID('e6aa4157-ee8a-449e-a2d5-3340a59e717d'),
dataset_uuid=DatasetUUID('cf9a54b1-1239-48de-9b25-d7fa927db125'))
"""
assert cls.validate_schema(doc), repr(doc)
return partial(
super(BackupTransactionState_Node, cls).from_bson(doc),
# Mandatory
dataset_uuid=DatasetUUID.safe_cast_uuid(doc['dataset_uuid']),
# Optional
ack_result_code=doc.get('ack_result_code'))
def _init_from_body(self, body):
r"""
>>> from uuid import UUID
>>> from common.inhabitants import Host, Node
>>> host = Host(uuid=UUID('00000000-7606-420c-8a98-a6c8728ac98d'))
>>> node = Node(uuid=UUID('11111111-79b4-49e0-b72e-8c4ce6a2aed9'))
>>> msg0 = BackupMessage(
... src=node,
... dst=host,
... uuid=UUID('4ac2536a-4a1e-4b08-ad4e-11bd675fdf15')
... )
>>> BackupMessage._init_from_body._without_bunzip2._without_json_loads(
... msg0, {})
Traceback (most recent call last):
...
Exception: Unparseable backup message body: {}
>>> msg1 = BackupMessage(
... src=node,
... dst=host,
... uuid=UUID('baebd0f2-fc58-417b-97ee-08a892cd5a8f'),
... status_code=1
... )
>>> BackupMessage._init_from_body._without_bunzip2._without_json_loads(
... msg1, {'dataset_uuid': '60fb966767ae4e3c9c86edac180dd4e6'})
>>> msg1 # doctest:+ELLIPSIS,+NORMALIZE_WHITESPACE
BackupMessage(src=Node(uuid=PeerUUID('11111111-...-8c4ce6a2aed9')),
dst=Host(uuid=PeerUUID('00000000-7606-420c-8a98-a6c8728ac98d')),
uuid=MessageUUID('baebd0f2-fc58-417b-97ee-08a892cd5a8f'),
status_code=1
[
dataset_uuid=DatasetUUID('60fb9667-67ae-4e3c-9c86-edac180dd4e6')
])
"""
# N2H
if 'dataset_uuid' in body:
self.dataset_uuid = DatasetUUID(body['dataset_uuid'])
else:
raise Exception('Unparseable backup message body: {!r}'
.format(body))
def __backup_some_phys_files(self, base_dir, files, ugroup,
__do_start_backup=True):
r"""Given some files, create a new dataset and start to backup them.
>>> # ugroup = UserGroup(
>>> # uuid=UserGroupUUID('00000000-bbbb-0000-0000-000000000001'),
>>> # name='AlphA',
>>> # private=True,
>>> # enc_key='\x01\xe6\x13\xdab)\xd2n\xd6\xafTH\x03h\x02\x12'
>>> # '\x17D\x1a\xeb\x8b6\xc0\x9b\xa6\x7f\xcc\x06N\xcf'
>>> # '\x8b\xcd'
>>> # )
>>> # __backup_some_phys_files(
>>> # base_dir='u'/home/john/FreeBrie',
>>> # files=[
>>> # LocalPhysicalFileStateRel(
>>> # rel_dir='',
>>> # rel_file=u'f1.mp3',
>>> # size=13829879,
>>> # time_changed=datetime(2012, 11, 5, 12,12,41,904430)),
>>> # LocalPhysicalFileStateRel(
>>> # rel_dir='',
>>> # rel_file=u'f2.avi',
>>> # size=3522710,
>>> # time_changed=datetime(2012, 11, 5, 12,12,41,988433)),
>>> # LocalPhysicalFileStateRel(
>>> # rel_dir=u'a/b',
>>> # rel_file=u'bbb',
>>> # size=4,
>>> # time_changed=datetime(2012, 10, 11, 15 33 42 19808)),
>>> # LocalPhysicalFileStateRel(
>>> # rel_dir=u'a/b/c',
>>> # rel_file=u'ccc',
>>> # size=4,
>>> # time_changed=datetime(2012, 10, 11, 15 33 41 979807))
>>> # ],
>>> # ugroup=ugroup)
@todo: complete the unit test, which is half-done!
@param base_dir: the directory being backed up.
@type base_dir: basestring
@param files: the iterable over the files which should be backed up.
Contains C{LocalPhysicalFileStateRel} objects.
The caller should ensure that C{files} is non-empty!
@type files: col.Iterable
@type ugroup: UserGroup
@return: the created dataset (if succeeded).
@rtype: DatasetOnPhysicalFiles, NoneType
"""
logger.debug('__backup_some_phys_files(%r, %r)',
base_dir, ugroup)
# Group files by rel_dir; then ignore base_dir,
# keep only rel_dir, rel_file, size and time_changed
files_grouped_by_rel_dir = \
((RelVirtualFile(rel_dir=f.rel_dir,
filename=f.rel_file,
# If we can read real stat, read it;
# otherwise we'll emulate it with fake_stat
stat=coalesce(os_ex.safe_stat( # real stat
os.path.join(base_dir,
f.rel_path)),
os_ex.fake_stat( # deleted file
st_mode=None,
atime=f.time_changed,
mtime=f.time_changed,
ctime=f.time_changed,
size=None)),
stat_getter=lambda f=f:
os_ex.safe_stat(
os.path.join(base_dir,
f.rel_path)),
file_getter=lambda f=f:
open(os.path.join(base_dir,
f.rel_path),
'rb'))
for f in per_rel_dir)
for rel_dir, per_rel_dir
in sorted_groupby(files, attrgetter('rel_dir')))
# Example:
# files_grouped_by_rel_dir = [
# [
# RelVirtualFile(...),
# RelVirtualFile(...),
# RelVirtualFile(...)
# ],
# [
# RelVirtualFile(...),
# [
# RelVirtualFile(...)
# ]
# ]
_path_map = {base_dir: {'ifiles': files_grouped_by_rel_dir,
'stat': os_ex.safe_stat(base_dir)}}
ds_uuid = DatasetUUID.safe_cast_uuid(gen_uuid())
ds = self.select_paths_for_backup(ds_name='',
ds_uuid=ds_uuid,
ugroup_uuid=ugroup.uuid,
sync=True,
paths_map=_path_map)
if ds is not None and __do_start_backup:
self.start_backup(ds_uuid)
return ds
def __backup_some_phys_files(self,
base_dir,
files,
ugroup,
__do_start_backup=True):
r"""Given some files, create a new dataset and start to backup them.
>>> # ugroup = UserGroup(
>>> # uuid=UserGroupUUID('00000000-bbbb-0000-0000-000000000001'),
>>> # name='AlphA',
>>> # private=True,
>>> # enc_key='\x01\xe6\x13\xdab)\xd2n\xd6\xafTH\x03h\x02\x12'
>>> # '\x17D\x1a\xeb\x8b6\xc0\x9b\xa6\x7f\xcc\x06N\xcf'
>>> # '\x8b\xcd'
>>> # )
>>> # __backup_some_phys_files(
>>> # base_dir='u'/home/john/FreeBrie',
>>> # files=[
>>> # LocalPhysicalFileStateRel(
>>> # rel_dir='',
>>> # rel_file=u'f1.mp3',
>>> # size=13829879,
>>> # time_changed=datetime(2012, 11, 5, 12,12,41,904430)),
>>> # LocalPhysicalFileStateRel(
>>> # rel_dir='',
>>> # rel_file=u'f2.avi',
>>> # size=3522710,
>>> # time_changed=datetime(2012, 11, 5, 12,12,41,988433)),
>>> # LocalPhysicalFileStateRel(
>>> # rel_dir=u'a/b',
>>> # rel_file=u'bbb',
>>> # size=4,
>>> # time_changed=datetime(2012, 10, 11, 15 33 42 19808)),
>>> # LocalPhysicalFileStateRel(
>>> # rel_dir=u'a/b/c',
>>> # rel_file=u'ccc',
>>> # size=4,
>>> # time_changed=datetime(2012, 10, 11, 15 33 41 979807))
>>> # ],
>>> # ugroup=ugroup)
@todo: complete the unit test, which is half-done!
@param base_dir: the directory being backed up.
@type base_dir: basestring
@param files: the iterable over the files which should be backed up.
Contains C{LocalPhysicalFileStateRel} objects.
The caller should ensure that C{files} is non-empty!
@type files: col.Iterable
@type ugroup: UserGroup
@return: the created dataset (if succeeded).
@rtype: DatasetOnPhysicalFiles, NoneType
"""
logger.debug('__backup_some_phys_files(%r, %r)', base_dir, ugroup)
# Group files by rel_dir; then ignore base_dir,
# keep only rel_dir, rel_file, size and time_changed
files_grouped_by_rel_dir = \
((RelVirtualFile(rel_dir=f.rel_dir,
filename=f.rel_file,
# If we can read real stat, read it;
# otherwise we'll emulate it with fake_stat
stat=coalesce(os_ex.safe_stat( # real stat
os.path.join(base_dir,
f.rel_path)),
os_ex.fake_stat( # deleted file
st_mode=None,
atime=f.time_changed,
mtime=f.time_changed,
ctime=f.time_changed,
size=None)),
stat_getter=lambda f=f:
os_ex.safe_stat(
os.path.join(base_dir,
f.rel_path)),
file_getter=lambda f=f:
open(os.path.join(base_dir,
f.rel_path),
'rb'))
for f in per_rel_dir)
for rel_dir, per_rel_dir
in sorted_groupby(files, attrgetter('rel_dir')))
# Example:
# files_grouped_by_rel_dir = [
# [
# RelVirtualFile(...),
# RelVirtualFile(...),
# RelVirtualFile(...)
# ],
# [
# RelVirtualFile(...),
# [
# RelVirtualFile(...)
# ]
# ]
_path_map = {
base_dir: {
'ifiles': files_grouped_by_rel_dir,
'stat': os_ex.safe_stat(base_dir)
}
}
ds_uuid = DatasetUUID.safe_cast_uuid(gen_uuid())
ds = self.select_paths_for_backup(ds_name='',
ds_uuid=ds_uuid,
ugroup_uuid=ugroup.uuid,
sync=True,
paths_map=_path_map)
if ds is not None and __do_start_backup:
self.start_backup(ds_uuid)
return ds
def __restore_files(self):
"""Internal procedure which actually restores the files.
@todo: the Fingerprint calculation should be turned into
"file is read by blocks and then repacked into 16KiB segments";
then recalculation of the fingerprint in case of FP mismatch
won't be needed.
"""
_message = self.message
my_host = self.manager.app.host
feature_set = self.manager.app.feature_set
ds = _message.dataset
wr_uuid = _message.wr_uuid
ugroup = _message.ugroup
restore_directory = self.__get_restore_directory()
assert _message.sync == (wr_uuid is None) == (ds.uuid is not None), \
(_message.sync, wr_uuid, ds)
base_dir_id = None # will be used later
if restore_directory is None:
logger.error('Do not know the restore directory')
else:
logger.debug('Going to restore dataset %r for %r to %r',
ds, ugroup, restore_directory)
if not os.path.exists(restore_directory):
os.makedirs(restore_directory)
group_key = ugroup.enc_key if feature_set.per_group_encryption \
else None
cryptographer = Cryptographer(group_key=group_key,
key_generator=None)
is_whole_dataset_restored = _message.sync
logger.debug('Restoring %s files from dataset: %r',
'all' if is_whole_dataset_restored else 'selected',
coalesce(ds, 'N/A'))
# TODO: use the "delete time" from the LocalPhysicalFileState!
_now = datetime.utcnow()
# If we are syncing-in the whole dataset, we should write it
# into the DB as a whole. The files/file_locals will be bound to it
# so that after restore, we'll know on this Host that these states
# are fully synced to the cloud already (in fact, they came
# from the cloud).
if _message.sync:
# Let's hack into the files and substitute the base_dir.
# TODO: do it better!
for f in _message.files.iterkeys():
f.base_dir = restore_directory
# Write the whole dataset to the DB
_small_files = _message.files.keys() # not iterkeys(0 for now!
_dirs = {restore_directory: (_small_files, [])}
# Given the information in the inbound message about
# the whole dataset, store this dataset in the DB.
dataset = DatasetWithDirectories(
name=ds.name,
sync=ds.sync,
directories=_dirs,
# TODO: transport real data
# from the node
uuid=DatasetUUID.safe_cast_uuid(ds.uuid),
ugroup_uuid=UserGroupUUID.safe_cast_uuid(
ugroup.uuid),
time_started=ds.time_started,
time_completed=_now)
with db.RDB() as rdbw:
# Do we already have the dataset?
_ds_in_progress = \
HostQueries.HostDatasets.get_my_ds_in_progress(
host_uuid=my_host.uuid,
ds_uuid=dataset.uuid,
rdbw=rdbw)
if _ds_in_progress is None:
# We don't have it, insert.
dummy_ds_uuid = \
HostQueries.HostDatasets.create_dataset_for_backup(
my_host.uuid, dataset, rdbw)
assert dummy_ds_uuid == dataset.uuid, \
(dummy_ds_uuid, dataset.uuid)
base_dir_id = \
HostQueries.HostFiles.add_or_get_base_directory(
restore_directory, ugroup.uuid, rdbw)
error_in_any_file_occured = False
#
# Finally, loop over the files and restore each one
#
for file_, file_blocks in _message.files.iteritems():
self.__restore_op_for_path(file_, file_blocks,
is_whole_dataset_restored,
base_dir_id,
restore_directory,
cryptographer,
ds)
# Loop over the files completed
if is_whole_dataset_restored:
logger.debug('Restoring %r completed, there were %s issues.',
ds,
'some' if error_in_any_file_occured else 'no')
if not error_in_any_file_occured:
with db.RDB() as rdbw:
logger.debug('Updating %r at host %s...',
ds, my_host.uuid)
ds_to_finish = \
Queries.Datasets.get_dataset_by_uuid(ds.uuid,
my_host.uuid,
rdbw)
ds_to_finish.time_completed = datetime.utcnow()
logger.debug('Updating %r as completed', dataset)
# Mark the current dataset as completed
# only after the response from the node is received.
Queries.Datasets.update_dataset(my_host.uuid,
ds_to_finish,
rdbw)
# Everything seems ok to this moment
with self.open_state(for_update=True) as state:
state.ack_result_code = RestoreMessage.ResultCodes.OK
def __restore_files(self):
"""Internal procedure which actually restores the files.
@todo: the Fingerprint calculation should be turned into
"file is read by blocks and then repacked into 16KiB segments";
then recalculation of the fingerprint in case of FP mismatch
won't be needed.
"""
_message = self.message
my_host = self.manager.app.host
feature_set = self.manager.app.feature_set
ds = _message.dataset
wr_uuid = _message.wr_uuid
ugroup = _message.ugroup
restore_directory = self.__get_restore_directory()
assert _message.sync == (wr_uuid is None) == (ds.uuid is not None), \
(_message.sync, wr_uuid, ds)
base_dir_id = None # will be used later
if restore_directory is None:
logger.error('Do not know the restore directory')
else:
logger.debug('Going to restore dataset %r for %r to %r', ds,
ugroup, restore_directory)
if not os.path.exists(restore_directory):
os.makedirs(restore_directory)
group_key = ugroup.enc_key if feature_set.per_group_encryption \
else None
cryptographer = Cryptographer(group_key=group_key,
key_generator=None)
is_whole_dataset_restored = _message.sync
logger.debug('Restoring %s files from dataset: %r',
'all' if is_whole_dataset_restored else 'selected',
coalesce(ds, 'N/A'))
# TODO: use the "delete time" from the LocalPhysicalFileState!
_now = datetime.utcnow()
# If we are syncing-in the whole dataset, we should write it
# into the DB as a whole. The files/file_locals will be bound to it
# so that after restore, we'll know on this Host that these states
# are fully synced to the cloud already (in fact, they came
# from the cloud).
if _message.sync:
# Let's hack into the files and substitute the base_dir.
# TODO: do it better!
for f in _message.files.iterkeys():
f.base_dir = restore_directory
# Write the whole dataset to the DB
_small_files = _message.files.keys() # not iterkeys(0 for now!
_dirs = {restore_directory: (_small_files, [])}
# Given the information in the inbound message about
# the whole dataset, store this dataset in the DB.
dataset = DatasetWithDirectories(
name=ds.name,
sync=ds.sync,
directories=_dirs,
# TODO: transport real data
# from the node
uuid=DatasetUUID.safe_cast_uuid(ds.uuid),
ugroup_uuid=UserGroupUUID.safe_cast_uuid(ugroup.uuid),
time_started=ds.time_started,
time_completed=_now)
with db.RDB() as rdbw:
# Do we already have the dataset?
_ds_in_progress = \
HostQueries.HostDatasets.get_my_ds_in_progress(
host_uuid=my_host.uuid,
ds_uuid=dataset.uuid,
rdbw=rdbw)
if _ds_in_progress is None:
# We don't have it, insert.
dummy_ds_uuid = \
HostQueries.HostDatasets.create_dataset_for_backup(
my_host.uuid, dataset, rdbw)
assert dummy_ds_uuid == dataset.uuid, \
(dummy_ds_uuid, dataset.uuid)
base_dir_id = \
HostQueries.HostFiles.add_or_get_base_directory(
restore_directory, ugroup.uuid, rdbw)
error_in_any_file_occured = False
#
# Finally, loop over the files and restore each one
#
for file_, file_blocks in _message.files.iteritems():
self.__restore_op_for_path(file_, file_blocks,
is_whole_dataset_restored,
base_dir_id, restore_directory,
cryptographer, ds)
# Loop over the files completed
if is_whole_dataset_restored:
logger.debug('Restoring %r completed, there were %s issues.',
ds, 'some' if error_in_any_file_occured else 'no')
if not error_in_any_file_occured:
with db.RDB() as rdbw:
logger.debug('Updating %r at host %s...', ds,
my_host.uuid)
ds_to_finish = \
Queries.Datasets.get_dataset_by_uuid(ds.uuid,
my_host.uuid,
rdbw)
ds_to_finish.time_completed = datetime.utcnow()
logger.debug('Updating %r as completed', dataset)
# Mark the current dataset as completed
# only after the response from the node is received.
Queries.Datasets.update_dataset(
my_host.uuid, ds_to_finish, rdbw)
# Everything seems ok to this moment
with self.open_state(for_update=True) as state:
state.ack_result_code = RestoreMessage.ResultCodes.OK
def create_directory(base_dir, rel_path, ds_name, group_uuid, sync, cryptographer, rdbw, ospath=posixpath):
"""Create new directory in the cloud.
@note: all the paths should be in POSIX format.
@param base_dir: the base directory path (in the dataset) where to upload
the file.
@type base_dir: basestring
@param rel_path: the name of directory which should be created.
@type rel_path: basestring
@param group_uuid: the UUID of the user group, for which the file
should be bound.
@type group_uuid: UserGroupUUID
@param sync: whether the created dataset should be considered a
"sync dataset".
@type sync: bool
@param rdbw: RelDB wrapper.
@type rdbw: DatabaseWrapperSQLAlchemy
@return: the UUID of newly created dataset.
@rtype: DatasetUUID
"""
upload_time = datetime.utcnow()
# For each FileToUpload, create fake stat
dir_fake_stat = fake_stat(isdir=True, atime=upload_time, mtime=upload_time, ctime=upload_time, size=None)
# Turn the original FileToUpload's to RelVirtualFile's
_vfile = RelVirtualFile(
rel_dir=ospath.dirname(rel_path),
filename=ospath.basename(rel_path),
stat=dir_fake_stat,
stat_getter=lambda dir_fake_stat=dir_fake_stat: dir_fake_stat,
)
# isinstance(ftu, FileToUpload)
# Group RelVirtualFile's by rel_dir
# _files_grouped_by_rel_dir = \
# ((rvf for rvf in per_rel_dir)
# for rel_dir, per_rel_dir
# in sorted_groupby(_vfiles, attrgetter('rel_dir')))
paths_map = {
base_dir: {
"ifiles": [[_vfile]],
"stat": fake_stat(isdir=True, atime=upload_time, mtime=upload_time, ctime=upload_time),
}
}
ds_uuid = DatasetUUID.safe_cast_uuid(gen_uuid())
dataset = DatasetOnVirtualFiles.from_paths(
ds_name, ds_uuid, group_uuid, sync, paths_map, upload_time, cryptographer
)
assert dataset is not None
# Use group_uuid as host_uuid
fake_host_uuid = None
dummy_ds_uuid = Queries.Datasets.create_dataset_for_backup(fake_host_uuid, dataset, rdbw)
dataset.time_completed = datetime.utcnow()
# That's all, folks!
Queries.Datasets.update_dataset(fake_host_uuid, dataset, rdbw)
return ds_uuid
)
# consists_of(_vfiles, RelVirtualFile)
# Group RelVirtualFile's by rel_dir
_files_grouped_by_rel_dir = (
(rvf for rvf in per_rel_dir) for rel_dir, per_rel_dir in sorted_groupby(_vfiles, attrgetter("rel_dir"))
)
paths_map = {
base_dir: {
"ifiles": _files_grouped_by_rel_dir,
"stat": fake_stat(isdir=True, atime=upload_time, mtime=upload_time, ctime=upload_time),
}
}
ds_uuid = DatasetUUID.safe_cast_uuid(gen_uuid())
dataset = DatasetOnVirtualFiles.from_paths(
ds_name, ds_uuid, group_uuid, sync, paths_map, upload_time, cryptographer
)
assert dataset is not None
thosts = list(TrustedQueries.HostAtNode.get_all_trusted_hosts(for_storage=True, rdbw=rdbw))
logger.debug("Uploading dataset %r,... like, to %r", dataset, thosts)
# Use group_uuid as host_uuid
fake_host_uuid = None
dummy_ds_uuid = Queries.Datasets.create_dataset_for_backup(fake_host_uuid, dataset, rdbw)
def create_directory(base_dir,
rel_path,
ds_name,
group_uuid,
sync,
cryptographer,
rdbw,
ospath=posixpath):
"""Create new directory in the cloud.
@note: all the paths should be in POSIX format.
@param base_dir: the base directory path (in the dataset) where to upload
the file.
@type base_dir: basestring
@param rel_path: the name of directory which should be created.
@type rel_path: basestring
@param group_uuid: the UUID of the user group, for which the file
should be bound.
@type group_uuid: UserGroupUUID
@param sync: whether the created dataset should be considered a
"sync dataset".
@type sync: bool
@param rdbw: RelDB wrapper.
@type rdbw: DatabaseWrapperSQLAlchemy
@return: the UUID of newly created dataset.
@rtype: DatasetUUID
"""
upload_time = datetime.utcnow()
# For each FileToUpload, create fake stat
dir_fake_stat = fake_stat(isdir=True,
atime=upload_time,
mtime=upload_time,
ctime=upload_time,
size=None)
# Turn the original FileToUpload's to RelVirtualFile's
_vfile = RelVirtualFile(
rel_dir=ospath.dirname(rel_path),
filename=ospath.basename(rel_path),
stat=dir_fake_stat,
stat_getter=lambda dir_fake_stat=dir_fake_stat: dir_fake_stat)
# isinstance(ftu, FileToUpload)
# Group RelVirtualFile's by rel_dir
# _files_grouped_by_rel_dir = \
# ((rvf for rvf in per_rel_dir)
# for rel_dir, per_rel_dir
# in sorted_groupby(_vfiles, attrgetter('rel_dir')))
paths_map = {
base_dir: {
'ifiles': [[_vfile]],
'stat':
fake_stat(isdir=True,
atime=upload_time,
mtime=upload_time,
ctime=upload_time)
}
}
ds_uuid = DatasetUUID.safe_cast_uuid(gen_uuid())
dataset = DatasetOnVirtualFiles.from_paths(ds_name, ds_uuid, group_uuid,
sync, paths_map, upload_time,
cryptographer)
assert dataset is not None
# Use group_uuid as host_uuid
fake_host_uuid = None
dummy_ds_uuid = Queries.Datasets.create_dataset_for_backup(
fake_host_uuid, dataset, rdbw)
dataset.time_completed = datetime.utcnow()
# That's all, folks!
Queries.Datasets.update_dataset(fake_host_uuid, dataset, rdbw)
return ds_uuid
for rel_dir, per_rel_dir
in sorted_groupby(_vfiles, attrgetter('rel_dir')))
paths_map = {
base_dir: {
'ifiles':
_files_grouped_by_rel_dir,
'stat':
fake_stat(isdir=True,
atime=upload_time,
mtime=upload_time,
ctime=upload_time)
}
}
ds_uuid = DatasetUUID.safe_cast_uuid(gen_uuid())
dataset = DatasetOnVirtualFiles.from_paths(ds_name, ds_uuid, group_uuid,
sync, paths_map, upload_time,
cryptographer)
assert dataset is not None
thosts = list(
TrustedQueries.HostAtNode.get_all_trusted_hosts(for_storage=True,
rdbw=rdbw))
logger.debug('Uploading dataset %r,... like, to %r', dataset, thosts)
# Use group_uuid as host_uuid
fake_host_uuid = None