def test_bootstrap_picks_up_user(self):
user = self.obj_factory.make_user()
TransactionLog.bootstrap(user)
txlog = get_filesync_store().find(TransactionLog, op_type=TransactionLog.OP_USER_CREATED).one()
self.assertTxLogDetailsMatchesUserDetails(user, txlog)
def test_bootstrap_picks_up_user(self):
user = self.obj_factory.make_user()
TransactionLog.bootstrap(user)
txlog = get_shard_store(user.shard_id).find(
TransactionLog, op_type=TransactionLog.OP_USER_CREATED).one()
self.assertTxLogDetailsMatchesUserDetails(user, txlog)
def test_bootstrap_picks_up_only_files_owned_by_the_given_user(self):
user = self.obj_factory.make_user(user_id=1)
photos = self._create_files_for_user(user, "image/jpeg")
# These files do not belong to the user we're bootstrapping now, so
# they won't show up on the TXLog.
self._create_files_for_user(self.obj_factory.make_user(user_id=2), "image/jpeg")
TransactionLog.bootstrap(user)
self.assertBootstrappingPickedUpFiles(photos)
def test_bootstrap_picks_up_only_live_files(self):
user = self.obj_factory.make_user()
photos = self._create_files_for_user(user, u'image/jpeg')
# Even though all files in this second UDF are dead, the UDF itself is
# alive so we will have a txlog for it.
self._create_files_for_user(user, u'image/jpeg', status=u'Dead')
TransactionLog.bootstrap(user)
self.assertBootstrappingPickedUpFiles(photos)
def test_bootstrap_picks_up_only_live_files(self):
user = self.obj_factory.make_user()
photos = self._create_files_for_user(user, u'image/jpeg')
# Even though all files in this second UDF are dead, the UDF itself is
# alive so we will have a txlog for it.
self._create_files_for_user(user, u'image/jpeg', status=u'Dead')
TransactionLog.bootstrap(user)
self.assertBootstrappingPickedUpFiles(photos)
def test_bootstrap_picks_up_public_folders(self):
with txlog_disabled():
user = self.obj_factory.make_user()
public_dir = self.obj_factory.make_directory(user, public=True)
self.obj_factory.make_directory(user)
public_url = get_public_file_url(public_dir)
self.assertIsNotNone(public_url)
TransactionLog.bootstrap(user)
self.assertBootstrappingPickedUpFolders([public_dir])
def test_bootstrap_picks_up_only_live_udfs(self):
user = self.obj_factory.make_user()
with txlog_disabled():
root_udf = UserVolume.get_root(self.sstore, user.id)
live_udf = self.obj_factory.make_udf(user=user)
live_udf2 = self.obj_factory.make_udf(user=user)
self.obj_factory.make_udf(user=user, status=STATUS_DEAD)
TransactionLog.bootstrap(user)
self.assertBootstrappingPickedUpUDFs([root_udf, live_udf, live_udf2])
def test_bootstrap_picks_up_only_live_udfs(self):
user = self.obj_factory.make_user()
with txlog_disabled():
root_udf = UserVolume.get_root(self.sstore, user.id)
live_udf = self.obj_factory.make_udf(user=user)
live_udf2 = self.obj_factory.make_udf(user=user)
self.obj_factory.make_udf(user=user, status=STATUS_DEAD)
TransactionLog.bootstrap(user)
self.assertBootstrappingPickedUpUDFs([root_udf, live_udf, live_udf2])
def test_bootstrap_picks_up_public_folders(self):
with txlog_disabled():
user = self.obj_factory.make_user()
public_dir = self.obj_factory.make_directory(user, public=True)
self.obj_factory.make_directory(user)
public_url = get_public_file_url(public_dir)
self.assertIsNotNone(public_url)
TransactionLog.bootstrap(user)
self.assertBootstrappingPickedUpFolders([public_dir])
def test_bootstrap_picks_up_only_files_owned_by_the_given_user(self):
user = self.obj_factory.make_user(user_id=1)
photos = self._create_files_for_user(user, u'image/jpeg')
# These files do not belong to the user we're bootstrapping now, so
# they won't show up on the TXLog.
self._create_files_for_user(self.obj_factory.make_user(user_id=2),
u'image/jpeg')
TransactionLog.bootstrap(user)
self.assertBootstrappingPickedUpFiles(photos)
def test_bootstrap_picks_up_shares(self):
user = self.obj_factory.make_user()
directory = self.obj_factory.make_directory(user)
share = self.obj_factory.make_share(directory)
self.store.commit()
TransactionLog.bootstrap(user)
txlog = get_filesync_store().find(TransactionLog, op_type=TransactionLog.OP_SHARE_ACCEPTED).one()
expected_attrs = self._get_dict_with_txlog_attrs_from_share(share, directory, TransactionLog.OP_SHARE_ACCEPTED)
self.assert_txlog_correct(txlog, expected_attrs)
def test_bootstrap_picks_up_only_files_in_live_udfs(self):
user = self.obj_factory.make_user()
root_udf = UserVolume.get_root(self.store, user.id)
photo_in_root = self.obj_factory.make_file(user, root_udf.root_node, "foo.jpg", "image/jpeg")
dead_udf = self.obj_factory.make_udf(user=user)
self.obj_factory.make_file(user, dead_udf.root_node, "foo-in-dead-udf.jpg", "image/jpeg")
dead_udf.delete()
self.clear_txlogs()
TransactionLog.bootstrap(user)
self.assertBootstrappingPickedUpFiles([photo_in_root])
def test_bootstrap_picks_up_only_folders_in_live_udfs(self):
user = self.obj_factory.make_user()
root_udf = UserVolume.get_root(self.store, user.id)
folder_in_root = self.obj_factory.make_directory(user, root_udf.root_node, "folder1", public=True)
dead_udf = self.obj_factory.make_udf(user=user)
self.obj_factory.make_directory(user, dead_udf.root_node, "folder2", public=True)
dead_udf.delete()
self.clear_txlogs()
TransactionLog.bootstrap(user)
self.assertBootstrappingPickedUpFolders([folder_in_root])
def test_bootstrap_picks_up_shares(self):
user = self.obj_factory.make_user()
directory = self.obj_factory.make_directory(user)
share = self.obj_factory.make_share(directory)
self.sstore.commit()
TransactionLog.bootstrap(user)
txlog = get_shard_store(user.shard_id).find(
TransactionLog, op_type=TransactionLog.OP_SHARE_ACCEPTED).one()
expected_attrs = self._get_dict_with_txlog_attrs_from_share(
share, directory, TransactionLog.OP_SHARE_ACCEPTED)
self.assert_txlog_correct(txlog, expected_attrs)
def make_transaction_log(self, tx_id=None, timestamp=None, owner_id=1,
op_type=TransactionLog.OP_DELETE):
"""Create a transaction log."""
txlog = TransactionLog(
uuid.uuid4(), owner_id, uuid.uuid4(), op_type, u"",
u"text/plain", generation=1, old_path=u"",
extra_data=u"")
if timestamp:
txlog.timestamp = timestamp
if tx_id:
txlog.id = tx_id
self.store.add(txlog)
self.store.flush()
return txlog
def test_bootstrap_picks_up_only_files_in_live_udfs(self):
user = self.obj_factory.make_user()
with txlog_disabled():
root_udf = UserVolume.get_root(self.sstore, user.id)
photo_in_root = self.obj_factory.make_file(
user, root_udf.root_node, u'foo.jpg', u'image/jpeg')
dead_udf = self.obj_factory.make_udf(user=user)
self.obj_factory.make_file(
user, dead_udf.root_node, u'foo-in-dead-udf.jpg',
u'image/jpeg')
dead_udf.delete()
TransactionLog.bootstrap(user)
self.assertBootstrappingPickedUpFiles([photo_in_root])
def test_record_move_for_directory(self):
user = self.obj_factory.make_user()
new_parent = self.obj_factory.make_directory(user=user,
name=u'new-parent')
current_parent = self.obj_factory.make_directory(
user=user, name=u'current-parent')
dir1 = self.obj_factory.make_directory(name=u'dir1',
parent=current_parent)
f = self._make_file(name=u'f.jpg', parent=dir1, mimetype=self.mimetype)
f_orig_path = f.full_path
dir_orig_path = dir1.full_path
dir1.move(new_parent.id, dir1.name)
f_extra_data = TransactionLog._get_extra_data_for_new_node(
f, f.volume.path)
# All TransactionLog entries created will have the moved directory's
# generation because in a move() we only update the directory's
# generation and not the generation of its descendants.
f_expected_attrs = self._get_dict_with_txlog_attrs_from(
f,
TransactionLog.OP_MOVE,
extra=dict(old_path=f_orig_path,
generation=dir1.generation,
extra_data_dict=f_extra_data))
dir_expected_attrs = self._get_dict_with_txlog_attrs_from(
dir1,
TransactionLog.OP_MOVE,
extra=dict(old_path=dir_orig_path, generation=dir1.generation))
self.assertStoredTransactionLogsMatch({
f.id: f_expected_attrs,
dir1.id: dir_expected_attrs
})
def test_bootstrap_picks_up_only_files_in_live_udfs(self):
user = self.obj_factory.make_user()
with txlog_disabled():
root_udf = UserVolume.get_root(self.sstore, user.id)
photo_in_root = self.obj_factory.make_file(user,
root_udf.root_node,
u'foo.jpg',
u'image/jpeg')
dead_udf = self.obj_factory.make_udf(user=user)
self.obj_factory.make_file(user, dead_udf.root_node,
u'foo-in-dead-udf.jpg', u'image/jpeg')
dead_udf.delete()
TransactionLog.bootstrap(user)
self.assertBootstrappingPickedUpFiles([photo_in_root])
def test_bootstrap_picks_up_only_folders_in_live_udfs(self):
user = self.obj_factory.make_user()
with txlog_disabled():
root_udf = UserVolume.get_root(self.sstore, user.id)
folder_in_root = self.obj_factory.make_directory(
user, root_udf.root_node, u'folder1', public=True)
dead_udf = self.obj_factory.make_udf(user=user)
self.obj_factory.make_directory(user,
dead_udf.root_node,
u'folder2',
public=True)
dead_udf.delete()
TransactionLog.bootstrap(user)
self.assertBootstrappingPickedUpFolders([folder_in_root])
def test_record_move_for_directory(self):
user = self.obj_factory.make_user()
new_parent = self.obj_factory.make_directory(
user=user, name=u'new-parent')
current_parent = self.obj_factory.make_directory(
user=user, name=u'current-parent')
dir1 = self.obj_factory.make_directory(
name=u'dir1', parent=current_parent)
f = self._make_file(name=u'f.jpg', parent=dir1, mimetype=self.mimetype)
f_orig_path = f.full_path
dir_orig_path = dir1.full_path
dir1.move(new_parent.id, dir1.name)
f_extra_data = TransactionLog._get_extra_data_for_new_node(
f, f.volume.path)
# All TransactionLog entries created will have the moved directory's
# generation because in a move() we only update the directory's
# generation and not the generation of its descendants.
f_expected_attrs = self._get_dict_with_txlog_attrs_from(
f, TransactionLog.OP_MOVE,
extra=dict(old_path=f_orig_path, generation=dir1.generation,
extra_data_dict=f_extra_data))
dir_expected_attrs = self._get_dict_with_txlog_attrs_from(
dir1, TransactionLog.OP_MOVE,
extra=dict(old_path=dir_orig_path, generation=dir1.generation))
self.assertStoredTransactionLogsMatch(
{f.id: f_expected_attrs, dir1.id: dir_expected_attrs})
def assertBootstrappingPickedUpFolders(self, folders):
"""Check there are TXLog entries for the given folders."""
folder_txlogs = self.store.find(TransactionLog, op_type=TransactionLog.OP_PUBLIC_ACCESS_CHANGED)
expected = {}
for folder in folders:
extra_data = TransactionLog._get_extra_data_for_new_node(folder, folder.volume.path)
expected[folder.id] = self._get_dict_with_txlog_attrs_from(
folder, TransactionLog.OP_PUBLIC_ACCESS_CHANGED, extra=dict(extra_data_dict=extra_data)
)
self.assertTransactionLogsMatch(folder_txlogs, expected)
def assertBootstrappingPickedUpFiles(self, files):
"""Check there are TXLog bootstrapping entries for the given files."""
file_txlogs = self.store.find(TransactionLog, op_type=TransactionLog.OP_PUT_CONTENT)
expected = {}
for node in files:
extra_data = TransactionLog._get_extra_data_for_new_node(node, node.volume.path)
expected[node.id] = self._get_dict_with_txlog_attrs_from(
node, TransactionLog.OP_PUT_CONTENT, extra=dict(generation=node.generation, extra_data_dict=extra_data)
)
self.assertTransactionLogsMatch(file_txlogs, expected)
def test_txlog_for_content_change(self):
node = self._make_file(mimetype=self.mimetype)
new_content = self.obj_factory.make_content()
node.content = new_content
extra_data = TransactionLog._get_extra_data_for_new_node(node, node.volume.path)
expected_attrs = self._get_dict_with_txlog_attrs_from(
node, TransactionLog.OP_PUT_CONTENT, extra=dict(extra_data_dict=extra_data)
)
self.assertStoredTransactionLogsMatch({node.id: expected_attrs})
def test__get_extra_data_for_new_node(self):
"""Check that _get_extra_data_for_new_node includes all we need."""
f = self._make_file()
f_extra_data = dict(
size=f.content.size, storage_key=unicode(f.content.storage_key),
publicfile_id=None, public_uuid=None, content_hash=f.content_hash,
when_created=get_epoch_secs(f.when_created),
last_modified=get_epoch_secs(f.when_last_modified),
kind=f.kind, volume_path=f.volume.path)
expected = TransactionLog._get_extra_data_for_new_node(
f, f.volume.path)
self.assertEqual(expected, f_extra_data)
def assertBootstrappingPickedUpFolders(self, folders):
"""Check there are TXLog entries for the given folders."""
folder_txlogs = self.sstore.find(
TransactionLog, op_type=TransactionLog.OP_PUBLIC_ACCESS_CHANGED)
expected = {}
for folder in folders:
extra_data = TransactionLog._get_extra_data_for_new_node(
folder, folder.volume.path)
expected[folder.id] = self._get_dict_with_txlog_attrs_from(
folder,
TransactionLog.OP_PUBLIC_ACCESS_CHANGED,
extra=dict(extra_data_dict=extra_data))
self.assertTransactionLogsMatch(folder_txlogs, expected)
def make_transaction_log(self,
tx_id=None,
timestamp=None,
owner_id=1,
op_type=TransactionLog.OP_DELETE):
"""Create a transaction log."""
txlog = TransactionLog(uuid.uuid4(),
owner_id,
uuid.uuid4(),
op_type,
u"",
u"text/plain",
generation=1,
old_path=u"",
extra_data=u"")
if timestamp:
txlog.timestamp = timestamp
if tx_id:
txlog.id = tx_id
self.sstore.add(txlog)
self.sstore.flush()
return txlog
def test_txlog_for_content_change(self):
node = self._make_file(mimetype=self.mimetype)
new_content = self.obj_factory.make_content()
node.content = new_content
extra_data = TransactionLog._get_extra_data_for_new_node(
node, node.volume.path)
expected_attrs = self._get_dict_with_txlog_attrs_from(
node,
TransactionLog.OP_PUT_CONTENT,
extra=dict(extra_data_dict=extra_data))
self.assertStoredTransactionLogsMatch({node.id: expected_attrs})
def assertBootstrappingPickedUpFiles(self, files):
"""Check there are TXLog bootstrapping entries for the given files."""
file_txlogs = self.sstore.find(TransactionLog,
op_type=TransactionLog.OP_PUT_CONTENT)
expected = {}
for node in files:
extra_data = TransactionLog._get_extra_data_for_new_node(
node, node.volume.path)
expected[node.id] = self._get_dict_with_txlog_attrs_from(
node,
TransactionLog.OP_PUT_CONTENT,
extra=dict(generation=node.generation,
extra_data_dict=extra_data))
self.assertTransactionLogsMatch(file_txlogs, expected)
def test_txlog_for_public_access_change_on_interesting_file(self):
node = self._make_file(mimetype=self.mimetype)
publicfile = self.store.add(PublicNode(node.id, node.owner_id))
self.store.flush()
node.publicfile_id = publicfile.id
public_url = get_public_file_url(node)
self.assertIsNotNone(public_url)
extra_data = TransactionLog._get_extra_data_for_new_node(node, node.volume.path)
expected_attrs = self._get_dict_with_txlog_attrs_from(
node, TransactionLog.OP_PUBLIC_ACCESS_CHANGED, extra=dict(extra_data_dict=extra_data)
)
self.assertStoredTransactionLogsMatch({node.id: expected_attrs})
def test_txlog_when_publishing_directory(self):
directory = self.obj_factory.make_directory()
publicfile = self.store.add(PublicNode(directory.id, directory.owner_id))
self.store.flush()
directory.publicfile_id = publicfile.id
public_url = get_public_file_url(directory)
self.assertIsNotNone(public_url)
extra_data = TransactionLog._get_extra_data_for_new_node(directory, directory.volume.path)
expected_attrs = self._get_dict_with_txlog_attrs_from(
directory, TransactionLog.OP_PUBLIC_ACCESS_CHANGED, extra=dict(extra_data_dict=extra_data)
)
self.assertStoredTransactionLogsMatch({directory.id: expected_attrs})
def test__get_extra_data_for_new_node(self):
"""Check that _get_extra_data_for_new_node includes all we need."""
f = self._make_file()
f_extra_data = dict(size=f.content.size,
storage_key=unicode(f.content.storage_key),
publicfile_id=None,
public_uuid=None,
content_hash=f.content_hash,
when_created=get_epoch_secs(f.when_created),
last_modified=get_epoch_secs(f.when_last_modified),
kind=f.kind,
volume_path=f.volume.path)
expected = TransactionLog._get_extra_data_for_new_node(
f, f.volume.path)
self.assertEqual(expected, f_extra_data)
def test_txlog_when_unpublishing_directory(self):
directory = self.obj_factory.make_directory()
# Change _publicfile_id directly because if we go via the public API
# (.publicfile_id) it'll generate a TransactionLog and that will
# complicate the actual test.
directory._publicfile_id = self.obj_factory.get_unique_integer()
self.assertIsNotNone(directory.publicfile_id)
self.assertTrue(directory.is_public)
directory.publicfile_id = None
extra_data = TransactionLog._get_extra_data_for_new_node(directory, directory.volume.path)
expected_attrs = self._get_dict_with_txlog_attrs_from(
directory, TransactionLog.OP_PUBLIC_ACCESS_CHANGED, extra=dict(extra_data_dict=extra_data)
)
self.assertStoredTransactionLogsMatch({directory.id: expected_attrs})
def test_txlog_for_public_access_change_on_interesting_file(self):
node = self._make_file(mimetype=self.mimetype)
publicfile = self.ustore.add(PublicNode(node.id, node.owner_id))
self.ustore.flush()
node.publicfile_id = publicfile.id
public_url = get_public_file_url(node)
self.assertIsNotNone(public_url)
extra_data = TransactionLog._get_extra_data_for_new_node(
node, node.volume.path)
expected_attrs = self._get_dict_with_txlog_attrs_from(
node,
TransactionLog.OP_PUBLIC_ACCESS_CHANGED,
extra=dict(extra_data_dict=extra_data))
self.assertStoredTransactionLogsMatch({node.id: expected_attrs})
def test_txlog_when_publishing_directory(self):
directory = self.obj_factory.make_directory()
publicfile = self.ustore.add(
PublicNode(directory.id, directory.owner_id))
self.ustore.flush()
directory.publicfile_id = publicfile.id
public_url = get_public_file_url(directory)
self.assertIsNotNone(public_url)
extra_data = TransactionLog._get_extra_data_for_new_node(
directory, directory.volume.path)
expected_attrs = self._get_dict_with_txlog_attrs_from(
directory,
TransactionLog.OP_PUBLIC_ACCESS_CHANGED,
extra=dict(extra_data_dict=extra_data))
self.assertStoredTransactionLogsMatch({directory.id: expected_attrs})
def test_txlog_when_unpublishing_directory(self):
directory = self.obj_factory.make_directory()
# Change _publicfile_id directly because if we go via the public API
# (.publicfile_id) it'll generate a TransactionLog and that will
# complicate the actual test.
directory._publicfile_id = self.obj_factory.get_unique_integer()
self.assertIsNotNone(directory.publicfile_id)
self.assertTrue(directory.is_public)
directory.publicfile_id = None
extra_data = TransactionLog._get_extra_data_for_new_node(
directory, directory.volume.path)
expected_attrs = self._get_dict_with_txlog_attrs_from(
directory,
TransactionLog.OP_PUBLIC_ACCESS_CHANGED,
extra=dict(extra_data_dict=extra_data))
self.assertStoredTransactionLogsMatch({directory.id: expected_attrs})