def setUp(self):
"""Set up."""
super(StorageDALTestCase, self).setUp()
self.obj_factory = DAOObjectFactory()
self.user_store = get_user_store()
self.get_shard_store = get_shard_store
self.save_utils_set_public_uuid = utils.set_public_uuid
def setUp(self):
"""Set up."""
super(StorageDALTestCase, self).setUp()
self.obj_factory = DAOObjectFactory()
self.user_store = get_user_store()
self.get_shard_store = get_shard_store
self.save_utils_set_public_uuid = utils.set_public_uuid
def main(options):
"""The main function."""
store = get_shard_store(options.from_shard_id)
user_ids = options.user_ids or ""
if options.action == DUMP:
get_column_names(store, True)
for c in COPYINFO:
table = c[0]
columns = c[1]['columns']
select = c[1]['query']
replacements = dict(columns=columns,
userids=user_ids,
shard_id=options.to_shard_id)
sql = "COPY (%s) TO '%s/%s.dat';" % (select % replacements,
options.path, table)
print "Dumping %s TO %s/%s.dat" % (table, options.path, table),
storage_tm.begin()
execute_sql(store, sql)
storage_tm.abort()
return
if options.action == LOAD:
get_column_names(store, False)
shard_store = get_shard_store(options.to_shard_id)
storage_tm.begin()
print "Dropping Indexes..."
for sql in DROP_NO_DUPLICATE_SQL:
shard_store.execute(sql)
for sql in CREATE_NO_DUPLICATE_SQL:
shard_store.execute(sql)
for sql in DROP_INDEXES_SQL:
shard_store.execute(sql)
for c in COPYINFO:
table = c[0]
columns = c[1]['columns']
sql = "COPY %s (%s) FROM '%s/%s.dat';" % (table, columns,
options.path, table)
print "Loading %s From %s/%s.dat" % (table, options.path,
table),
execute_sql(shard_store, sql)
user_store = get_user_store()
user_store.execute(INITUSER % dict(shard_id=options.to_shard_id,
userids=user_ids))
print "Rebuilding Indexes..."
for sql in DROP_NO_DUPLICATE_SQL:
execute_sql(shard_store, sql)
for sql in CREATE_INDEXES_SQL:
execute_sql(shard_store, sql)
storage_tm.commit()
return
if options.action == DELETE:
shard_store = get_shard_store(options.from_shard_id)
storage_tm.begin()
print "Deleting user data..."
for sql in DELETE_SQL:
execute_sql(shard_store, sql % dict(userids=user_ids))
storage_tm.commit()
def main(options):
"""The main function."""
store = get_shard_store(options.from_shard_id)
user_ids = options.user_ids or ""
if options.action == DUMP:
get_column_names(store, True)
for c in COPYINFO:
table = c[0]
columns = c[1]['columns']
select = c[1]['query']
replacements = dict(columns=columns,
userids=user_ids,
shard_id=options.to_shard_id)
sql = "COPY (%s) TO '%s/%s.dat';" % (select % replacements,
options.path, table)
print "Dumping %s TO %s/%s.dat" % (table, options.path, table),
storage_tm.begin()
execute_sql(store, sql)
storage_tm.abort()
return
if options.action == LOAD:
get_column_names(store, False)
shard_store = get_shard_store(options.to_shard_id)
storage_tm.begin()
print "Dropping Indexes..."
for sql in DROP_NO_DUPLICATE_SQL:
shard_store.execute(sql)
for sql in CREATE_NO_DUPLICATE_SQL:
shard_store.execute(sql)
for sql in DROP_INDEXES_SQL:
shard_store.execute(sql)
for c in COPYINFO:
table = c[0]
columns = c[1]['columns']
sql = "COPY %s (%s) FROM '%s/%s.dat';" % (table, columns,
options.path, table)
print "Loading %s From %s/%s.dat" % (table, options.path, table),
execute_sql(shard_store, sql)
user_store = get_user_store()
user_store.execute(
INITUSER % dict(shard_id=options.to_shard_id, userids=user_ids))
print "Rebuilding Indexes..."
for sql in DROP_NO_DUPLICATE_SQL:
execute_sql(shard_store, sql)
for sql in CREATE_INDEXES_SQL:
execute_sql(shard_store, sql)
storage_tm.commit()
return
if options.action == DELETE:
shard_store = get_shard_store(options.from_shard_id)
storage_tm.begin()
print "Deleting user data..."
for sql in DELETE_SQL:
execute_sql(shard_store, sql % dict(userids=user_ids))
storage_tm.commit()
def bootstrap(cls, user):
store = get_shard_store(user.shard_id)
cls.record_user_created(user)
# Number of TransactionLog rows we inserted.
rows = 1
for udf in store.find(UserVolume, owner_id=user.id,
status=STATUS_LIVE):
cls.record_udf_created(udf)
rows += 1
# If this becomes a problem it can be done as a single INSERT, but
# we'd need to duplicate the get_public_file_url() in plpython.
udf_join = Join(
StorageObject,
UserVolume, StorageObject.volume_id == UserVolume.id)
conditions = [StorageObject.kind == 'Directory',
StorageObject.owner_id == user.id,
StorageObject.status == STATUS_LIVE,
StorageObject._publicfile_id != None, # NOQA
UserVolume.status == STATUS_LIVE]
dirs = store.using(udf_join).find(StorageObject, *conditions)
for directory in dirs:
cls.record_public_access_change(directory)
rows += 1
# XXX: If this takes too long it will get killed by the transaction
# watcher. Need to check what's the limit we could have here.
# Things to check:
# * If it still takes too long, we could find out the IDs of the
# people who have a lot of music/photos, run it just for them with
# the transaction watcher disabled and then run it for everybody
# else afterwards.
query = """
INSERT INTO txlog.transaction_log (
node_id, owner_id, volume_id, op_type, path, generation,
mimetype, extra_data)
SELECT O.id, O.owner_id, O.volume_id, ?,
txlog.path_join(O.path, O.name), O.generation, O.mimetype,
txlog.get_extra_data_to_recreate_file_1(
kind, size, storage_key, publicfile_id,
public_uuid, content_hash,
extract(epoch from O.when_created at time zone 'UTC'),
extract(epoch
from O.when_last_modified at time zone 'UTC'),
UserDefinedFolder.path
) as extra_data
FROM Object as O
JOIN UserDefinedFolder on UserDefinedFolder.id = O.volume_id
LEFT JOIN ContentBlob on ContentBlob.hash = O.content_hash
WHERE
O.kind != 'Directory'
AND O.owner_id = ?
AND O.status = 'Live'
AND UserDefinedFolder.status = 'Live'
"""
params = (cls.OPERATIONS_MAP[cls.OP_PUT_CONTENT], user.id)
rows += store.execute(query, params=params).rowcount
# Cannot create TransactionLogs for Shares in a single INSERT like
# above because TransactionLogs and Shares live in separate databases.
share_join = LeftJoin(
Share, StorageUser, Share.shared_to == StorageUser.id)
conditions = [Share.shared_by == user.id,
Share.status == STATUS_LIVE,
Share.accepted == True] # NOQA
shares = get_user_store().using(share_join).find(Share, *conditions)
for share in shares:
cls.record_share_accepted(share)
rows += 1
return rows
def ustore(self):
""" gets the store, dont cache, threading issues may arise
"""
return get_user_store()
def bootstrap(cls, user):
store = get_shard_store(user.shard_id)
cls.record_user_created(user)
# Number of TransactionLog rows we inserted.
rows = 1
for udf in store.find(UserVolume, owner_id=user.id, status=STATUS_LIVE):
cls.record_udf_created(udf)
rows += 1
# If this becomes a problem it can be done as a single INSERT, but
# we'd need to duplicate the get_public_file_url() in plpython.
udf_join = Join(StorageObject, UserVolume, StorageObject.volume_id == UserVolume.id)
conditions = [
StorageObject.kind == "Directory",
StorageObject.owner_id == user.id,
StorageObject.status == STATUS_LIVE,
StorageObject._publicfile_id != None, # NOQA
UserVolume.status == STATUS_LIVE,
]
dirs = store.using(udf_join).find(StorageObject, *conditions)
for directory in dirs:
cls.record_public_access_change(directory)
rows += 1
# XXX: If this takes too long it will get killed by the transaction
# watcher. Need to check what's the limit we could have here.
# Things to check:
# * If it still takes too long, we could find out the IDs of the
# people who have a lot of music/photos, run it just for them with
# the transaction watcher disabled and then run it for everybody
# else afterwards.
query = """
INSERT INTO txlog.transaction_log (
node_id, owner_id, volume_id, op_type, path, generation,
mimetype, extra_data)
SELECT O.id, O.owner_id, O.volume_id, ?,
txlog.path_join(O.path, O.name), O.generation, O.mimetype,
txlog.get_extra_data_to_recreate_file_1(
kind, size, storage_key, publicfile_id,
public_uuid, content_hash,
extract(epoch from O.when_created at time zone 'UTC'),
extract(epoch
from O.when_last_modified at time zone 'UTC'),
UserDefinedFolder.path
) as extra_data
FROM Object as O
JOIN UserDefinedFolder on UserDefinedFolder.id = O.volume_id
LEFT JOIN ContentBlob on ContentBlob.hash = O.content_hash
WHERE
O.kind != 'Directory'
AND O.owner_id = ?
AND O.status = 'Live'
AND UserDefinedFolder.status = 'Live'
"""
params = (cls.OPERATIONS_MAP[cls.OP_PUT_CONTENT], user.id)
rows += store.execute(query, params=params).rowcount
# Cannot create TransactionLogs for Shares in a single INSERT like
# above because TransactionLogs and Shares live in separate databases.
share_join = LeftJoin(Share, StorageUser, Share.shared_to == StorageUser.id)
conditions = [Share.shared_by == user.id, Share.status == STATUS_LIVE, Share.accepted == True] # NOQA
shares = get_user_store().using(share_join).find(Share, *conditions)
for share in shares:
cls.record_share_accepted(share)
rows += 1
return rows
def ustore(self):
""" gets the store, dont cache, threading issues may arise
"""
return get_user_store()