def copy_table(self,
table_to_copy,
database_in,
database_out,
new_table_name=None,
use_chunking=True,
memory_in_gigabytes=2):
in_metadata = database_in.metadata
out_metadata = database_out.metadata
in_table = Table(table_to_copy, in_metadata, autoload=True)
out_table = in_table.tometadata(out_metadata)
if new_table_name is not None:
out_table.rename(new_table_name)
out_metadata.create_all()
if use_chunking:
memory_in_bytes = memory_in_gigabytes * 1024**2
row_size = self._row_byte_size(in_table)
chunk_size = int(.1 * memory_in_bytes / row_size)
num_rows = in_table.count().execute().fetchone()[0]
num_inserted_rows = 0
while num_inserted_rows < num_rows:
qry = in_table.select().offset(num_inserted_rows).limit(
chunk_size)
result = qry.execute()
data = [
dict((col.key, x[col.name]) for col in in_table.c)
for x in result
]
out_table.insert().execute(*data)
num_inserted_rows += len(data)
else:
qry = in_table.select()
data = [
dict((col.key, x[col.name]) for col in in_table.c)
for x in qry.execute()
]
out_table.insert().execute(*data)
def copy_table(self, table_to_copy,
database_in, database_out,
new_table_name = None,
use_chunking = True,
memory_in_gigabytes = 2):
in_metadata = database_in.metadata
out_metadata = database_out.metadata
in_table = Table(table_to_copy, in_metadata, autoload=True)
out_table = in_table.tometadata(out_metadata)
if new_table_name is not None:
out_table.rename(new_table_name)
out_metadata.create_all()
if use_chunking:
memory_in_bytes = memory_in_gigabytes * 1024**2
row_size = self._row_byte_size(in_table)
chunk_size = int( .1 * memory_in_bytes / row_size )
num_rows = in_table.count().execute().fetchone()[0]
num_inserted_rows = 0
while num_inserted_rows < num_rows:
qry = in_table.select().offset(num_inserted_rows).limit(chunk_size)
result = qry.execute()
data = [ dict( (col.key, x[col.name]) for col in in_table.c)
for x in result ]
out_table.insert().execute(*data)
num_inserted_rows += len(data)
else:
qry = in_table.select()
data = [ dict( (col.key, x[col.name]) for col in in_table.c)
for x in qry.execute() ]
out_table.insert().execute(*data)
header('Connect to v3')
success('Connected to v3')
header('Cleaning up db')
project_priority_areas_v3.delete().execute()
for c in [
Task, ProjectInfo, Project, License, AreaOfInterest, PriorityArea, User
]:
s3.query(c).delete()
s3.commit()
success('Cleaned up')
#
# Users
#
count = users_v2.count().scalar()
header('Importing %s users' % count)
i = 0
for user_v2 in s2.query(users_v2):
if user_v2.id != 24529:
continue
user = User()
user.id = user_v2.id
user.role = 0
user.username = user_v2.username
user.mapping_level = 1
user.tasks_mapped = 0
user.tasks_validated = 0
user.tasks_invalidated = 0
s3.add(user)
i += 1
