def _prepare_columns(self):
try:
existing_columns = self._get_column_names_from_db()
except (OperationalError, ProgrammingError):
return
# Get table
meta = MetaData(bind=self._engine)
table = Table(self.df_name, meta, autoload=True)
# Add columns
for column_name in set([c['name'] for c in self.columns]).difference(existing_columns):
column = next((filter(lambda c: c['name'] == column_name, self.columns)))
column_name = column['name']
if column_name in ['uuid_in_df']:
column_dtype = VARCHAR(32)
elif column_name in ['creation_time_in_df']:
column_dtype = map_dtype('double', self._config.current_db['engine'])['sql']
else:
column_dtype = map_dtype(column['dtype'], self._config.current_db['engine'])['sql']
create_column(Column(column_name, column_dtype), table)
def upgrade(migrate_engine):
meta.bind = migrate_engine
migrate.create_column(version_id, endpoint_templates)
assert endpoint_templates.c.version_id is version_id
migrate.create_column(version_list, endpoint_templates)
assert endpoint_templates.c.version_list is version_list
migrate.create_column(version_info, endpoint_templates)
assert endpoint_templates.c.version_info is version_info
def upgrade(migrate_engine):
meta.bind = migrate_engine
migrate.create_column(owner_id, services)
assert services.c.owner_id is owner_id
def upgrade(migrate_engine):
meta.bind = migrate_engine
migrate.create_column(user_uid, users)
assert users.c.uid is user_uid
def upgrade(migrate_engine):
meta.bind = migrate_engine
migrate.create_column(tenant_uid, tenants)
assert tenants.c.uid is tenant_uid
def upgrade(migrate_engine):
meta.bind = migrate_engine
session = SA_SESSIONMAKER()
try:
session.execute(
text("""ALTER TABLE "BURST_CONFIGURATIONS"
RENAME TO "BurstConfiguration"; """))
# Dropping tables which don't exist in the new version
session.execute(text("""DROP TABLE "MAPPED_LOOK_UP_TABLE_DATA";"""))
session.execute(text("""DROP TABLE "MAPPED_DATATYPE_MEASURE_DATA";"""))
session.execute(
text("""DROP TABLE "MAPPED_SPATIAL_PATTERN_VOLUME_DATA";"""))
session.execute(text("""DROP TABLE "MAPPED_SIMULATION_STATE_DATA";"""))
session.execute(text("""DROP TABLE "WORKFLOW_STEPS";"""))
session.execute(text("""DROP TABLE "WORKFLOW_VIEW_STEPS";"""))
# Dropping tables which will be repopulated from the H5 files
session.execute(
text("""DROP TABLE "MAPPED_COHERENCE_SPECTRUM_DATA";"""))
session.execute(
text("""DROP TABLE "MAPPED_COMPLEX_COHERENCE_SPECTRUM_DATA";"""))
session.execute(
text("""DROP TABLE "MAPPED_CONNECTIVITY_ANNOTATIONS_DATA";"""))
session.execute(
text("""DROP TABLE "MAPPED_CONNECTIVITY_MEASURE_DATA";"""))
session.execute(text("""DROP TABLE "MAPPED_CONNECTIVITY_DATA";"""))
session.execute(
text("""DROP TABLE "MAPPED_CORRELATION_COEFFICIENTS_DATA";"""))
session.execute(text("""DROP TABLE "MAPPED_COVARIANCE_DATA";"""))
session.execute(
text("""DROP TABLE "MAPPED_CROSS_CORRELATION_DATA";"""))
session.execute(text("""DROP TABLE "MAPPED_FCD_DATA";"""))
session.execute(text("""DROP TABLE "MAPPED_FOURIER_SPECTRUM_DATA";"""))
session.execute(
text("""DROP TABLE "MAPPED_INDEPENDENT_COMPONENTS_DATA";"""))
session.execute(
text("""DROP TABLE "MAPPED_LOCAL_CONNECTIVITY_DATA";"""))
session.execute(
text("""DROP TABLE "MAPPED_PRINCIPAL_COMPONENTS_DATA";"""))
session.execute(
text("""DROP TABLE "MAPPED_PROJECTION_MATRIX_DATA";"""))
session.execute(text("""DROP TABLE "MAPPED_REGION_MAPPING_DATA";"""))
session.execute(
text("""DROP TABLE "MAPPED_REGION_VOLUME_MAPPING_DATA";"""))
session.execute(
text("""DROP TABLE "MAPPED_TIME_SERIES_REGION_DATA";"""))
session.execute(text("""DROP TABLE "MAPPED_TIME_SERIES_EEG_DATA";"""))
session.execute(text("""DROP TABLE "MAPPED_TIME_SERIES_MEG_DATA";"""))
session.execute(text("""DROP TABLE "MAPPED_TIME_SERIES_SEEG_DATA";"""))
session.execute(
text("""DROP TABLE "MAPPED_TIME_SERIES_SURFACE_DATA";"""))
session.execute(
text("""DROP TABLE "MAPPED_TIME_SERIES_VOLUME_DATA";"""))
session.execute(text("""DROP TABLE "MAPPED_SENSORS_DATA" """))
session.execute(text("""DROP TABLE "MAPPED_TRACTS_DATA";"""))
session.execute(text("""DROP TABLE "MAPPED_STIMULI_REGION_DATA";"""))
session.execute(text("""DROP TABLE "MAPPED_STIMULI_SURFACE_DATA";"""))
session.execute(text("""DROP TABLE "MAPPED_STRUCTURAL_MRI_DATA" """))
session.execute(text("""DROP TABLE "MAPPED_SURFACE_DATA" """))
session.execute(text("""DROP TABLE "MAPPED_VALUE_WRAPPER_DATA";"""))
session.execute(text("""DROP TABLE "MAPPED_VOLUME_DATA" """))
session.execute(
text("""DROP TABLE "MAPPED_WAVELET_COEFFICIENTS_DATA";"""))
session.execute(text("""DROP TABLE "DATA_TYPES_GROUPS";"""))
session.execute(text("""DROP TABLE "MAPPED_ARRAY_DATA";"""))
session.execute(
text("""DROP TABLE "MAPPED_SPATIO_TEMPORAL_PATTERN_DATA" """))
session.execute(text("""DROP TABLE "MAPPED_SPATIAL_PATTERN_DATA";"""))
session.execute(text("""DROP TABLE "WORKFLOWS";"""))
session.execute(text("""DROP TABLE "MAPPED_TIME_SERIES_DATA";"""))
session.commit()
except Exception as excep:
LOGGER.exception(excep)
finally:
session.close()
# MIGRATING USERS #
users_table = meta.tables['USERS']
for column in USER_COLUMNS:
create_column(column, users_table)
session = SA_SESSIONMAKER()
try:
user_ids = eval(
str(
session.execute(
text("""SELECT U.id
FROM "USERS" U """)).fetchall()))
for id in user_ids:
session.execute(
text("""UPDATE "USERS" SET display_name = username,
gid ='""" + uuid.uuid4().hex + """' WHERE id = """ +
str(id[0])))
session.commit()
except Exception as excep:
LOGGER.exception(excep)
finally:
session.close()
UniqueConstraint("gid", table=users_table)
# Migrating BurstConfiguration
burst_config_table = meta.tables['BurstConfiguration']
for column in BURST_COLUMNS:
create_column(column, burst_config_table)
session = SA_SESSIONMAKER()
try:
session.execute(
text("""ALTER TABLE "BurstConfiguration"
RENAME COLUMN _dynamic_ids TO dynamic_ids"""))
session.execute(
text("""ALTER TABLE "BurstConfiguration"
RENAME COLUMN _simulator_configuration TO simulator_gid"""
))
ranges = session.execute(
text("""SELECT OG.id, OG.range1, OG.range2
FROM "OPERATION_GROUPS" OG""")).fetchall()
session.execute(
text(
"""DELETE FROM "BurstConfiguration" WHERE status = \'error\'"""
))
ranges_1 = []
ranges_2 = []
for r in ranges:
ranges_1.append(str(r[1]))
ranges_2.append(str(r[2]))
new_ranges_1 = migrate_range_params(ranges_1)
new_ranges_2 = migrate_range_params(ranges_2)
operation_groups = session.execute(
text("""SELECT * FROM "OPERATION_GROUPS" """)).fetchall()
for op_g in operation_groups:
op = eval(
str(
session.execute(
text(
"""SELECT fk_operation_group, parameters, meta_data
FROM "OPERATIONS" O WHERE O.fk_operation_group = """ +
str(op_g[0]))).fetchone()))
burst_id = eval(op[2])['Burst_Reference']
if 'time_series' in op[1]:
session.execute(
text(
"""UPDATE "BurstConfiguration" as B SET fk_metric_operation_group = """
+ str(op[0]) + """ WHERE B.id = """ + str(burst_id)))
else:
session.execute(
text(
"""UPDATE "BurstConfiguration" as B SET fk_operation_group = """
+ str(op[0]) + """ WHERE B.id = """ + str(burst_id)))
for i in range(len(ranges_1)):
range1 = str(new_ranges_1[i]).replace('\'', '')
range2 = str(new_ranges_2[i]).replace('\'', '')
session.execute(
text("""UPDATE "BurstConfiguration" SET
range1 = '""" + range1 + """'
WHERE fk_operation_group = """ + str(ranges[i][0])))
session.execute(
text("""UPDATE "OPERATION_GROUPS" SET
range1 = '""" + range1 + """'
WHERE id = """ + str(ranges[i][0])))
if range2 != 'None':
session.execute(
text("""UPDATE "BurstConfiguration" SET
range2 = '""" + range2 + """'
WHERE fk_operation_group = """ + str(ranges[i][0])))
session.execute(
text("""UPDATE "OPERATION_GROUPS" SET
range2 = '""" + range2 + """'
WHERE id = """ + str(ranges[i][0])))
session.commit()
except Exception:
session.close()
finally:
session.close()
# Drop old column
drop_column(BURST_DELETED_COLUMN, burst_config_table)
# Create constraints only after the rows are populated
fk_burst_config_constraint_1 = ForeignKeyConstraint(
["fk_simulation"], ["OPERATIONS.id"], table=burst_config_table)
fk_burst_config_constraint_2 = ForeignKeyConstraint(
["fk_operation_group"], ["OPERATION_GROUPS.id"],
table=burst_config_table)
fk_burst_config_constraint_3 = ForeignKeyConstraint(
["fk_metric_operation_group"], ["OPERATION_GROUPS.id"],
table=burst_config_table)
fk_burst_config_constraint_1.create()
fk_burst_config_constraint_2.create()
fk_burst_config_constraint_3.create()
# MIGRATING Operations #
session = SA_SESSIONMAKER()
try:
burst_ref_metadata = session.execute(
text("""SELECT id, meta_data FROM "OPERATIONS"
WHERE meta_data like '%Burst_Reference%' """)).fetchall()
for metadata in burst_ref_metadata:
metadata_dict = eval(str(metadata[1]))
session.execute(
text("""UPDATE "OPERATIONS" SET parameters = '""" +
json.dumps(metadata_dict['Burst_Reference']) +
"""' WHERE id = """ + str(metadata[0])))
session.execute(
text("""ALTER TABLE "OPERATIONS"
RENAME COLUMN parameters TO view_model_gid"""
))
# Name it back to the old name, because we have to keep both tables so we can create BurstConfigurationH5s
session.execute(
text("""ALTER TABLE "BurstConfiguration"
RENAME TO "BURST_CONFIGURATION"; """
))
session.commit()
except Exception as excep:
LOGGER.exception(excep)
finally:
session.close()
session = SA_SESSIONMAKER()
try:
session.execute(text("""DROP TABLE "ALGORITHMS"; """))
session.execute(text("""DROP TABLE "ALGORITHM_CATEGORIES"; """))
session.execute(text("""DROP TABLE "DATA_TYPES"; """))
session.commit()
except Exception as excep:
# If the drops fail, it could mean we are using postgresql
session.close()
session = SA_SESSIONMAKER()
LOGGER.exception(excep)
try:
session.execute(
text("""DROP TABLE if exists "ALGORITHMS" cascade; """))
session.execute(
text(
"""DROP TABLE if exists "ALGORITHM_CATEGORIES" cascade; """
))
session.execute(
text("""DROP TABLE if exists "DATA_TYPES" cascade; """))
session.commit()
except Exception as excep:
LOGGER.exception(excep)
finally:
session.close()
op_table = meta.tables['OPERATIONS']
create_column(OP_COLUMN, op_table)
drop_column(OP_DELETED_COLUMN, op_table)
def upgrade(migrate_engine):
meta.bind = migrate_engine
migrate.create_column(user_uid, users)
assert users.c.uid is user_uid
def upgrade(migrate_engine):
meta.bind = migrate_engine
migrate.create_column(owner_id, services)
assert services.c.owner_id is owner_id
def upgrade(migrate_engine):
meta.bind = migrate_engine
migrate.create_column(tenant_uid, tenants)
assert tenants.c.uid is tenant_uid