def load_table_metadata(config, tbl_name):
cfg = ConfigManager(config, tbl_name)
engine = BaseData.get_engine(cfg.db_dir, cfg.db_name + ".db")
sess = BaseData.get_session_from_engine(engine)
TableClass = ClassManager.get_class_orm(tbl_name, engine)
UserClass = type(tbl_name, (Record, ), {})
# Map to SQL orm
mapper(UserClass, TableClass)
# Display queried info for single table and break
return sess, UserClass, cfg
def remove_columns_from_table(config_file, table_name, list_file, alias,
silent, integrity_cancel):
"""
:param silent:
:param config_file:
:param table_name:
:param list_file:
:param alias:
:return:
"""
_cfg, _tbl, _sil, _al = config_file, table_name, silent, alias
config, config_file = ConfigManager.confirm_config_set(config_file)
if alias != "None":
table_name = ConfigManager.get_name_by_alias(alias, config)
assert table_name is not None, TableNameAssertString.TABLE_NOT_FOUND
if list_file == "None":
raise ListFileNotProvidedError
cfg = ConfigManager(config, table_name)
columns_to_remove = set(
line.rstrip("\r\n") for line in open(list_file, "r"))
if not silent:
_remove_columns_display_message_prelude(
config[ConfigKeys.DATABASES][ConfigKeys.db_name],
config[ConfigKeys.DATABASES][ConfigKeys.rel_work_dir], table_name,
alias, columns_to_remove)
engine = BaseData.get_engine(cfg.db_dir, cfg.db_name + ".db")
sess = BaseData.get_session_from_engine(engine)
TableClass = ClassManager.get_class_orm(table_name, engine)
update_manager = UpdateManager(cfg, ClassManager.get_class_as_dict(cfg),
sess)
table_copy_csv = update_manager.create_table_copy(
datetime.today().strftime("%Y%m%d"), TableClass, silent)
new_attrs = {
key: value
for key, value in ClassManager.correct_dict(
ClassManager.get_class_as_dict(cfg)).items()
if key not in columns_to_remove
}
UpdatedDBClass, metadata = ClassManager.generate_class(
cfg.table_name, new_attrs, cfg.db_dir, cfg.db_name, cfg.table_dir)
ClassManager.write_class(new_attrs, cfg.classes_file)
cfg.update_config_file(table_name)
update_manager.delete_old_table_and_populate(engine, TableClass,
UpdatedDBClass,
table_copy_csv, table_name,
sess, silent)
if not silent:
_remove_columns_display_message_epilogue()
if not integrity_cancel:
integrity_check(_cfg, _tbl, _al, _sil)
def delete_from_table(config_file, table_name, list_file, alias, silent, integrity_cancel):
"""
:param silent:
:param config_file:
:param table_name:
:param list_file:
:param alias:
:return:
"""
_cfg, _tbl, _sil, _al = config_file, table_name, silent, alias
config, config_file = ConfigManager.confirm_config_set(config_file)
if alias != "None":
table_name = ConfigManager.get_name_by_alias(alias, config)
assert table_name is not None, TableNameAssertString.TABLE_NOT_FOUND
if list_file == "None":
raise ListFileNotProvidedError
cfg = ConfigManager(config, table_name)
ids_to_remove = set(line.rstrip("\r\n") for line in open(list_file, "r"))
if not silent:
_delete_records_display_message_prelude(
config[ConfigKeys.DATABASES][ConfigKeys.db_name],
config[ConfigKeys.DATABASES][ConfigKeys.rel_work_dir],
table_name,
alias,
ids_to_remove
)
engine = BaseData.get_engine(cfg.db_dir, cfg.db_name + ".db")
sess = BaseData.get_session_from_engine(engine)
TableClass = ClassManager.get_class_orm(table_name, engine)
UserClass = type(table_name, (Record,), {})
mapper(UserClass, TableClass)
for _id in ids_to_remove:
print_if_not_silent(silent, " ..Removing record %s" % _id)
try:
os.remove(sess.query(UserClass).filter_by(_id=_id).first().full_path())
except TypeError:
continue
sess.delete(sess.query(UserClass).filter_by(_id=_id).first())
sess.commit()
if not silent:
_remove_columns_display_message_epilogue()
if not integrity_cancel:
integrity_check(_cfg, _tbl, _al, _sil)
def integrity_check(config_file, table_name, alias, silent):
""" Function called from dbdm that checks integrity and issues in project at all levels
:param config_file:
:param table_name:
:param alias:l
:return:
"""
config, config_file = ConfigManager.confirm_config_set(config_file)
if alias != "None":
table_name = ConfigManager.get_name_by_alias(alias, config)
if table_name is None or (table_name == "None" and alias == "None"):
tables_to_search = list(config[ConfigKeys.TABLES_TO_DB].keys())
else:
tables_to_search = [
table_name,
]
py_fixfile_name = "%s.%s.fix" % (datetime.today().strftime("%Y%m%d"),
str(randint(1, 1001)))
if not silent:
_integrity_check_display_message_prelude(
config[ConfigKeys.DATABASES][ConfigKeys.db_name],
config[ConfigKeys.DATABASES][ConfigKeys.working_dir],
tables_to_search, py_fixfile_name)
im = IntegrityManager(config, py_fixfile_name, tables_to_search)
# TODO: Implement table-level checks
im.initialize_fix_file()
im.initial_project_check()
# im.table_check()
for table in tables_to_search:
cfg = ConfigManager(config, table)
engine = BaseData.get_engine(cfg.db_dir, cfg.db_name + ".db")
sess = BaseData.get_session_from_engine(engine)
TableClass = ClassManager.get_class_orm(table, engine)
UserClass = type(table, (Record, ), {})
# Map to SQL orm
mapper(UserClass, TableClass)
im.record_check(sess, UserClass, table)
if im.issues_found == 0:
os.remove(py_fixfile_name)
if not silent:
_integrity_check_display_message_epilogue(im.issues_found,
py_fixfile_name)
del im
def remove_table_from_database(config_file, table_name, alias, silent,
integrity_cancel):
""" Function removes a given table from a database
:param silent:
:param config_file:
:param table_name:
:param alias:
:return:
"""
_cfg, _tbl, _sil, _al = config_file, table_name, silent, alias
config, config_file = ConfigManager.confirm_config_set(config_file)
if alias != "None":
table_name = ConfigManager.get_name_by_alias(alias, config)
assert table_name is not None, TableNameAssertString.TABLE_NOT_FOUND
if not silent:
_remove_table_display_message_prelude(
config[ConfigKeys.DATABASES][ConfigKeys.db_name],
config[ConfigKeys.DATABASES][ConfigKeys.rel_work_dir], table_name,
alias)
cfg = ConfigManager(config, table_name)
engine = BaseData.get_engine(cfg.db_dir, cfg.db_name + ".db")
sess = BaseData.get_session_from_engine(engine)
TableClass = ClassManager.get_class_orm(table_name, engine)
UserClass = type(table_name, (Record, ), {})
mapper(UserClass, TableClass)
all_records = sess.query(UserClass).all()
for record in all_records:
if record:
print_if_not_silent(silent, " ..Removing record %s" % record._id)
try:
os.remove(record.full_path())
except OSError:
continue
TableClass.drop(engine)
cfg.remove_table_from_config_file(table_name)
os.remove(cfg.classes_file)
shutil.rmtree(cfg.table_dir)
if not silent:
_remove_columns_display_message_epilogue()
if not integrity_cancel:
integrity_check(_cfg, _tbl, _al, _sil)
def get_table(config_path, table_name=None, alias=None):
""" Primary method for users importing
:param config_path:
:param table_name:
:param alias:
:return:
"""
# Load config data from file
cfg, config_path = ConfigManager.confirm_config_set(config_path)
if alias:
table_name = ConfigManager.get_name_by_alias(alias, cfg)
config = ConfigManager(cfg, table_name)
# Get session to return
engine = BaseData.get_engine(config.db_dir, config.db_name + ".db")
sess = BaseData.get_session_from_engine(engine)
# Generate table class and name based on presence of alias
TableClass = ClassManager.get_class_orm(table_name, engine)
UserClass = type(table_name, (Record, ), {})
# Map to SQL orm
mapper(UserClass, TableClass)
return RecordList(sess, UserClass, config)
def move_project(config_file, path, integrity_cancel, silent):
"""
:param config_file:
:param path:
:param integrity_cancel:
:param silent:
:return:
"""
assert path != 'None', "Path (-p) does not exist, exiting"
current_path = os.path.abspath(
os.path.abspath(os.path.relpath(config_file)))
path = os.path.abspath(os.path.relpath(path))
config, config_file = ConfigManager.confirm_config_set(config_file)
old_path = config[ConfigKeys.DATABASES][ConfigKeys.working_dir]
assert os.path.dirname(old_path) != os.path.abspath(path), \
"Project exists in directory, cancelling"
project_name = config[ConfigKeys.DATABASES][ConfigKeys.rel_work_dir]
if not silent:
_move_project_display_message_prelude(project_name, old_path, path),
# Move directory from old location to new location
shutil.move(current_path, path)
# Update config file with new location
abs_path_working_dir = os.path.abspath(os.path.join(path, project_name))
db_name = project_name
config[ConfigKeys.DATABASES] = {
ConfigKeys.db_name:
db_name,
ConfigKeys.working_dir:
abs_path_working_dir,
ConfigKeys.rel_work_dir:
db_name,
ConfigKeys.migrations_dir:
os.path.join(abs_path_working_dir, Directories.MIGRATIONS),
ConfigKeys.config_dir:
os.path.join(abs_path_working_dir, Directories.CONFIG),
ConfigKeys.db_dir:
os.path.join(abs_path_working_dir, Directories.DATABASE),
ConfigKeys.rel_db_dir:
os.path.join(db_name, Directories.DATABASE),
}
with open(
os.path.join(path, project_name, Directories.CONFIG,
os.path.basename(config_file)), "w") as W:
config.write(W)
# Update location for each record in each table
tables_in_database = config[ConfigKeys.TABLES_TO_DB].keys()
engine = BaseData.get_engine(
os.path.join(config[ConfigKeys.DATABASES][ConfigKeys.working_dir],
Directories.DATABASE),
config[ConfigKeys.DATABASES][ConfigKeys.rel_work_dir] + ".db")
sess = BaseData.get_session_from_engine(engine)
for tbl_name in tables_in_database:
TableClass = ClassManager.get_class_orm(tbl_name, engine)
UserClass = type(tbl_name, (Record, ), {})
mapper(UserClass, TableClass)
for record in sess.query(UserClass).all():
record.location = os.path.join(abs_path_working_dir,
Directories.DATABASE, tbl_name)
sess.commit()
if not integrity_cancel:
integrity_check(abs_path_working_dir, tbl_name, "None", silent)
if not silent:
_move_project_display_message_epilogue()