def legacy_database_fixer(database_path, database, database_name, database_exists): database_directory = os.path.dirname(database_path) old_database_path = database_path old_filename = os.path.basename(old_database_path) new_filename = f"Pre_Alembic_{old_filename}" pre_alembic_path = os.path.join(database_directory, new_filename) pre_alembic_database_exists = False if os.path.exists(pre_alembic_path): database_path = pre_alembic_path pre_alembic_database_exists = True datas = [] if database_exists: Session, engine = db_helper.create_database_session(database_path) database_session = Session() result = inspect(engine).has_table("alembic_version") if not result: if not pre_alembic_database_exists: os.rename(old_database_path, pre_alembic_path) pre_alembic_database_exists = True if pre_alembic_database_exists: Session, engine = db_helper.create_database_session(pre_alembic_path) database_session = Session() api_table = database.api_table() media_table = database.media_table() legacy_api_table = api_table.legacy(database_name) legacy_media_table = media_table.legacy() result = database_session.query(legacy_api_table) post_db = result.all() for post in post_db: post_id = post.id created_at = post.created_at new_item = {} new_item["post_id"] = post_id new_item["text"] = post.text new_item["price"] = post.price new_item["paid"] = post.paid new_item["postedAt"] = created_at new_item["medias"] = [] result2 = database_session.query(legacy_media_table) media_db = result2.filter_by(post_id=post_id).all() for media in media_db: new_item2 = {} new_item2["media_id"] = media.id new_item2["post_id"] = media.post_id new_item2["links"] = [media.link] new_item2["directory"] = media.directory new_item2["filename"] = media.filename new_item2["size"] = media.size new_item2["media_type"] = media.media_type new_item2["downloaded"] = media.downloaded new_item2["created_at"] = created_at new_item["medias"].append(new_item2) datas.append(new_item) print database_session.close() export_sqlite2(old_database_path, datas, database_name, legacy_fixer=True)
def legacy_sqlite_updater( legacy_metadata_path: str, api_type: str, subscription: create_user, delete_metadatas: list, ): final_result = [] if os.path.exists(legacy_metadata_path): cwd = os.getcwd() alembic_location = os.path.join(cwd, "database", "archived_databases", api_type.lower()) db_helper.run_migrations(alembic_location, legacy_metadata_path) database_name = "user_data" session, engine = db_helper.create_database_session( legacy_metadata_path) database_session: Session = session() db_collection = db_helper.database_collection() database = db_collection.database_picker(database_name) if database: if api_type == "Messages": api_table_table = database.table_picker(api_type, True) else: api_table_table = database.table_picker(api_type) media_table_table = database.media_table.media_legacy_table if api_table_table: result = database_session.query(api_table_table).all() result2 = database_session.query(media_table_table).all() for item in result: item = item.__dict__ item["medias"] = [] for item2 in result2: if item["post_id"] != item2.post_id: continue item2 = item2.__dict__ item2["links"] = [item2["link"]] item["medias"].append(item2) print item["user_id"] = subscription.id item["postedAt"] = item["created_at"] final_result.append(item) delete_metadatas.append(legacy_metadata_path) database_session.close() return final_result, delete_metadatas
def __init__(self, api=None, subscription=None) -> None: if api: username = subscription.username download_info = subscription.download_info if download_info: self.downloaded = True metadata_locations = download_info["metadata_locations"] directory = download_info["directory"] for parent_type, value in metadata_locations.items(): for api_type, metadata_path in value.items(): Session, engine = db_helper.create_database_session( metadata_path) database_session = Session() database_name = api_type.lower() db_collection = db_helper.database_collection() database = db_collection.chooser(database_name) api_table = database.api_table media_table = database.media_table result = database_session.query(media_table).all() media_type_list = media_types() for r in result: item = getattr(media_type_list, r.media_type) item.append(r) media_type_list = media_type_list.__dict__ for location, v in media_type_list.items(): if location == "Texts": continue media_set = v string = "Download Processing\n" string += f"Name: {username} | Type: {api_type} | Count: {len(media_set)} {location} | Directory: {directory}\n" print(string) pool = multiprocessing() pool.starmap(self.download, product( media_set, [api])) database_session.commit() else: self.downloaded = False
def export_sqlite(archive_path, datas, parent_type, legacy_fixer=False, api=None): metadata_directory = os.path.dirname(archive_path) os.makedirs(metadata_directory, exist_ok=True) cwd = os.getcwd() api_type: str = os.path.basename(archive_path).removesuffix(".db") database_path = archive_path database_name = parent_type if parent_type else api_type database_name = database_name.lower() db_collection = db_helper.database_collection() database = db_collection.chooser(database_name) alembic_location = os.path.join(cwd, "database", "databases", database_name) database_exists = os.path.exists(database_path) if database_exists: if os.path.getsize(database_path) == 0: os.remove(database_path) database_exists = False if not legacy_fixer: x = legacy_database_fixer(database_path, database, database_name, database_exists) db_helper.run_migrations(alembic_location, database_path, api) print Session, engine = db_helper.create_database_session(database_path) database_session = Session() api_table = database.api_table media_table = database.media_table # api_table = db_helper.api_table() # media_table = db_helper.media_table() for post in datas: post_id = post["post_id"] postedAt = post["postedAt"] date_object = None if postedAt: if not isinstance(postedAt, datetime): date_object = datetime.strptime(postedAt, "%d-%m-%Y %H:%M:%S") else: date_object = postedAt result = database_session.query(api_table) post_db = result.filter_by(post_id=post_id).first() if not post_db: post_db = api_table() post_db.post_id = post_id post_db.text = post["text"] if post["price"] == None: post["price"] = 0 post_db.price = post["price"] post_db.paid = post["paid"] if date_object: post_db.created_at = date_object database_session.add(post_db) for media in post["medias"]: if media["media_type"] == "Texts": continue media_id = media.get("media_id", None) result = database_session.query(media_table) media_db = result.filter_by(media_id=media_id).first() if not media_db: media_db = result.filter_by(filename=media["filename"], created_at=date_object).first() if not media_db: media_db = media_table() if legacy_fixer: media_db.size = media["size"] media_db.downloaded = media["downloaded"] media_db.media_id = media_id media_db.post_id = post_id media_db.link = media["links"][0] media_db.preview = media.get("preview", False) media_db.directory = media["directory"] media_db.filename = media["filename"] media_db.media_type = media["media_type"] media_db.linked = media.get("linked", None) if date_object: media_db.created_at = date_object database_session.add(media_db) print print print database_session.commit() database_session.close() return Session, api_type, database
def export_sqlite(database_path: str, api_type, datas): metadata_directory = os.path.dirname(database_path) os.makedirs(metadata_directory, exist_ok=True) database_name = os.path.basename(database_path).replace(".db", "") cwd = os.getcwd() alembic_location = os.path.join(cwd, "database", "databases", database_name.lower()) db_helper.run_migrations(alembic_location, database_path) Session, engine = db_helper.create_database_session(database_path) db_collection = db_helper.database_collection() database = db_collection.database_picker(database_name) if not database: return database_session = Session() api_table = database.table_picker(api_type) if not api_table: return for post in datas: post_id = post["post_id"] postedAt = post["postedAt"] date_object = None if postedAt: if not isinstance(postedAt, datetime): date_object = datetime.strptime(postedAt, "%d-%m-%Y %H:%M:%S") else: date_object = postedAt result = database_session.query(api_table) post_db = result.filter_by(post_id=post_id).first() if not post_db: post_db = api_table() if api_type == "Messages": post_db.user_id = post["user_id"] post_db.post_id = post_id post_db.text = post["text"] if post["price"] is None: post["price"] = 0 post_db.price = post["price"] post_db.paid = post["paid"] post_db.archived = post["archived"] if date_object: post_db.created_at = date_object database_session.add(post_db) for media in post["medias"]: if media["media_type"] == "Texts": continue created_at = media["created_at"] if not isinstance(created_at, datetime): date_object = datetime.strptime(created_at, "%d-%m-%Y %H:%M:%S") else: date_object = postedAt media_id = media.get("media_id", None) result = database_session.query(database.media_table) media_db = result.filter_by(media_id=media_id).first() if not media_db: media_db = result.filter_by(filename=media["filename"], created_at=date_object).first() if not media_db: media_db = database.media_table() media_db.media_id = media_id media_db.post_id = post_id if "_sa_instance_state" in post: media_db.size = media["size"] media_db.downloaded = media["downloaded"] media_db.link = media["links"][0] media_db.preview = media.get("preview", False) media_db.directory = media["directory"] media_db.filename = media["filename"] media_db.api_type = api_type media_db.media_type = media["media_type"] media_db.linked = media.get("linked", None) if date_object: media_db.created_at = date_object database_session.add(media_db) print print print database_session.commit() database_session.close() return Session, api_type, database
async def fix_sqlite( profile_directory, download_directory, metadata_directory, format_directories, authed: create_auth, site_name, username, metadata_directory_format, ): items = content_types().__dict__.items() final_metadatas = [] for api_type, value in items: mandatory_directories = {} mandatory_directories["profile_directory"] = profile_directory mandatory_directories["download_directory"] = download_directory mandatory_directories["metadata_directory"] = metadata_directory formatted_directories = await format_directories( mandatory_directories, authed, site_name, username, metadata_directory_format, "", api_type, ) final_metadata_directory = formatted_directories["metadata_directory"] if all(final_metadata_directory != x for x in final_metadatas): final_metadatas.append(final_metadata_directory) print print for final_metadata in final_metadatas: archived_database_path = os.path.join(final_metadata, "Archived.db") if os.path.exists(archived_database_path): Session2, engine = db_helper.create_database_session( archived_database_path) database_session: Session = Session2() cwd = os.getcwd() for api_type, value in items: database_path = os.path.join(final_metadata, f"{api_type}.db") database_name = api_type.lower() alembic_location = os.path.join(cwd, "database", "archived_databases", database_name) result = inspect(engine).has_table(database_name) if result: db_helper.run_migrations(alembic_location, archived_database_path) db_helper.run_migrations(alembic_location, database_path) Session3, engine2 = db_helper.create_database_session( database_path) db_collection = db_helper.database_collection() database_session2: Session = Session3() database = db_collection.database_picker("user_data") if not database: return table_name = database.table_picker(api_type, True) if not table_name: return archived_result = database_session.query(table_name).all() for item in archived_result: result2 = (database_session2.query(table_name).filter( table_name.post_id == item.post_id).first()) if not result2: item2 = item.__dict__ item2.pop("id") item2.pop("_sa_instance_state") item = table_name(**item2) item.archived = True database_session2.add(item) database_session2.commit() database_session2.close() database_session.commit() database_session.close() os.remove(archived_database_path)
def legacy_database_fixer(database_path, database, database_name): database_directory = os.path.dirname(database_path) old_database_path = database_path old_filename = os.path.basename(old_database_path) new_filename = f"Pre_Alembic_{old_filename}" new_database_path = os.path.join(database_directory, new_filename) saved = False if os.path.exists(new_database_path): database_path = new_database_path saved = True Session, engine = db_helper.create_database_session(database_path) database_session = Session() datas = [] result = engine.dialect.has_table(engine, 'alembic_version') if not result: if not saved: os.rename(old_database_path, new_database_path) Session, engine = db_helper.create_database_session(new_database_path) database_session = Session() api_table = database.api_table() media_table = database.media_table() Base = declarative_base() # DON'T FORGET TO REMOVE # database_name = "posts" # DON'T FORGET TO REMOVE legacy_api_table = api_table.legacy(Base, database_name) legacy_media_table = media_table.legacy(Base) result = database_session.query(legacy_api_table) post_db = result.all() for post in post_db: post_id = post.id created_at = post.created_at new_item = {} new_item["post_id"] = post_id new_item["text"] = post.text new_item["price"] = post.price new_item["paid"] = post.paid new_item["postedAt"] = created_at new_item["medias"] = [] result2 = database_session.query(legacy_media_table) media_db = result2.filter_by(post_id=post_id).all() for media in media_db: new_item2 = {} new_item2["media_id"] = media.id new_item2["post_id"] = media.post_id new_item2["links"] = [media.link] new_item2["directory"] = media.directory new_item2["filename"] = media.filename new_item2["size"] = media.size new_item2["media_type"] = media.media_type new_item2["downloaded"] = media.downloaded new_item2["created_at"] = created_at new_item["medias"].append(new_item2) datas.append(new_item) print database_session.close() x = make_metadata(old_database_path, datas, database_name, legacy_fixer=True) print