def __init__(
self,
api: start,
option: dict[str, Any] = {},
pool: Optional[Pool] = None,
max_threads: int = -1,
) -> None:
self.api = api
create_user.__init__(self, option, self)
if not self.username:
self.username = f"u{self.id}"
self.lists = []
self.links = api.ContentTypes()
self.subscriptions: list[create_user] = []
self.chats = None
self.archived_stories = {}
self.mass_messages = []
self.paid_content: list[create_message | create_post] = []
temp_pool = pool if pool else api_helper.multiprocessing()
self.pool = temp_pool
self.session_manager = self._session_manager(self, max_threads=max_threads)
self.auth_details = auth_details()
self.guest = False
self.active: bool = False
self.errors: list[ErrorDetails] = []
self.extras: dict[str, Any] = {}
def __init__(
self,
option: dict[str, Any] = {},
pool: Pool = None,
max_threads: int = -1,
) -> None:
create_user.__init__(self, option)
if not self.username:
self.username = f"u{self.id}"
self.lists = {}
self.links = content_types()
self.subscriptions: list[create_user] = []
self.chats = None
self.archived_stories = {}
self.mass_messages = []
self.paid_content = []
temp_pool = pool if pool else api_helper.multiprocessing()
self.pool = temp_pool
self.session_manager = api_helper.session_manager(
self, max_threads=max_threads, use_cookies=False)
self.auth_details: auth_details = auth_details()
self.profile_directory = option.get("profile_directory", "")
self.guest = False
self.active: bool = False
self.errors: list[error_details] = []
self.extras: dict[str, Any] = {}
def fix_directories(posts, base_directory, site_name, api_type, media_type,
username, all_files, json_settings):
new_directories = []
def fix_directory(post):
new_post_dict = post.convert(keep_empty_items=True)
for media in post.medias:
if media.links:
path = urlparse.urlparse(media.links[0]).path
else:
path = media.filename
new_filename = os.path.basename(path)
filename, ext = os.path.splitext(new_filename)
ext = ext.replace(".", "")
file_directory_format = json_settings["file_directory_format"]
filename_format = json_settings["filename_format"]
date_format = json_settings["date_format"]
text_length = json_settings["text_length"]
download_path = base_directory
today = datetime.today()
today = today.strftime("%d-%m-%Y %H:%M:%S")
new_media_dict = media.convert(keep_empty_items=True)
option = {}
option = option | new_post_dict | new_media_dict
option["site_name"] = site_name
option["filename"] = filename
option["api_type"] = api_type
option["media_type"] = media_type
option["ext"] = ext
option["username"] = username
option["date_format"] = date_format
option["maximum_length"] = text_length
option["directory"] = download_path
prepared_format = prepare_reformat(option)
file_directory = main_helper.reformat(prepared_format,
file_directory_format)
prepared_format.directory = file_directory
old_filepath = ""
x = [x for x in all_files if media.filename == os.path.basename(x)]
if x:
# media.downloaded = True
old_filepath = x[0]
old_filepath = os.path.abspath(old_filepath)
print
new_filepath = main_helper.reformat(prepared_format,
filename_format)
if prepared_format.text:
pass
setattr(media, "old_filepath", old_filepath)
setattr(media, "new_filepath", new_filepath)
new_directories.append(os.path.dirname(new_filepath))
pool = multiprocessing()
pool.starmap(fix_directory, product(posts))
new_directories = list(set(new_directories))
return posts, new_directories
def __init__(self, api: api_types, config: Config) -> None:
self.api = api
self.max_threads = config.settings.max_threads
self.config = config
self.lists = None
self.pool: Pool = api_helper.multiprocessing()
global_settings = self.get_global_settings()
site_settings = self.get_site_settings()
profile_root_directory = main_helper.check_space(
global_settings.profile_directories
)
root_metadata_directory = main_helper.check_space(
site_settings.metadata_directories
)
root_download_directory = main_helper.check_space(
site_settings.download_directories
)
self.base_directory_manager = DirectoryManager(
site_settings,
profile_root_directory,
root_metadata_directory,
root_download_directory,
)
def fix_directories(posts, all_files, Session, folder, site_name, parent_type,
api_type, username, base_directory, json_settings):
new_directories = []
def fix_directories(post):
final_type = ""
if parent_type:
final_type = f"{api_type}{os.path.sep}{parent_type}"
print
final_type = final_type if final_type else api_type
database_session = Session()
post_id = post.id
result = database_session.query(folder.media_table)
media_db = result.filter_by(post_id=post_id).all()
for media in media_db:
if media.link:
path = urlparse.urlparse(media.link).path
else:
path = media.filename
new_filename = os.path.basename(path)
filename, ext = os.path.splitext(new_filename)
ext = ext.replace(".", "")
file_directory_format = json_settings["file_directory_format"]
filename_format = json_settings["filename_format"]
date_format = json_settings["date_format"]
text_length = json_settings["text_length"]
download_path = base_directory
today = datetime.today()
today = today.strftime("%d-%m-%Y %H:%M:%S")
option = {}
option["site_name"] = site_name
option["post_id"] = post_id
option["media_id"] = media.id
option["username"] = username
option["api_type"] = final_type if parent_type else api_type
option["media_type"] = media.media_type
option["filename"] = filename
option["ext"] = ext
option["text"] = post.text
option["postedAt"] = media.created_at
option["price"] = post.price
option["date_format"] = date_format
option["text_length"] = text_length
option["directory"] = download_path
prepared_format = prepare_reformat(option)
file_directory = main_helper.reformat(prepared_format,
file_directory_format)
prepared_format.directory = file_directory
old_filepath = ""
old_filepaths = [
x for x in all_files if media.filename in os.path.basename(x)
]
if not old_filepaths:
old_filepaths = [
x for x in all_files
if str(media.id) in os.path.basename(x)
]
print
if old_filepaths:
old_filepath = old_filepaths[0]
print
new_filepath = main_helper.reformat(prepared_format,
filename_format)
if old_filepath and old_filepath != new_filepath:
if os.path.exists(new_filepath):
os.remove(new_filepath)
if os.path.exists(old_filepath):
if media.size:
media.downloaded = True
moved = None
while not moved:
try:
moved = shutil.move(old_filepath, new_filepath)
except OSError as e:
print(traceback.format_exc())
print
print
else:
print
if prepared_format.text:
pass
media.directory = file_directory
media.filename = os.path.basename(new_filepath)
database_session.commit()
new_directories.append(os.path.dirname(new_filepath))
database_session.close()
pool = multiprocessing()
pool.starmap(fix_directories, product(posts))
new_directories = list(set(new_directories))
return posts, new_directories
def fix_directories(posts, all_files, database_session: scoped_session, folder,
site_name, parent_type, api_type, username, base_directory,
json_settings):
new_directories = []
def fix_directories(post: api_table, media_db: list[media_table]):
delete_rows = []
final_type = ""
if parent_type:
final_type = f"{api_type}{os.path.sep}{parent_type}"
final_type = final_type if final_type else api_type
post_id = post.post_id
media_db = [x for x in media_db if x.post_id == post_id]
for media in media_db:
media_id = media.media_id
if media.link:
path = urlparse.urlparse(media.link).path
else:
path: str = media.filename
new_filename = os.path.basename(path)
original_filename, ext = os.path.splitext(new_filename)
ext = ext.replace(".", "")
file_directory_format = json_settings["file_directory_format"]
filename_format = json_settings["filename_format"]
date_format = json_settings["date_format"]
text_length = json_settings["text_length"]
download_path = base_directory
today = datetime.today()
today = today.strftime("%d-%m-%Y %H:%M:%S")
option = {}
option["site_name"] = site_name
option["post_id"] = post_id
option["media_id"] = media_id
option["username"] = username
option["api_type"] = final_type if parent_type else api_type
option["media_type"] = media.media_type
option["filename"] = original_filename
option["ext"] = ext
option["text"] = post.text
option["postedAt"] = media.created_at
option["price"] = post.price
option["date_format"] = date_format
option["text_length"] = text_length
option["directory"] = download_path
option["preview"] = media.preview
prepared_format = prepare_reformat(option)
file_directory = main_helper.reformat(prepared_format,
file_directory_format)
prepared_format.directory = file_directory
old_filepath = ""
if media.linked:
filename_format = f"linked_{filename_format}"
old_filepaths = [
x for x in all_files
if original_filename in os.path.basename(x)
]
if not old_filepaths:
old_filepaths = [
x for x in all_files
if str(media_id) in os.path.basename(x)
]
print
if not media.linked:
old_filepaths = [
x for x in old_filepaths if "linked_" not in x
]
if old_filepaths:
old_filepath = old_filepaths[0]
new_filepath = main_helper.reformat(prepared_format,
filename_format)
if old_filepath and old_filepath != new_filepath:
if os.path.exists(new_filepath):
os.remove(new_filepath)
moved = None
while not moved:
try:
if os.path.exists(old_filepath):
if media.size:
media.downloaded = True
found_dupes = [
x for x in media_db if
x.filename == new_filename and x.id != media.id
]
delete_rows.extend(found_dupes)
os.makedirs(os.path.dirname(new_filepath),
exist_ok=True)
if media.linked:
if os.path.dirname(
old_filepath) == os.path.dirname(
new_filepath):
moved = shutil.move(
old_filepath, new_filepath)
else:
moved = shutil.copy(
old_filepath, new_filepath)
else:
moved = shutil.move(old_filepath, new_filepath)
else:
break
except OSError as e:
print(traceback.format_exc())
print
print
if os.path.exists(new_filepath):
if media.size:
media.downloaded = True
if prepared_format.text:
pass
media.directory = file_directory
media.filename = os.path.basename(new_filepath)
new_directories.append(os.path.dirname(new_filepath))
return delete_rows
result = database_session.query(folder.media_table)
media_db = result.all()
pool = multiprocessing()
delete_rows = pool.starmap(fix_directories, product(posts, [media_db]))
delete_rows = list(chain(*delete_rows))
for delete_row in delete_rows:
database_session.query(folder.media_table).filter(
folder.media_table.id == delete_row.id).delete()
database_session.commit()
new_directories = list(set(new_directories))
return posts, new_directories