def __init__(self):
self.collection = get_collection_from_db()
self.options = {
# 'format': 'bestaudio/best',
'outtmpl': f'{DL_PATH}%(id)s-%(title)s-%(format)s.%(ext)s',
'noplaylist': True,
# progress_hooks': [my_hook],
# 'simulate': True,
'writeinfojson': True,
# 'download_archive': DL_PATH,
'include_ads': False,
'call_home': False,
'sleep_interval': DL_DELAY,
'progress_hooks': [self.dl_hook],
'format': 'best[height<=360][width<=640]', # Download in 360p or lower,
'verbose': True,
# 'force-ipv4': True,
'print-traffic': False,
# 'proxy': ''
}
self.ydl = YoutubeDL(self.options)
import pandas as pd
from src.database.db_utils import get_collection_from_db
__DB_CONNECTION = get_collection_from_db()
def get_metadata(v_id: str, keys: list) -> dict:
item = __DB_CONNECTION.find_one({'v_id': v_id})
if not item:
raise KeyError(f'V_id: {v_id} not found in database!')
return dict([(key, item.get(key, 'Failed')) for key in keys])
def get_metaframe(df: pd.DataFrame, keys: list = None) -> pd.DataFrame:
if keys is None:
keys = [
'song_name', 'creator', 'v_likes', 'v_dislikes', 'v_duration',
'v_avg_rating'
]
df = df.copy()
df.reindex(columns=[*df.columns.tolist(), *keys])
for idx, v_id in df.iterrows():
for key, val in get_metadata(v_id.iloc[0], keys).items():
df.at[idx, key] = val
return df
def __init__(self):
self.collection = get_collection_from_db()
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.lfm_generator = Loader(CACHE_PATH).shuffled_list(N_CRAWLS)
self.collection = get_collection_from_db()
def __init__(self):
self.collection = get_collection_from_db()
self.resolution = RES_RSCLD
def reset_failed():
collection = get_collection_from_db()
collection.update_many({'v_filepath': FAIL_PATH}, {'$set': {'v_filepath': EMPTY_PATH}})
def __init__(self, seed=42):
self.collection = get_collection_from_db()
self.seed = seed