def __init__(self, mode=None):
self._mode = mode
if self._mode == 'pool':
self.db_pool = cx_Oracle.SessionPool(Config.get_secret('ORA_USERNAME'), Config.get_secret('ORA_PASSWORD'),
Config.get_secret('CONNECTION_STRING'),
encoding="UTF-8", nencoding="UTF-8",
# Pour que les accents soient bien gérés
min=0, max=20, increment=1)
def __init__(self, receiver_email, subject):
self.smtp_server = Config.get_secret("SMTP_SERVER")
self.sender_email = Config.get_secret("SENDER_EMAIL")
if Config.get_secret("WORKING_ENV") == "PROD":
self.server = smtplib.SMTP(self.smtp_server)
self.receiver_email = receiver_email
self.subject = subject
self.is_ok = True
def job_neo_youtuber(number_of_videos_to_retrievse: int = None):
neo_youtuber = NeoYoutuber()
youtube = YoutubeHelper()
if number_of_videos_to_retrievse is not None:
neo_youtuber.max_search_result = number_of_videos_to_retrievse
try:
neo_youtuber.search_videos(youtube_helper=youtube)
for item_number in range(neo_youtuber.max_search_result):
neo_youtuber.get_last_video_url_and_title(item_number=item_number)
neo_youtuber.check_if_video_already_exist()
if not neo_youtuber.is_new_video:
continue
neo_youtuber.retreive_last_video()
neo_youtuber.cut_video_by_scenes()
neo_youtuber.shuffle_and_create_new_video()
neo_youtuber.create_new_thumbnail()
neo_youtuber.upload_to_youtuber()
neo_youtuber.update_database()
except Exception as e:
logging.error(e)
mail_helper = MailHelper(
receiver_email=Config.get_secret("RECEIVER_EMAIL"),
subject=neo_youtuber.mail_subject)
mail_helper.is_ok = neo_youtuber.is_job_success
mail_helper.send_mail()
def list_files(self, path=None):
"""
Lister les fichiers présents dans un dossier
Parameters
----------
path: Path vers le dossier
Return Liste des fichier présents dans le dossier
-------
"""
if Config.get_secret('WORKING_ENV') == 'LOCAL':
return [file for file in os.listdir(Config.get_secret('ZIP_FILES_PATH'))]
files = []
if path is not None:
self.ftp.cwd(path)
self.ftp.dir(files.append)
return [filename.split(' ')[-1] for filename in files]
def __init__(self):
os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1"
self._api_service_name = "youtube"
self._api_version = "v3"
self._service_accpount_secrets = Config.get_secret(
"yt_json_credentials")
self._credentials = Credentials.from_service_account_file(
self._service_accpount_secrets)
self.youtube = googleapiclient.discovery.build(
self._api_service_name,
self._api_version,
credentials=self._credentials)
def send_mail(self):
"""
Envoi du mail et fermeture du serveur.
"""
try:
# Todo à utiliser plutard en prod ("python -m smtpd -c DebuggingServer -n localhost:1025")
if Config.get_secret("WORKING_ENV") == "PROD":
self.server.sendmail(self.sender_email,
self.receiver_email.split(','),
self.create_mail().as_string())
except Exception as e:
# Print any error messages to stdout
print(f"Probleme lors de l'envoi du mail: {e}")
def cx_connection(self):
"""
Méthode permettant la récupération d'une connexion dsn
Returns
-------
Connection
Objet connection cx_Oracle ouvert vers la BDD
"""
database_connection = None
try:
if self._mode == 'pool':
database_connection = self.db_pool.acquire()
else:
dsn_tns = cx_Oracle.makedsn(Config.get_secret('ORA_SERVER'), Config.get_secret('ORA_PORT'),
Config.get_secret('ORA_SID'))
database_connection = cx_Oracle.connect(user=Config.get_secret('ORA_USERNAME'),
password=Config.get_secret('ORA_PASSWORD'),
dsn=dsn_tns, encoding="UTF-8", nencoding="UTF-8")
except Exception as ex_conn:
logging.exception(f' Erreur lors de la connection avec Oracle : {ex_conn}')
except cx_Oracle.DatabaseError as ora_error:
logging.exception(f'cx_oracle error: {ora_error}')
return database_connection
def __init__(self):
super().__init__(job_name="Neo_Youtuber")
self.is_new_video = False
self.mail_subject = [self.job_name]
self.last_video_title = None
self.last_video_id = None
self.last_video_url = None
self.thumbnail_url = None
self.path_to_videos = f"{self.path_to_elements}/videos"
self.path_to_thumbnails = f"{self.path_to_elements}/thumbnails"
self.path_to_cutted_videos = f"{self.path_to_elements}/cutted_videos"
self.path_to_final_result_videos = f"{self.path_to_elements}/final_result_videos"
self.path_to_cat_dog_dataset = f"{self.path_to_elements}/cat_dog_dataset/PetImages"
self.path_to_intro = f"{self.path_to_elements}/intro_outro"
self.yt_id_to_steal_from = "UCzn2gx8zzhF0A4Utk8TEDNQ"
self.yt_url_to_steal_from = "https://www.youtube.com/channel/UCzn2gx8zzhF0A4Utk8TEDNQ"
self.yt_api_key = Config.get_secret("yt_api_key")
self.database = None
self.number_of_scenes = 0
self.cutted_videos = []
self.database_table_name = "neo_youtuber.json"
self.max_search_result = 1
self.search_response = None
import dash_html_components as html
import plotly.graph_objects as go
import pandas as pd
import numpy as np
from dash.dependencies import Input, Output, State, ALL, MATCH
import time
import json
import math
from datetime import datetime
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
from configparser import ConfigParser
from application.data.data_loader import DataLoader
from application.config.config import Config
configuration = Config()
parser = ConfigParser()
parser.read("settings.ini")
data = DataLoader(parser)
data.load_data()
latest_update = data.latest_load.strftime("%m/%d/%Y, %H:%M:%S")
state = {
"indicators": ["cases"],
"regions": ["World"],
"axis": {
"x": "date",
def __init__(self, job_name):
self.job_name = job_name
self.path_to_elements = Config.get_secret(
f"Path_Elements_{self.job_name}")
self.is_job_success = True
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import scoped_session, sessionmaker
from application.config.config import Config
conf = Config()
##Increase Timeout
##Retry incase of sql server gone away error
engine = create_engine(conf.SQLITE_URI,
convert_unicode=True,
connect_args={'connect_timeout': 120},
pool_recycle=3600,
pool_pre_ping=True)
db_session = scoped_session(
sessionmaker(autocommit=False, autoflush=False, bind=engine))
Base = declarative_base()
Base.query = db_session.query_property()
def init_db():
import application.models.UserMaster
Base.metadata.create_all(bind=engine)
def shutdown_db_session():
db_session.remove()