def clear(): _ = system('cls') sleep(3) if GetConfig("BASE", "getallurl") == 'True': massurls.get_urls(GetConfig("FILTER", "sitemap")) else: autoparse()
def takethis(get): print(123) i = 0 soup = BeautifulSoup(get, 'html.parser') with open('temp.txt', 'w', encoding='utf-8') as g: for get in soup.find_all("loc", None): y = GetConfig("FILTER", "filterurl") print(get) g.write("%s\n" % get) i = i + 1 if i == -1: CloseApp('Достигнут конец файла') else: try: r = open('temp.txt', encoding='utf-8').read().splitlines(1) with open(GetConfig("BASE", "parsefile"), 'w', encoding='utf-8') as f: for tt in r: tt = tt.replace(' ', '') a = tt[5: -7] f.write("%s\n" % a) i = i - 1 except FileNotFoundError: log(5, 'MAIN', 'File >temp.txt< not found') print(code.ERR, 'Временный файл не найден, подробнее в > report.log') sleep(GetConfig("BASE", "pauseonerror")) CloseApp('Временный файл не найден') f.close() try: os.remove('temp.txt') except OSError: pass
def get(self) -> (dict, int): """ GET request endpoint. Send POST request to Sendgrid to test the validity of the current api key :return: a dictionary containing boolean value indicating the validity of the api key and a reason if the key is invalid """ if get_jwt_claims()["admin"]: sendgrid_api_key = GetConfig.configure('sendgrid', 'api_key') if sendgrid_api_key: sendgrid_response = \ SendgridHelper.send_test_request(sendgrid_api_key) return \ SendgridHelper.handle_sendgrid_response(sendgrid_response) else: logger.error("Sendgrid API key not present " "in configurations file") return { "api_key": False, "reason": "Invalid Sendgrid API key" }, 200 else: return {"message": "Not Authorized"}, 403
def replace_sendgrid_key(new_api_key: str) -> (bool, str): """ Replace Sendgrid API key environment variable export statement within the provided file argument :param new_api_key: Sendgrid API key that will replace the current :return: Whether the replacement process was successful and a reason if the process failed """ try: config = GetConfig.configure() config["sendgrid"]["api_key"] = new_api_key GetConfig.save_config(config) return True, None except IOError as ioe: logger.error("New Sendgrip API Key not committed to config file", ioe.with_traceback(ioe.__traceback__)) return False, "Could not replace API key as it does not exist"
def get_urls(url): EditConfig('BASE', 'getallurl', 'False') get = get_html(url) if get.status_code == 200: massurls.takethis(get.text) else: log(4, 'MAIN', f'Response: {get.status_code}') print(code.ERR, f'Что-то пошло не так, код ошибки: {get.status_code}') sleep(GetConfig("BASE", "pauseonerror")) onetwo()
def download_url(url, filename, file): try: with DownloadProgressBar(unit='B', unit_scale=True, miniters=1, desc=filename) as t: urllib.request.urlretrieve(url, filename=filename, reporthook=t.update_to) try: shutil.move(os.path.abspath('.')+'//'+filename, PATH+'//'+filename) log(2, 'MAIN', 'Download - Done. File moved to folder') print('\n',code.LOG,'Вызван мастер загрузки. Это может занять некоторое время...') call_upload(file) sleep(4) onetwo() except FileNotFoundError: log(4, 'MAIN', f'FileNotFoundError >{filename}< - shutil.move in download_url') print(code.ERR,'Ошибка при перемещении загруженного видео в папку, подробнее в > report.log') sleep(GetConfig("BASE", "pauseonerror")) onetwo() except urllib.error.URLError: log(4, 'MAIN', 'Failed to start download, it may not be possible to access the file') print(code.ERR,'Не удалось начать скачивание, возможно нету доступа к файлу.') sleep(GetConfig("BASE", "pauseonerror")) onetwo()
def autoparse(): if len(lines1) == NUM: CloseApp('Достигнут конец файла') else: print(code.INF, f'Загружено {NUM} из {len(lines1)}') log(2, 'MAIN', f'Uploaded {NUM} of {len(lines1)}') with open(GetConfig("BASE", "parsefile"), "r", encoding="utf-8") as RLL: URL = RLL.readlines()[NUM] print(URL) y = GetConfig("FILTER", "filterurl") if fuzz.partial_ratio(URL, y) > 90: get = get_html(URL) if get.status_code == 200: get_content((get.text)) else: log(4, 'MAIN', f'Response: {get.status_code}') print(code.ERR, f'Что-то пошло не так, код ошибки: {get.status_code}') sleep(GetConfig("BASE", "pauseonerror")) onetwo() else: log(4, 'MAIN', f'Over 90% link match was expected. The entered one matches only on {fuzz.partial_ratio(URL, y)}%') print(code.ERR,f'Ожидалось совпадение ссылки более 90%. Введенная совпадает лишь на {fuzz.partial_ratio(URL, y)}%') sleep(GetConfig("BASE", "pauseonerror")) onetwo()
import unittest from datetime import datetime import sqlalchemy from sqlalchemy.orm import sessionmaker, scoped_session from app import create_app from db import db from models.attribute_range import AttributeRange from models.attributes import Attributes from models.users import Users sys.path.append('../..') from settings import GetConfig config = GetConfig.configure('postgres') class AlertWidgetDecoratorTestCase(unittest.TestCase): """ Test whether the minimum and maximum values of attributes are persisted after an importer is run """ def setUp(self): """ Create testing client version of flask app along with a database session and then generate an admin JWT """ self.test_app = create_app(DATABASE_NAME='scheduler',
def get_content(get): soup = BeautifulSoup(get, 'html.parser') log(2, 'MAIN', 'Filtering the received content') print(code.LOG,'Фильтруем полученный контент') i = 0 for get in soup.find_all("meta"): if get.get("itemprop", None) == "name": name = get.get("content", None) file1 = get.get("content", None) name = name.replace('–', '\nНазвание:') try: with open(PATH+'//'+file1+'.txt', "r", encoding="utf-8") as check: log(2, 'MAIN', f'File already exists > {file1}') a = GetConfig("BASE", "forceupload") if a == 'True': if exists(PATH+'//'+file1+'.mp4') == True: print(code.LOG, 'Обнаружен незагруженный файл, вызван мастер загрузки.\nЭто может занять некоторое время...') call_upload(file1) onetwo() else: print(code.WARN, "Файл уже существует и был загружен ранее, пропускаем.") log(3, 'MAIN', 'Additional content by filters is written to a file') onetwo() else: print(code.WARN, "Файл уже существует!") onetwo() except FileNotFoundError: pass elif get.get("itemprop", None) == "uploadDate": uploaddate = get.get("content", None) f = open(PATH + '//' + file1 + '.txt', 'a', encoding='utf-8') if len(GetConfig("FILTER", "studio")) > 3: log(2, 'MAIN', 'Studio filter enable, check settings.ini for change') if fuzz.partial_ratio(GetConfig("FILTER", "studio"), name) > 93: f.write("Студия: #%s\n" % name) f.write("Актеры: ") print(code.INF,'Дополнительный контент по фильтрам записан в файл.') log(2, 'MAIN', 'Additional content by filters is written to a file') else: f.close() try: os.remove(PATH+'//'+file1+'.txt') log(2, 'MAIN', 'File %s deleted' % PATH+'//'+file1+'.txt') print(code.LOG, 'Студия не совпадает с указанной в фильтре, пропускаем.') log(2, 'MAIN', 'Studio does not match in filter, skip') onetwo() except Exception as f: log(4, 'MAIN', f'Error with >> %s. {f}' % PATH + '//' + file1 + '.txt') print(code.ERR,'Произошла ошибка при попытке удаления файла, подробнее в > report.log') sleep(GetConfig("BASE", "pauseonerror")) onetwo() else: log(2, 'MAIN', 'Studio filter disable, check settings.ini for change') f.write("Студия: #%s\n" % name) f.write("Актеры: ") f.close() print(code.INF, 'Дополнительный контент по фильтрам записан в файл.') log(2, 'MAIN', 'Additional content by filters is written to a file') for get in soup.find_all("div", class_="tags-list"): y = '<a class ="label" href="https://xxxmax.net/actor/' tag = get #Вроде костыль а удобно for tag in soup.find_all('a', href=True, title=True): if fuzz.partial_ratio(tag, y) > 97: with open(PATH + '//' + file1 + '.txt', 'a', encoding='utf-8') as g: name1 = tag.get("title").replace(' ','_') g.write("#%s, " % name1) for get in soup.find_all("iframe"): if get.get("src", None): urlll = get.get("src", None) y = GetConfig("FILTER", "filtervid") if fuzz.partial_ratio(urlll, y) == 100: print(code.LOG,'Отловили ссылку к ролику, пытаемся скачать.') video = get_html(urlll) print(code.LOG,'Статус:',video.status_code) log(2, 'MAIN', f'Response: {video.status_code}') sleep(1) soup = BeautifulSoup(video.text, 'html.parser') for video in soup.find_all("source"): if video.get("src", None): with open(PATH + '//' + file1 + '.txt', 'a', encoding='utf-8') as file: file.write("\nДата загрузки: %s" % uploaddate[0:10]) link = video.get("src", None) log(2, 'MAIN', f'Download started: {link}, {file1}') download_url(link, file1+'.mp4', file1) else: log(4, 'MAIN', f'Failed to start download video. More details: {file1}, RESP:{video.status_code}.') print(code.ERR,'Не удалось начать загрузку видео.') sleep(GetConfig("BASE", "pauseonerror")) onetwo()
import urllib.request import colorama import os import subprocess colorama.init() from fuzzywuzzy import fuzz from tqdm import tqdm from time import sleep from os import system from bs4 import BeautifulSoup from colorstatscode import code from settings import GetConfig, EditConfig log(1, 'MAIN', 'Stage passed') PATH = (os.path.abspath('.')+'/content') HEADERS = {'user-agent': GetConfig("BASE", "useragent"), 'accept': '*/*'} global NUM NUM = 0 def call_upload(file): txtname = ('content/%s.txt' % file) videoname = ('content/%s.mp4' % file) exw(videoname, txtname) if os.path.exists('content'): log(2, 'MAIN', 'Folder >content< already created') pass else: os.mkdir("content") print(code.WARN, "Создана папка 'content' для хранения загруженных файлов.")
async def uploader(nameclip, nametxttoload, namecliptoload): await client.start() dialogs = await client.get_dialogs() for dialog in dialogs: print("%s - %s" % (dialog.name, dialog.entity.id)) chat_id = GetConfig("LOADER", "channel") chat_id2 = GetConfig("LOADER", "channel2") #nameclip = 'rr.mp4' #nametxttoload = 'Task-1.txt' #namecliptoload = 'Temp\\5a746c60-1acf-4c62-9fc8-20d7ee0de6c9done.mp4' #chat_id = '1213129356' #chat_id2 = '1266958853' entity = None entity2 = None dialogs = await client.get_dialogs() for dialog in dialogs: id = str(dialog.entity.id) if id == chat_id: entity = dialog.entity break if not entity: sys.exit(1) dialogs1 = await client.get_dialogs() for dialog1 in dialogs1: id1 = str(dialog1.entity.id) if id1 == chat_id2: entity2 = dialog1.entity break if not entity: sys.exit(1) start = time.time() async for message1 in client.iter_messages(entity2, limit=1): print('TAKE1') print('Gat it:', message1.id) async for message1 in client.iter_messages(entity2, limit=1): print('TAKE2') print('Gat it:', message1.id) try: with open(nametxttoload, "r", encoding="utf-8") as t1: t11 = t1.readlines()[0:4] print('STAGE1') try: async for message1 in client.iter_messages(entity2, limit=1): print('TAKE3') print('Gat it:', message1.id) await client.send_file(entity, nameclip, video_note=True, supports_streaming=True, caption="".join(t11), filename=nameclip, timeout=1200) await asyncio.sleep(4) async for message in client.iter_messages(entity2, limit=1): print('TAKE4') print('Gat it:', message.id) await client.send_file(entity2, namecliptoload, reply_to=message.id, video_note=True, supports_streaming=True, filename=nameclip) async for message1 in client.iter_messages(entity2, limit=1): print('TAKE5') print('Gat it:', message1.id) except Exception as e: await client.send_message( entity, f"[UPLOADER] Произошла ошибка во время загрузки\n\n**Ошибка:** {e}" ) except Exception as e: await client.send_message( entity, f"[UPLOADER] Произошла ошибка во время загрузки\n\n**Ошибка:** {e}" ) await client.disconnect() try: os.remove(nameclip) os.remove(namecliptoload) sys.exit(0) except Exception: sys.exit(1)
#! /usr/bin/env python # -*- coding: utf-8 -*- import asyncio import time import sys import os import utils from telethon.client.telegramclient import TelegramClient from logger import log, CloseApp from settings import GetConfig from telethon import events from utils import progress, humanbytes, time_formatter, convert_from_bytes api_id = GetConfig("LOADER", 'apiid') #'1926838'#keyline[0] api_hash = GetConfig("LOADER", 'apihash') #'bd5be9e930902b2a02f0396932dddabb'#keyline[1] try: client = TelegramClient('loader', api_id, api_hash) except Exception as ap: sys.exit(1) async def uploader(nameclip, nametxttoload, namecliptoload): await client.start() dialogs = await client.get_dialogs() for dialog in dialogs: print("%s - %s" % (dialog.name, dialog.entity.id))