def migrate_to_mongo():
db = Database()
mongo = MongoDatabase()
sites, channels, news = db.get_all()
print(len(sites), len(channels), len(news))
migrate_sites(mongo, sites)
migrate_channels(mongo, channels)
migrate_news(mongo, news)
def __init__(self,
token,
name="",
img="",
short="",
rss="",
db=None,
mongo=None):
self.__img = img
self.__name = name
self.__short = short
self.__rss = rss
self.__channels = []
self.__articles = []
if mongo:
self.__db = MongoDatabase(mongo) if not db else db
else:
self.__db = Database()
def __init__(
self,
site,
counter,
sleeper,
token,
owner=0,
debug=False,
proxy=None,
mongo=None,
error=None,
):
self._site = site
self._sleeper = sleeper
self._token = token
self._owner = owner
self._debug = debug
self._proxy = proxy
self._mongo = mongo
self._error = error
db_name = "news_test" if debug else "news"
self._db = MongoDatabase(mongo,
db_name=db_name) if mongo else Database()
self._start(counter, sleeper)
class Site:
def __init__(self,
token,
name="",
img="",
short="",
rss="",
db=None,
mongo=None):
self.__img = img
self.__name = name
self.__short = short
self.__rss = rss
self.__channels = []
self.__articles = []
if mongo:
self.__db = MongoDatabase(mongo) if not db else db
else:
self.__db = Database()
@property
def name(self):
return self.__name
@name.setter
def name(self, name):
self.__name = name
@property
def img(self):
return self.__img
@img.setter
def img(self, img):
self.__img = img
@property
def rss(self):
return self.__rss
@rss.setter
def rss(self, rss):
self.__rss = rss
@property
def channels(self):
return self.__channels
@channels.setter
def channels(self, channels):
for channel in channels:
if "short" not in channel:
channel["short"] = ""
self.__channels = channels
@property
def short(self):
return self.__short
@short.setter
def short(self, short):
self.__short = short
@property
def articles(self):
return self.__articles
def quick_check_article_exists(self, db, link):
return db.quick_check_news(
link,
re.findall(r"https?://[\-\w.]*/(.*)$", link)[0], self.__name)
def db_update(self, token, debug=False, proxy=None):
site_check = self.__db.check_site(self.__name, self.__rss)
if site_check == 1:
self.__db.update_site(self.__name, self.__rss)
elif site_check == 0:
self.__db.insert_site(self.__name, self.__rss)
for channel in self.__channels:
public = 1 if re.findall("^[A-Za-z_]*$", channel["link"]) else 0
channel_check = self.__db.check_channel(
self.__name,
channel["id"],
channel["link"],
channel["template"],
public,
channel["short"],
)
if channel_check == 1:
logging.info("Need to update {}...".format(self.__name))
self.__db.update_channel(
self.__name,
channel["id"],
channel["link"],
channel["template"],
public,
channel["short"],
)
elif channel_check == 0:
logging.info("Need to insert {}...".format(self.__name))
self.__db.insert_channel(
self.__name,
channel["id"],
channel["link"],
channel["template"],
public,
channel["short"],
)
if not channel_check == 2 or not site_check == 2:
self.init_channel(channel, token, debug=debug, proxy=proxy)
def init_channel(self, channel, token, debug=False, proxy=None):
if debug:
return
logging.info("Update {}...".format(self.__name))
tg = Telegram(token, proxy=proxy)
tg.set_title(
channel["id"],
self.__name + " ⚡️" if channel["template"] == 5 else self.__name,
)
tg.set_description(
channel["id"],
"Hier werden alle Nachrichten aus dem RSS-News-Feed von {} gepostet. "
"🤓\n\nMehr News 📰\n@DerNewsChannel\n\nBei Fragen und Problemen "
"🤔\n@DerNewsChat".format(self.__name),
)
def add_article(self,
title="",
text="",
img="",
link="",
tags: list = None,
date=""):
n = news.Article(self.name, short=self.short)
if title:
n.title = title
if text:
n.text = text
if img:
n.img = img
if link:
n.link = link
if tags:
n.tags = tags
if date:
n.date = date
self.__articles.append(n)
def add_channel(self, c_id=0, name="", template=0, c_hash=""):
pass
def post(self,
token,
variant=0,
share_link=0,
owner=0,
debug=False,
proxy=None):
for article in self.__articles[::-1]:
if len(self.__articles) > 10:
time.sleep(2)
result = bool(self.__channels)
for channel in self.__channels:
ihash = channel["hash"] if "hash" in channel else ""
result = result and article.send(
self.__db,
channel["id"],
channel["template"],
channel["link"],
channel["short"],
token,
share_link=share_link,
instant_hash=ihash,
owner=owner,
debug=debug,
proxy=proxy,
)
if result:
article.insert_db(self.__db)
def check_article_exists(self, link):
db = Database()
return db.check_news(link) or db.check_news(
re.findall(r"https?://[\-\w.]*/(.*)$", link)[0])
def clean_up_db():
db = Database()
db.clean_up()
def send_summary(serious=False, full=False, fast=False):
db = Database()
articles = [{
"name": a[2].replace(".", "․"),
"count": a[1],
"date": a[0],
"added": a[3]
} for a in db.count_articles()]
site_list = {}
date_list = {}
results = {}
for a in articles:
if a["name"] not in site_list:
site_list[a["name"]] = []
date_list[a["name"]] = a["added"]
site_list[a["name"]].append({
"count":
a["count"],
"date":
datetime.datetime.strptime(a["date"], "%Y-%m-%d"),
})
for site, data in sorted(site_list.items(), key=lambda s: s[0].lower()):
try:
channels = next(
(item for item in get_channels("channels.json")
if item["name"] == site.replace("․", ".")),
None,
)
except TypeError:
continue
if not channels:
continue
mini = min([x["date"] for x in data])
days = abs((datetime.datetime.today() - mini).days)
counter = 0
for x in data:
counter += x["count"]
if channels["category"] not in results:
results[channels["category"]] = "*{}*\n".format(
channels["category"].upper())
if (datetime.datetime.today() - datetime.datetime.strptime(
date_list[site], "%Y-%m-%d")).days > 13:
results[channels["category"]] += "\n*{}* (Ø{} M/Tag)\n".format(
site, "{:.1f}".format(counter / days).replace(".", ","))
else:
results[channels["category"]] += "\n*{} 🆕* \n".format(site)
try:
for channel in channels["channels"]:
try:
members = "" # "[[{}👥]]".format(db.get_members(channel['id'])[0])
except IndexError:
members = ""
if not channel["link"][:4] == "http":
results[channels["category"]] += "@{} {}\n".format(
channel["link"].replace("_", "\_"), members)
elif channel["template"] == 5:
results[channels[
"category"]] += "[⚡️ Instant View]({}) {}\n".format(
channel["link"], members)
else:
results[channels["category"]] += "[{}]({}) {}\n".format(
site, channel["link"], members)
except TypeError:
continue
other = "*SONSTIGES*\n\n"
tg = Telegram()
for category, text in sorted(results.items()):
if text.count("\n") < 6:
other += text + "\n\n"
results.pop(category, "")
results["Sonstiges"] = other
results["Info"] = INFO
ORDER = ["News", "Technik"]
i = 0
channel_id = CHANNEL_TEST if not serious else CHANNEL_MAIN
send = (["Info", "Sonstiges"] + [
c
for c in results if c not in ORDER and not c in ["Sonstiges", "Info"]
] + ORDER[::-1])
if not full:
for category in send:
i = tg.edit(
i,
results[category],
channel_id,
{},
disable_notification=True,
disable_preview=True,
)
else:
tg.unpin(channel_id)
if not fast:
time.sleep(300)
first = tg.send(
results[send[0]],
channel_id,
{},
disable_notification=True,
disable_preview=True,
)
for i in reversed(range(first)):
tg.delete(i, channel_id)
tg.pin(channel_id, first)
time.sleep(5)
for category in send[1:]:
tg.send(
results[category],
channel_id,
{},
disable_notification=True,
disable_preview=True,
)
time.sleep(1)
def test_parallel_speed_check(benchmark, number=100, processes=10):
parallel_benchmark_db(benchmark, Database(), number, processes)
def test_sql_check_speed(benchmark, number=5):
benchmark_database(benchmark, Database(), Database(), number)
def test_mongo_check_speed(benchmark, number=1000):
benchmark_database(benchmark, Database(), MongoDatabase(), number)