def msg_monitor(bot: Client, m: Message):
"""
Сохраняет в БД все сообщения кроме API и Secret ключей
Полезно для анализа взаимодействия с ботом
"""
with db.atomic():
Msg.create(tg_id=m.from_user.id, text=m.text)
def get_drink_type(self, drink_type):
logger.debug("Getting Drinky Type %s ", drink_type)
try:
with db.atomic():
return DrinkType.create(name=drink_type)
except ieerror:
return DrinkType.get(DrinkType.name == drink_type)
def quotes():
form = QuoteAddForm()
form.collections.choices = [(collection.name, collection.name)
for collection in Collection.select().where(
Collection.user == current_user.get_id())]
if form.validate_on_submit():
with db.atomic() as txn:
quote = Quote.create(
content=form.content.data,
author=form.author.data,
user=current_user.get_id(),
)
for collection_name in form.collections.data:
collection = Collection.get(
Collection.name == collection_name,
Collection.user == current_user.get_id(),
)
QuoteCollection.create(
quote=quote,
collection=collection,
)
return redirect(url_for('quotes'))
else:
quotes = Quote.select().where(Quote.user == current_user.get_id())
return render_template('quotes.html', form=form, quotes=quotes)
def welcoming_task(message):
chat_id = message.chat.id
new_members_names = []
new_members_info = []
users = []
for member in message.new_chat_members:
new_members_names.append(member.first_name)
new_members_info.append(user_info(member))
if not member.is_bot:
user = {
'user_id' : member.id,
'chat_id' : chat_id,
'first_name': member.first_name,
'last_name' : member.last_name,
'is_member' : True
}
users.append(user)
if users: # Bot check
with db.atomic():
User.insert_many(users).on_conflict_replace().execute()
if str(message.chat.id) == config.mm_chat:
welcoming_msg = "{}, {}!\nЕсли здесь впервые, то ознакомься с правилами — /rules, " \
"и представься, если несложно.".format(random.choice(config.welcome_list),
', '.join(new_members_names))
else:
welcoming_msg = "{}, {}!\n".format(random.choice(config.welcome_list), ', '.join(new_members_names))
my_bot.reply_to(message, welcoming_msg)
action_log("User(s) {} joined the chat.".format(', '.join(new_members_info)))
async def jobs_dispatcher():
if Worker.select().count() == 0:
for i in range(1):
Worker.create(state="available")
workers = Worker.select().where(Worker.state == "available")
# no available workers, wait
if workers.count() == 0:
return
with db.atomic('IMMEDIATE'):
jobs = Job.select().where(Job.state == "scheduled")
# no jobs to process, wait
if jobs.count() == 0:
await asyncio.sleep(3)
return
for i in range(min(workers.count(), jobs.count())):
job = jobs[i]
worker = workers[i]
job.state = "running"
job.started_time = datetime.now()
job.save()
worker.state = "busy"
worker.save()
jobs_in_memory_state[job.id] = {
"worker": worker.id,
"task": asyncio.ensure_future(run_job(worker, job)),
}
def day_time_selection():
if request.method == "POST":
final_selected = request.form.getlist("selected_course")
session_user = session.get("user_id")
for item in final_selected:
new_record = SelectedCourseInfo.create(faculty_id=session_user, course_id=item)
try:
with db.atomic():
new_record.save()
print(new_record)
except Exception as e:
flash("Something went wrong.")
return redirect(url_for("day_time_selection"))
flash("Your selected course has been sent to Moderator for review")
return redirect(url_for("faculty_selection"))
if request.method == "GET":
selected_courses = []
for item in request.args.getlist("selected_courses"):
selected_courses.append(Courses.get(id=item))
selected_term = Semester.get(id=request.args['selected_term'])
return render_template(
"select_days_and_time.html",
selected_term=selected_term,
selected_courses=selected_courses
)
def parse_petition(petitions):
for petition in petitions:
# get_petition_status(petition)
data = []
petition_url = 'https://petition.president.gov.ua/petition/' + str(petition)
pages = [petition_url + '/votes/' + str(i) for i in range(1, max_page(petition_url) + 1)]
for page in pages:
html = requests.get(page).text
soup = BeautifulSoup(html, 'lxml')
rows = soup.find_all('div', class_=re.compile(r'^table_row$'))
for r in rows:
position_number = r.find('div', class_='table_cell number').string.replace('.', '')
username = r.find('div', class_='table_cell name').string
day, month, year = r.find('div', class_='table_cell date').string.split(' ')
new_month = m.parse(month)[0].inflect({'nomn'}).word.title()
sign_date = datetime.strptime(' '.join([day, new_month, year]), '%d %B %Y')
data.append((petition, position_number, username, sign_date))
if Vote.select().where(Vote.petition == petition):
print(f'petition {petition} was in db with',
Vote.delete().where(Vote.petition == petition).execute(), 'rows')
with db.atomic():
# by default SQLite limits the number of bound variables in a SQL query to 999
for batch in chunked(data, 249):
Vote.insert_many(batch, fields=['petition', 'position_number', 'username', 'sign_date']).execute()
status = get_petition_status(petition)
Petition.update(status=status).where(Petition.petition_id == petition).execute()
if petitions:
set_gender()
def parse_rss(self):
# self.remove_old_history()
rss_list = Rss.select()
rss_card_dict = {}
post_url_list = [
rss_history.url for rss_history in History.select().where(
History.publish_at == datetime.today().strftime("%Y-%m-%d"))
]
for rss in rss_list:
rss_history_list = []
card_list = [
CardItem(title=rss.title, url=rss.url, pic_url=rss.cover)
]
feed = feedparser.parse(rss.feed)
for entry in feed.entries:
if entry.link not in post_url_list and self.is_today(entry):
card_list.append(
CardItem(
title=f'{entry.title}',
url=entry.link,
pic_url=
'https://ftp.bmp.ovh/imgs/2020/07/6cdb9f606677c9e3.jpg'
))
rss_history_list.append(History(url=entry.link))
if len(card_list) > 1:
rss_card_dict[rss.title] = card_list
with db.atomic():
History.bulk_create(rss_history_list, batch_size=10)
return rss_card_dict
def welcoming_task(message):
chat_id = message.chat.id
new_members_names = []
new_members_info = []
users = []
for member in message.new_chat_members:
new_members_names.append(member.first_name)
new_members_info.append(user_info(member))
if not member.is_bot:
user = {
'user_id': member.id,
'chat_id': chat_id,
'first_name': member.first_name,
'last_name': member.last_name,
'is_member': True
}
users.append(user)
if users: # Bot check
with db.atomic():
User.insert_many(users).on_conflict_replace().execute()
if str(message.chat.id) == config.mm_chat:
welcoming_msg = "{}, {}!\nЕсли здесь впервые, то ознакомься с правилами — /rules, " \
"и представься, если несложно.".format(random.choice(config.welcome_list),
', '.join(new_members_names))
else:
welcoming_msg = "{}, {}!\n".format(random.choice(config.welcome_list),
', '.join(new_members_names))
my_bot.reply_to(message, welcoming_msg)
action_log("User(s) {} joined the chat.".format(
', '.join(new_members_info)))
def get_user(self, username):
logger.debug("Getting user %s ", username)
try:
with db.atomic():
return User.create(username=username)
except ieerror:
return User.get(User.username == username)
async def create_or_update_proxy(raw_protocol: Proxy.PROTOCOLS,
auth_data: str, domain: str, port: int,
start_checking_time: int,
end_checking_time: int,
additional_info: CheckerResult):
if raw_protocol is None or domain is None or port is None or auth_data is None or start_checking_time is None\
or end_checking_time is None:
raise ValueError("Processor.create_or_update_proxy: Bad arguments")
if raw_protocol < 0 or raw_protocol >= len(Proxy.PROTOCOLS):
raise ValueError("Processor.create_or_update_proxy: Bad protocol")
# to microseconds
response_time = int(
round((end_checking_time - start_checking_time) * 1000000))
async with db.atomic():
# TODO: fix spontaneous issue with unique constraint:
# peewee.IntegrityError: duplicate key value violates unique constraint
# "proxies_raw_protocol_auth_data_domain_port"
proxy, was_created = await db.get_or_create(
Proxy,
raw_protocol=raw_protocol,
auth_data=auth_data,
domain=domain,
port=port,
)
if was_created:
proxy.number_of_bad_checks = 0
if proxy.bad_proxy or proxy.uptime is None or proxy.uptime == 0:
proxy.uptime = int(time.time())
if proxy.bad_uptime is None or proxy.bad_uptime == 0 or \
proxy.number_of_bad_checks >= settings.DEAD_PROXY_THRESHOLD:
proxy.bad_uptime = int(time.time())
proxy.response_time = response_time
proxy.number_of_bad_checks = 0
proxy.last_check_time = int(time.time())
if additional_info is not None:
if additional_info.ipv4 is not None:
proxy.white_ipv4 = additional_info.ipv4
checking_time = int(end_checking_time - start_checking_time)
if checking_time > settings.PROXY_CHECKING_TIMEOUT:
checking_time = settings.PROXY_CHECKING_TIMEOUT
proxy.checking_period = settings.MIN_PROXY_CHECKING_PERIOD \
+ (checking_time / settings.PROXY_CHECKING_TIMEOUT) \
* (settings.MAX_PROXY_CHECKING_PERIOD - settings.MIN_PROXY_CHECKING_PERIOD)
# TODO: bad and not working proxies period
proxy.next_check_time = proxy.last_check_time + proxy.checking_period
await db.update(proxy)
def update_news():
try:
news_dicts = parse_news()
with db.atomic():
NewsModel.insert_many(news_dicts).execute()
return jsonify(news_dicts)
except Exception as e:
return generate_error(500, str(e))
def insert_users_into_database(users, chat_id):
users = [{'chat_id' : chat_id,
'user_id' : user.id,
'first_name': user.first_name,
'last_name' : user.last_name,
'is_member' : True} for user in users if user.first_name]
with db.atomic():
User.insert_many(users).on_conflict_replace().execute()
async def test_atomic_with_delete(flushdb):
for i in range(3):
await User.create(username=f'u{i}')
async with db.atomic():
user = await User.get(User.username == 'u1')
await user.delete_instance()
usernames = [u.username async for u in User.select()]
assert sorted(usernames) == ['u0', 'u2']
async with db.atomic():
async with db.atomic():
user = await User.get(User.username == 'u2')
await user.delete_instance()
usernames = [u.username async for u in User.select()]
assert usernames == ['u0']
def wrap(*args, **kwargs):
with db.atomic() as txn:
try:
res = func(*args, **kwargs)
except Exception:
logger.exception("execute %s error!", func.__name__)
txn.rollback()
return {"resultCode": -1, "resultMsg": "执行失败"}
return res
def worker(url):
try:
data = InputParser(url).get_data()
# insert data to db
with db.atomic():
Offers.insert_many(data, fields=FIELDS).execute()
with open('elapsed.log', 'a') as file:
file.write(f'{url}\n')
except Exception:
logger.exception(f'{url} failed to parse page')
def create_user(message: Message):
"""
Создаем нового юзера в базе данных
"""
tg_id = message.from_user.id
username = handle_username(message)
with db.atomic():
user = User.create(tg_id=tg_id, username=username)
return user
def add_purchase(self, buyer, drink_type, purchase_date, num_cans):
logger.debug("Creating purchase %s ", num_cans)
try:
with db.atomic():
return Purchase.create(buyer=buyer,
drink_type=drink_type,
purchase_date=purchase_date,
num_cans=num_cans)
except ieerror as ie:
logger.debug("Error: %s", ie.message)
return None
def insert_users_into_database(users, chat_id):
users = [{
'chat_id': chat_id,
'user_id': user.id,
'first_name': user.first_name,
'last_name': user.last_name,
'is_member': True
} for user in users if user.first_name]
with db.atomic():
User.insert_many(users).on_conflict_replace().execute()
def copy_db():
# Name_2, Petition_2, User
# db.drop_tables([Peticia], safe=True)
# db.create_tables([Peticia], safe=True)
# db.create_tables([Name], safe=True)
# query = Name_2.select()
# data = []
# for i in query:
# data.append((i.username, i.gender))
#
# with db.atomic():
# # by default SQLite limits the number of bound variables in a SQL query to 999
# for batch in chunked(data, 450):
# Name.insert_many(batch, fields=[Name.username, Name.gender]).execute()
#
#
#
# db.create_tables([Petition], safe=True)
# query = Petition_2.select()
# for i in query:
# Petition.create(
# petition_id=i.petition_id,
# status=i.status,
# title=i.title,
# article=i.article,
# answer=i.answer
# )
#
#
#
db.create_tables([Vote], safe=True)
for i in User.select(User.petition_id).distinct():
data = []
# p = Petition.get(petition_id=i.petition_id).petition_id
p = i.petition_id
query = User.select().where(User.petition == i.petition_id)
for user in query:
data.append((p, user.position_number, user.username,
user.sign_date, user.gender))
with db.atomic():
# by default SQLite limits the number of bound variables in a SQL query to 999
for batch in chunked(data, 198):
# User.insert_many(batch, fields=[User.petition, User.position_number,
# User.username, User.sign_date, User.gender]).execute()
Vote.insert_many(batch).execute()
def create_order(order_id, text, params, user, status):
with db.atomic():
order = Order.create(order_id=order_id,
tg_id=user.tg_id,
text=text,
side=params['side'],
pair=params['symbol'],
price=str(params['price']),
quantity=str(params['quantity']),
type=params['type'],
b_msg_id=user.temp['msg_ids'][1],
u_msg_id=user.temp['msg_ids'][0],
status=status)
return order
def signup():
form = SignupForm()
if form.validate_on_submit():
with db.atomic() as txn:
try:
User.create(
email=form.email.data,
password=generate_password_hash(form.password.data),
)
except IntegrityError:
flash('An account with that email already exists.')
return redirect(url_for('signup'))
flash('Account created.')
return redirect(url_for('login'))
else:
return render_template('signup.html', form=form)
def quote(quote_id):
try:
quote = Quote.get(
Quote.id == quote_id,
Quote.user == current_user.get_id(),
)
except Quote.DoesNotExist:
flash('Quote not found')
return redirect(url_for('quotes'))
quote_collections = [
collection.name
for collection in Collection.select().join(QuoteCollection).where(
QuoteCollection.quote == quote, )
]
quote.collections = quote_collections
form = QuoteEditForm(obj=quote)
form.collections.choices = [(collection.name, collection.name)
for collection in Collection.select().where(
Collection.user == current_user.get_id())]
if form.validate_on_submit():
if form.id.data != quote_id:
flash('Quote ID mismatch!')
return redirect(url_for('quotes'))
if form.form_delete.data:
quote.delete_instance(recursive=True)
flash('Quote deleted.')
return redirect(url_for('quotes'))
quote.content = form.content.data
quote.author = form.author.data
quote.save()
if set(quote_collections) != set(form.collections.data):
flash('Collections updated.')
QuoteCollection.delete().where(
QuoteCollection.quote == quote, ).execute()
with db.atomic() as txn:
for collection_name in form.collections.data:
QuoteCollection.create(
quote=quote,
collection=Collection.get(
Collection.name == collection_name, ),
)
flash('Quote updated.')
return redirect(url_for('quotes'))
else:
return render_template('quote.html', form=form)
def upload_all_groups(self, ):
"""
upload AffinityPropagation model for to define a users' group and updating all users
:return None:
"""
dataset = []
users = []
count = Users.select().count()
for i in range(0, count, 100):
usrs = Users.select().offset(i).limit(100).execute()
for u in usrs:
users.append(model_to_dict(u))
for u in users:
dataset.append(self.user_preparation(u))
from sklearn.cluster import AffinityPropagation
clustering = AffinityPropagation(affinity='euclidean',
convergence_iter=15,
copy=True,
damping=0.5,
max_iter=200,
preference=None,
verbose=False).fit(dataset)
all_users = Users.select().execute()
for u in all_users:
u.group = self.get_group(u)
with db.atomic() as txn:
try:
Users.bulk_update(all_users, fields=[Users.group])
except peewee.Error as e:
logger.error(str(e))
db.rollback()
@test
def unsave():
txn.rollback()
@not_test
def save():
import pickle
pickle.dump(clustering, open(PATH_TO_DIR + "clustering", "wb"))
def create_categories(data):
category_names = []
for product in data:
category_names.append(product['category'])
category_names = set(category_names)
categories = []
for category_name in category_names:
categories.append({'name': category_name})
with db.atomic():
for batch in chunked(categories, 100):
Category.insert_many(batch).execute()
result = {}
for cat in Category.select():
result[cat.name] = cat.id
return result
def buy(user_uid, goods_list):
"""
goods_list = [{"id": 111, "amount": 1}]
还应该判断货币是否足够，并扣除货币
"""
if not isinstance(goods_list, (list, tuple)):
raise ParamError(
"goods_list should be type of list, but {} found".format(
type(goods_list)))
order = None
with db.atomic() as transaction:
try:
user = User.get_with_uid(user_uid)
if not user:
raise DataNotFound(
"user not found for uid={}".format(user_uid))
order = Order.create_data(user=user_uid)
for goods in goods_list:
goods_id, amount = goods["id"], goods["amount"]
goods = Goods.check_amount(goods_id=goods_id, amount=amount)
if not goods:
raise GoodsNotEnough()
if user.money < goods.price:
raise MoneyNotEnough()
OrderItem.create_data(order=order.uid,
goods=goods.uid,
price=goods.price,
amount=amount)
user.money -= goods.price
goods.amount -= amount
user.save()
goods.save()
except GoodsNotEnough:
transaction.rollback()
order = None
except MoneyNotEnough:
transaction.rollback()
order = None
except Exception as e:
print(e)
transaction.rollback()
order = None
return order
def create_team_records():
file = open(PATH_TO_TEAM_CODES, 'r')
teams = []
while True:
try:
name = sanitize_string(file.readline())
code = sanitize_string(file.readline())
if code == '':
break
teams.append({
'name': name,
'code': code
})
except Exception:
break
file.close()
print(len(teams))
with db.atomic():
Team.insert_many(teams).execute()
def set_gender():
query = Vote.select().where(Vote.gender.is_null())
query_genders = get_genders()
data = []
for i in query:
try:
name = " ".join(i.username.split()).split(' ')[1]
except IndexError:
name = " ".join(i.username.split()).split(' ')[0]
gender = query_genders.get(name, None)
if gender is not None:
if gender:
i.gender = True
data.append(i)
elif not gender:
i.gender = False
data.append(i)
with db.atomic():
Vote.bulk_update(data, fields=['gender'], batch_size=165)
def post_team(room_code):
sessid = request.cookies.get("sessid")
you = Player.select(Player, Room).join(Room).where(Player.sessid == sessid, Room.code == room_code).get()
room = you.room
if you.queue_position != room.leader_queue:
return HTTPError(400, "You are not the leader")
team_ids = request.params.getall("team[]")
team = room.players.where(Player.id << team_ids).execute()
mission = Mission.create(room=room, leader=you, team_size=0)
with db.atomic():
for plr in team:
PlayerMission.insert(player=plr, mission=mission).execute()
room.phase = "vote"
room.save()
message_data = {
"mission_id": mission.id,
"leader": {"id": you.id, "name": you.name},
"team": [{"id": plr.id, "name": plr.name} for plr in team],
}
post_message([Msg("team", message_data)], room=room)
def collections():
form = CollectionAddForm()
if form.validate_on_submit():
with db.atomic() as txn:
try:
Collection.create(
name=form.name.data,
user=current_user.get_id(),
)
except IntegrityError:
flash('A collection with that name already exists.')
return redirect(url_for('collections'))
else:
collections = (Collection.select(
Collection,
fn.COUNT(QuoteCollection.id).alias('quote_count'),
).join(QuoteCollection, JOIN.LEFT_OUTER).group_by(Collection).where(
Collection.user == current_user.get_id(), ))
return render_template(
'collections.html',
form=form,
collections=collections,
)
def cadastro():
erros = campos_presentes_na_requisicao('nome email senha tipo_usuario')
if erros:
return render_template('usuario/cadastro.html', erros=erros), 400
nome = request.form.get('nome')
email = request.form.get('email')
senha = request.form.get('senha')
tipo_usuario = int(request.form.get('tipo_usuario'))
# Cadastrar usuário
with db.atomic() as transacao:
novo_usuario = Usuario.create(nome=nome,
email=email,
tipo_usuario=tipo_usuario,
hash_senha=senha_hasheada(senha))
# Criar professor ou aluno, dependendo do tipo_usuario
try:
resposta, codigo = cadastrar[TipoUsuario(tipo_usuario)](
novo_usuario)
if codigo != 200:
db.rollback()
flash(resposta)
session['usuario'] = model_to_dict(novo_usuario)
return redirect(url_for('pagina_inicial'))
except Exception as e:
db.rollback()
print(e)
flash(
'Algo de errado aconteceu ao cadastrar o usuário. Por favor, tente mais tarde.'
)
return render_template('usuario/cadastro.html'), 500
def get(self, request):
r = {}
for k in request.args:
r[k] = request.args[k][0]
assert "user_id" in r
assert "type" in r
if r["type"] == "get_predict":
if "predict_size" in r:
count = int(r["predict_size"])
else:
count = 1
user = Users.get(Users.user_id == r["user_id"])
predict = user.predict_films
if len(predict) < count:
predict += get_predict(user, 20)
with db.atomic() as txn:
user.predict_films = predict[count:]
user.save()
@test
def not_save():
txn.rollback()
return response.json(predict[:count])
elif r["type"] == "get_liked":
user = Users.get(Users.user_id == r["user_id"])
return response.json(user.liked)
else:
response.text("error")
def post_message(messages, players=None, room=None):
"""
Posts the message to specified players or the entire room (exactly one must
be specified).
Message is sent to both the message queue and the database.
@param messages: the list of messages to post
@param players: the list of targeted players
@param room: the targeted room
"""
if (((players is None) and (room is None)) or
((players is not None and room is not None))):
raise ValueError('Specify exactly one of room or players')
if room is not None:
players = room.players.execute()
with db.atomic():
for player in players:
for msg in messages:
models.Message.insert(
code=msg.code, data=json.dumps(msg.data), player=player.id
).execute()
for q in message_queue[player.id]:
q.put(msg)
for q in message_queue[player.id]:
q.put(StopIteration)
next_station = train["LocationCode"]
dest_station = train["DestinationCode"]
line_code = train["Line"]
if dest_station is None or next_station is None or line_code == "No" or train["Car"] is None:
continue
cars = int(train["Car"]) if train["Car"] != "-" else 6
cars = 8 if cars == 2 else cars
if train["Min"].isnumeric():
time = int(train["Min"])
else:
if "-" in train["Min"]:
continue
time = 1 if train["Min"] == "ARR" else 0
ts = datetime.datetime.fromtimestamp(int(float(filename.split("/")[-1])))
ds = dict(cars=cars, line_code=line_code, time=time, next_station=next_station, dest_station=dest_station, timestamp=ts, trackgroup=int(train["Group"]))
datasource.append(ds)
print("set... ({})".format(len(datasource)))
print("go! ")
with db.atomic():
HistoricalTrainPosition.insert_many(datasource).execute()
#HistoricalTrainPosition.create(cars=cars, line_code=line_code, time=time,
# next_station=next_station, dest_station=dest_station,
# timestamp=ts, trackgroup=int(train["Group"]))
import time
from random import choice
from models import Journal, db
LEVEL_CHOICE = [10, 20, 30, 40, 50]
objs = list(Journal.select())
count = len(objs)
start = time.time()
with db.atomic():
for obj in objs:
obj.level = choice(LEVEL_CHOICE)
obj.save(only=['level'])
now = time.time()
print(f'peewee, J: Rows/sec: {count / (now - start): 10.2f}')
def db_save(model):
"""
Атомарная транзакция сохранения в базе данных
"""
with db.atomic():
model.save()
