def clear_finished_jobs(self):
job_history_limit_each_spider = 100
with session_scope() as session:
spiders = list(session.query(distinct(HistoricalJob.spider_id)))
for row in spiders:
spider_id = row[0]
with session_scope() as session:
over_limitation_jobs = list(session.query(HistoricalJob)\
.filter(HistoricalJob.spider_id==spider_id)\
.order_by(desc(HistoricalJob.complete_time))\
.slice(job_history_limit_each_spider, 1000)\
.all())
for over_limitation_job in over_limitation_jobs:
self._remove_histical_job(over_limitation_job)
def _check_and_give_xsi_reward(
self, ctx) -> Optional[Awaitable[discord.Message]]:
"""성공시, 사용자에게 반환할 디스코드 메시지를 반환합니다."""
with session_scope() as session:
xsi_reward = get_xsi_reward(session, ctx.author.id)
if xsi_reward is None:
return None
sticker_before, sticker_after = add_sticker(
session, ctx.author.id, xsi_reward.sticker)
xsi_reward.is_received = True
session.commit()
self.bot.logger.info(
f"xsi_reward {xsi_reward} - user sticker {sticker_before} -> {sticker_after}"
)
return ctx.send(
f"{ctx.author.mention}, **크시봇 3주년 기념 이벤트** 보상을 확인해주세요!",
embed=discord.Embed(
title=f"{xsi_reward.sticker}장 추가 획득!",
description=
f"스티커 `{sticker_before}장` ➡️ `{sticker_after}장`",
).set_footer(
text="1/13~1/15 동안 크시 호감도를 50 올릴 때마다 스티커 1장이 지급됩니다. (최대 5장)"
),
)
def create_adress(adress: dict, salon_id: str) -> str:
"""
Create an adress for a salon
:param adress: Data for the adress
:param salon_id: ID of the salon that this adress is created for
:return: UUID of the created adress object
"""
try:
validate_adress(adress)
except ValueError as e:
raise e
adress_id = uuid.uuid4().hex
new_adress = Adresses(
id=adress_id,
salon_id=salon_id,
city=adress["city"],
zip_code=adress["zip_code"],
street=adress["street"],
building_no=adress["building_no"],
number_of_seats=adress["number_of_seats"],
created_at=datetime.datetime.utcnow()
)
try:
with session_scope() as session:
session.add(new_adress)
except IntegrityError as e:
raise e
return adress_id
def get(self, project, spider):
with session_scope() as session:
project = session.query(Project).filter_by(name=project).first()
spider = session.query(Spider).filter_by(project_id=project.id,
name=spider).first()
job_settings = {
setting.setting_key: setting.value
for setting in session.query(SpiderSettings).filter_by(
spider_id=spider.id)
}
# default setting values
if 'concurrency' not in job_settings:
job_settings['concurrency'] = 1
if 'timeout' not in job_settings:
job_settings['timeout'] = 3600
template = get_template_loader().load('spidersettings.html')
context = {}
context['settings'] = job_settings
context['project'] = project
context['spider'] = spider
spider_parameters = session.query(SpiderParameter)\
.filter_by(spider_id = spider.id)\
.order_by(SpiderParameter.parameter_key)
context['spider_parameters'] = spider_parameters
return self.write(template.generate(**context))
def post(self):
project_name = self.get_argument('project')
with session_scope() as session:
project = session.query(Project).filter_by(
name=project_name).first()
spiders = session.query(Spider).filter_by(project_id=project.id)
for spider in spiders:
triggers = session.query(Trigger).filter_by(
spider_id=spider.id)
session.query(SpiderExecutionQueue).filter_by(
spider_id=spider.id).delete()
session.commit()
for trigger in triggers:
self.scheduler_manager.remove_schedule(
project_name, spider.name, trigger_id=trigger.id)
for history_log in session.query(HistoricalJob).filter(
HistoricalJob.spider_id == spider.id):
try:
os.remove(history_log.log_file)
except:
pass
try:
os.remove(history_log.items_file)
except:
pass
session.delete(history_log)
session.delete(spider)
session.delete(project)
ProjectWorkspace(project_name).delete_egg(project_name)
def update_visit(visit_data: dict):
"""
Updates a visit based on provided data.
:param visit_data: Dict with the updated data of the visit
:return:
"""
try:
date_start = datetime.datetime.strptime(visit_data["visit_date_start"],
"%d/%m/%Y %H:%M")
date_end = datetime.datetime.strptime(visit_data["visit_date_end"],
"%d/%m/%Y %H:%M")
except ValueError as e:
raise ValueError("The date format is wrong: " + str(e))
if not date_available(date_start, date_end, visit_data["hairdresser_id"],
visit_data["id"]):
return {"success": False}
try:
if update_services(visit_data["id"], visit_data["services"]):
with session_scope() as session:
visit = (session.query(Visits).filter(
Visits.id == visit_data["id"]).first())
visit.date_start = date_start
visit.date_end = date_end
return {"success": True}
return {"success": False}
except Exception as e:
return {"success": False, "error": str(e)}
async def on_raw_reaction_add(self, payload: discord.RawReactionActionEvent):
if payload.user_id == self.bot.user.id:
return
if payload.guild_id != self.bot.guild.id:
return
with session_scope() as session:
attacher = await self.bot.guild.fetch_member(payload.user_id)
if is_admin(attacher):
channel = await self.bot.fetch_channel(payload.channel_id)
recipient = (await channel.fetch_message(payload.message_id)).author
await self.apply_rules(
await self.get_rules_by_admin_reaction(
session, recipient.id, payload.channel_id, str(payload.emoji)
),
get_or_create_user(session, recipient.id),
recipient,
)
await self.apply_rules(
await self.get_rules_by_self_reaction(
session, attacher.id, payload.message_id, str(payload.emoji)
),
get_or_create_user(session, attacher.id),
attacher,
)
def __init__(self, domain, client_id, client_secret):
"""
:auth: an auth dict with client_id, client_secret and domain
"""
self.token = None
self.auth = {'client_id': client_id, 'client_secret': client_secret, 'domain': domain}
# Azure API requires 'a few minutes' to be waited in case of throttle exception.
# Value in seconds
self.request_throttled_interval = 300
# Amount of retries completed
self.throttle_retries = 0
self.graph_url = CONFIG.get('azure', 'GRAPH_ENDPOINT')%self.auth['domain']
self.headers = {}
# List of all groups fetched from Azure
self.azure_group_list = {}
# List of all licenses fetched from Azure
self.azure_license_list = {}
# List of all users which need to be disabled in Azure once the CSV file has been parsed through
self.azure_user_list = []
with session_scope() as session:
for user in session.query(User).filter(User.is_active==True):
self.azure_user_list.append(user.immutable_id)
def login(email: str, raw_password: str) -> dict:
"""
Log ins the user based on the username and password they provide
:param email: email of the user
:param raw_password:direct password inputted by the user into the form
:return: Session_id and the corresponding account_id or (None, None) if failed
"""
payload = dict()
payload["session_id"] = None
payload["account_id"] = None
payload["correct_pass"] = False
payload["email_exists"] = True
if not user_with_given_email_exists(email):
payload["email_exists"] = False
return payload
with session_scope() as session:
user = session.query(Accounts).filter(Accounts.email == email).first()
encrypted_from_db = user.hashed_password
account_id = user.id
if sha256_crypt.verify(raw_password, encrypted_from_db):
payload["session_id"] = create_session_for_user(account_id)
payload["account_id"] = account_id
payload["correct_pass"] = True
return payload
else:
return payload
def post(self):
project = self.get_argument('project')
spider = self.get_argument('spider')
cron = self.get_argument('cron')
with session_scope() as session:
project = session.query(Project).filter(
Project.name == project).first()
spider = session.query(Spider).filter(
Spider.project_id == project.id,
Spider.name == spider).first()
try:
self.scheduler_manager.add_schedule(project, spider, cron)
response_data = {
'status': 'ok',
}
self.write(json.dumps(response_data))
except SpiderNotFound:
response_data = {
'status': 'error',
'errormsg': 'spider not found',
}
self.write(json.dumps(response_data))
except ProjectNotFound:
response_data = {
'status': 'error',
'errormsg': 'project not found',
}
self.write(json.dumps(response_data))
except InvalidCronExpression:
response_data = {
'status': 'error',
'errormsg': 'invalid cron expression.',
}
self.write(json.dumps(response_data))
def _clear_running_jobs(self):
with session_scope() as session:
jobs = list(
session.query(SpiderExecutionQueue).filter(
SpiderExecutionQueue.status.in_([0, 1])))
for job in jobs:
self._remove_histical_job(job)
def update_services(visit_id: str, services: list) -> bool:
"""
Adds and or removes services for a given visit based on a provided list
of services
:param visit_id:
:param services:
:return:
"""
try:
with session_scope() as session:
old_services = session.query(VisitsServices.service_id).filter(
VisitsServices.visit_id == visit_id).all()
old_services = [x[0] for x in old_services]
for _id in old_services:
if _id not in services:
deletion = session.query(VisitsServices).filter(
VisitsServices.visit_id == visit_id).filter(
VisitsServices.service_id == _id)
deletion.delete()
session.commit()
for _id in services:
if _id not in old_services:
session.add(
VisitsServices(service_id=_id, visit_id=visit_id))
except Exception as e:
print(e)
return False
return True
def riaParser(url, loops=1, timeout=2):
while loops != 0:
ria = requests.get(url, headers={'User-Agent': UserAgent().chrome})
processes = []
if ria.status_code == 200:
result = []
ria_soup = bs(ria.content, features="html.parser")
# Получаем следующую ссылку
try:
url = "https://ria.ru" + ria_soup.find(
class_="list-items-loaded").get("data-next-url")
except AttributeError:
url = "https://ria.ru" + ria_soup.find(
class_="list-more").get("data-url")
news_block = ria_soup.findAll(class_="list-item")
result = news_scraper(news_block)
with session_scope() as session:
for first in result:
if first['Text'] is not None:
instance = RiaNews(title=first['Title'],
date=first['Date'],
url=first['URL'],
url_next=url,
tags=first['Tags'],
text=first['Text'])
session.add(instance)
loops = loops - 1
return
def _remove_histical_job(self, job):
'''
@type job: HistoricalJob
'''
with session_scope() as session:
job = session.query(HistoricalJob).filter(
HistoricalJob.id == job.id).first()
if job.items_file:
try:
os.remove(job.items_file)
except Exception as e:
logger.warning(e.message)
if job.log_file:
try:
os.remove(job.log_file)
except Exception as e:
logger.warning(e.message)
original_log_file = os.path.join('logs', job.project_name,
job.spider_name,
'%s.log' % job.id)
if os.path.exists(original_log_file):
os.remove(original_log_file)
original_items_file = os.path.join('items', job.project_name,
job.spider_name,
'%s.jl' % job.id)
if os.path.exists(original_items_file):
os.remove(original_items_file)
session.delete(job)
def create_salon(salon_data: dict) -> dict:
"""
Create new salon based on provided data
:param salon_data: JSON with salon data
:return: Dict with info about the success or failure of the creation
"""
try:
validate_salon(salon_data)
except ValueError as e:
return {"success": False, "error": str(e)}
_id = uuid.uuid4().hex
try:
adress_id = create_adress(salon_data['adress'], _id)
except Exception as e:
return {"success": False, "error": str(e)}
new_service = Salons(id=_id,
adress_id=adress_id,
opening_hour=salon_data["opening_hour"],
closing_hour=salon_data["closing_hour"],
created_at=datetime.datetime.utcnow())
try:
with session_scope() as session:
session.add(new_service)
return {"success": True}
except IntegrityError as e:
return {"success": False, "error": str(e)}
def get_azure_id(self, immutable_id):
"""
Return Azure User ID, None is returned if a user with given ID is not found.
"""
with session_scope() as session:
azure_id = session.query(User.azure_user_id).filter_by(immutable_id=immutable_id).first()
return azure_id
def faucet():
update_counters()
with session_scope() as session:
bver = str(get_latest_version(session))
message = ''
if request.method == 'POST':
try:
acct = request.form.get('acct')
app.logger.info('acct: {}'.format(acct))
amount = float(request.form.get('amount'))
app.logger.info('amount: {}'.format(amount))
if amount < 0:
message = 'Amount must be >= 0'
elif not is_valid_account(acct):
message = 'Invalid account format'
else:
response = requests.get(config['FAUCET_HOST'],
params={
'address': acct,
'amount':
format(amount * 1e6, '.0f')
})
if response.status_code == 200:
message = 'Sent {0} <small>Libra</small> to <a href="/account/{1}">{1}</a>'.format(
amount, acct)
except:
message = 'Invalid request logged!'
app.logger.exception(message)
if message:
message = faucet_alert_template.format(message)
return faucet_template.format(bver, message)
async def add_rule_by_self(
self,
ctx,
name: str,
message: discord.Message,
emoji: str,
reward_sticker: Optional[int],
reward_role: Optional[discord.Role],
):
with session_scope() as session:
rule = Rule(
name=name, by_admin=False, message_id=message.id, emoji=str(emoji)
)
if reward_sticker is not None:
rule.reward_sticker = reward_sticker
if reward_role is not None:
rule.reward_role_id = reward_role.id
session.add(rule)
session.commit()
self.bot.logger.info("rule add: " + repr(rule))
await message.add_reaction(emoji)
await ctx.send(f"규칙 #{rule.id} 추가됨!")
def can_access_admin(session_id, account_id):
if session_is_valid(session_id, account_id):
with session_scope() as session:
if session.query(Accounts.is_admin).filter(
Accounts.id == account_id).first()[0]:
return True
return False
def all_objects_from_db(object_table):
"""
Gets all user objects from the database
:return: a user object
"""
with session_scope() as session:
for _object in session.query(object_table).all():
yield _object
def get(self, project, spider, jobid):
with session_scope() as session:
job = session.query(HistoricalJob).filter_by(id=jobid).first()
log_file = job.log_file
with open(log_file, 'r') as f:
log = f.read()
loader = get_template_loader()
self.write(loader.load("log.html").generate(log=log))
def is_customer(account_id: str) -> bool:
with session_scope() as session:
user = session.query(Accounts).filter(
Accounts.id == account_id).first()
if user.account_type == "customer":
return True
else:
return False
def user_exists(self, immutable_id):
with session_scope() as session:
user = session.query(User).filter_by(immutable_id=immutable_id).first()
if user:
return True
else:
return False
def get_next_job(self):
with session_scope() as session:
next_jobs = list(
session.query(WebhookJob).filter(
WebhookJob.status == 0).order_by(WebhookJob.id).slice(
0, 1))
if next_jobs:
return next_jobs[0]
async def list_rules(self, ctx):
with session_scope() as session:
content = "\n".join(repr(rule) for rule in list_all_rules(session))
if not content:
await ctx.send("규칙 없음")
return
await ctx.send(content, allowed_mentions=discord.AllowedMentions.none())
async def show_sticker(self, ctx):
with session_scope() as session:
user = get_user(session, ctx.author.id)
if user is None:
await ctx.send("아직 스티커를 받지 않았습니다.")
return
await ctx.send(f"스티커를 `{user.sticker}`장 갖고 있습니다.")
def poll_task_queue(self):
if self.task_queue.empty():
with session_scope() as session:
tasks_to_run = session.query(SpiderExecutionQueue).filter_by(
status=0).order_by(SpiderExecutionQueue.update_time).slice(
0, 10)
for task in tasks_to_run:
self.task_queue.put(task)
def __init__(self):
with session_scope() as session:
# reset all running jobs to PENDING status on start
for job in session.query(WebhookJob).filter(
WebhookJob.status == 1):
job.status = 0
session.add(job)
session.commit()
async def reward_test(self, ctx, formlink: str):
with session_scope() as session:
user = search_by_formlink(session, formlink)
if user is None:
await ctx.send(f"{ctx.author.mention}, 해당하는 사용자를 찾지 못했습니다.")
return
await ctx.send(f"{ctx.author.mention}, {user}")
async def load_quiz(self, ctx):
io = BytesIO()
await ctx.message.attachments[0].save(io)
with session_scope() as session:
instances = [Quiz(**instance) for instance in json.load(io)]
session.add_all(instances)
await ctx.send(f"`{len(instances)}`개 데이터를 추가했습니다.")
def add_user_to_db(self, immutable_id, user_id, is_active):
user = User()
user.immutable_id = immutable_id
user.azure_user_id = user_id
user.is_active = is_active
user.created = datetime.datetime.now()
user.synced = datetime.datetime.now()
with session_scope() as session:
session.add(user)
def delete_users(self):
"""
Delete all the users that were not found on the CSV file but were still present in the database.
"""
for user in self.azure_user_list:
with session_scope() as session:
user = session.query(User).filter_by(immutable_id=user).first()
self.remove_user_from_db(user.immutable_id)
self.delete_user(user.azure_user_id)
def update_user_in_db(self, immutable_id):
"""
Updates the synced time with the current time
"""
with session_scope() as session:
user = session.query(User).filter_by(immutable_id=immutable_id).first()
if user:
user.synced = datetime.datetime.now()
session.add(user)
def user_exists(self, immutable_id):
"""
Returns True if user exists in the database
"""
with session_scope() as session:
user_filter = exists().where(User.immutable_id==immutable_id)
for user in session.query(User.immutable_id).filter(user_filter):
if user:
return True
else:
return False
def is_active(self, immutable_id):
with session_scope() as session:
for active in session.query(User.is_active).filter(User.immutable_id==immutable_id):
return active
def remove_user_from_db(self, immutable_id):
with session_scope() as session:
user = session.query(User).filter_by(immutable_id=immutable_id).first()
session.delete(user)
