def clear_finished_jobs(self): job_history_limit_each_spider = 100 with session_scope() as session: spiders = list(session.query(distinct(HistoricalJob.spider_id))) for row in spiders: spider_id = row[0] with session_scope() as session: over_limitation_jobs = list(session.query(HistoricalJob)\ .filter(HistoricalJob.spider_id==spider_id)\ .order_by(desc(HistoricalJob.complete_time))\ .slice(job_history_limit_each_spider, 1000)\ .all()) for over_limitation_job in over_limitation_jobs: self._remove_histical_job(over_limitation_job)
def _check_and_give_xsi_reward( self, ctx) -> Optional[Awaitable[discord.Message]]: """성공시, 사용자에게 반환할 디스코드 메시지를 반환합니다.""" with session_scope() as session: xsi_reward = get_xsi_reward(session, ctx.author.id) if xsi_reward is None: return None sticker_before, sticker_after = add_sticker( session, ctx.author.id, xsi_reward.sticker) xsi_reward.is_received = True session.commit() self.bot.logger.info( f"xsi_reward {xsi_reward} - user sticker {sticker_before} -> {sticker_after}" ) return ctx.send( f"{ctx.author.mention}, **크시봇 3주년 기념 이벤트** 보상을 확인해주세요!", embed=discord.Embed( title=f"{xsi_reward.sticker}장 추가 획득!", description= f"스티커 `{sticker_before}장` ➡️ `{sticker_after}장`", ).set_footer( text="1/13~1/15 동안 크시 호감도를 50 올릴 때마다 스티커 1장이 지급됩니다. (최대 5장)" ), )
def create_adress(adress: dict, salon_id: str) -> str: """ Create an adress for a salon :param adress: Data for the adress :param salon_id: ID of the salon that this adress is created for :return: UUID of the created adress object """ try: validate_adress(adress) except ValueError as e: raise e adress_id = uuid.uuid4().hex new_adress = Adresses( id=adress_id, salon_id=salon_id, city=adress["city"], zip_code=adress["zip_code"], street=adress["street"], building_no=adress["building_no"], number_of_seats=adress["number_of_seats"], created_at=datetime.datetime.utcnow() ) try: with session_scope() as session: session.add(new_adress) except IntegrityError as e: raise e return adress_id
def get(self, project, spider): with session_scope() as session: project = session.query(Project).filter_by(name=project).first() spider = session.query(Spider).filter_by(project_id=project.id, name=spider).first() job_settings = { setting.setting_key: setting.value for setting in session.query(SpiderSettings).filter_by( spider_id=spider.id) } # default setting values if 'concurrency' not in job_settings: job_settings['concurrency'] = 1 if 'timeout' not in job_settings: job_settings['timeout'] = 3600 template = get_template_loader().load('spidersettings.html') context = {} context['settings'] = job_settings context['project'] = project context['spider'] = spider spider_parameters = session.query(SpiderParameter)\ .filter_by(spider_id = spider.id)\ .order_by(SpiderParameter.parameter_key) context['spider_parameters'] = spider_parameters return self.write(template.generate(**context))
def post(self): project_name = self.get_argument('project') with session_scope() as session: project = session.query(Project).filter_by( name=project_name).first() spiders = session.query(Spider).filter_by(project_id=project.id) for spider in spiders: triggers = session.query(Trigger).filter_by( spider_id=spider.id) session.query(SpiderExecutionQueue).filter_by( spider_id=spider.id).delete() session.commit() for trigger in triggers: self.scheduler_manager.remove_schedule( project_name, spider.name, trigger_id=trigger.id) for history_log in session.query(HistoricalJob).filter( HistoricalJob.spider_id == spider.id): try: os.remove(history_log.log_file) except: pass try: os.remove(history_log.items_file) except: pass session.delete(history_log) session.delete(spider) session.delete(project) ProjectWorkspace(project_name).delete_egg(project_name)
def update_visit(visit_data: dict): """ Updates a visit based on provided data. :param visit_data: Dict with the updated data of the visit :return: """ try: date_start = datetime.datetime.strptime(visit_data["visit_date_start"], "%d/%m/%Y %H:%M") date_end = datetime.datetime.strptime(visit_data["visit_date_end"], "%d/%m/%Y %H:%M") except ValueError as e: raise ValueError("The date format is wrong: " + str(e)) if not date_available(date_start, date_end, visit_data["hairdresser_id"], visit_data["id"]): return {"success": False} try: if update_services(visit_data["id"], visit_data["services"]): with session_scope() as session: visit = (session.query(Visits).filter( Visits.id == visit_data["id"]).first()) visit.date_start = date_start visit.date_end = date_end return {"success": True} return {"success": False} except Exception as e: return {"success": False, "error": str(e)}
async def on_raw_reaction_add(self, payload: discord.RawReactionActionEvent): if payload.user_id == self.bot.user.id: return if payload.guild_id != self.bot.guild.id: return with session_scope() as session: attacher = await self.bot.guild.fetch_member(payload.user_id) if is_admin(attacher): channel = await self.bot.fetch_channel(payload.channel_id) recipient = (await channel.fetch_message(payload.message_id)).author await self.apply_rules( await self.get_rules_by_admin_reaction( session, recipient.id, payload.channel_id, str(payload.emoji) ), get_or_create_user(session, recipient.id), recipient, ) await self.apply_rules( await self.get_rules_by_self_reaction( session, attacher.id, payload.message_id, str(payload.emoji) ), get_or_create_user(session, attacher.id), attacher, )
def __init__(self, domain, client_id, client_secret): """ :auth: an auth dict with client_id, client_secret and domain """ self.token = None self.auth = {'client_id': client_id, 'client_secret': client_secret, 'domain': domain} # Azure API requires 'a few minutes' to be waited in case of throttle exception. # Value in seconds self.request_throttled_interval = 300 # Amount of retries completed self.throttle_retries = 0 self.graph_url = CONFIG.get('azure', 'GRAPH_ENDPOINT')%self.auth['domain'] self.headers = {} # List of all groups fetched from Azure self.azure_group_list = {} # List of all licenses fetched from Azure self.azure_license_list = {} # List of all users which need to be disabled in Azure once the CSV file has been parsed through self.azure_user_list = [] with session_scope() as session: for user in session.query(User).filter(User.is_active==True): self.azure_user_list.append(user.immutable_id)
def login(email: str, raw_password: str) -> dict: """ Log ins the user based on the username and password they provide :param email: email of the user :param raw_password:direct password inputted by the user into the form :return: Session_id and the corresponding account_id or (None, None) if failed """ payload = dict() payload["session_id"] = None payload["account_id"] = None payload["correct_pass"] = False payload["email_exists"] = True if not user_with_given_email_exists(email): payload["email_exists"] = False return payload with session_scope() as session: user = session.query(Accounts).filter(Accounts.email == email).first() encrypted_from_db = user.hashed_password account_id = user.id if sha256_crypt.verify(raw_password, encrypted_from_db): payload["session_id"] = create_session_for_user(account_id) payload["account_id"] = account_id payload["correct_pass"] = True return payload else: return payload
def post(self): project = self.get_argument('project') spider = self.get_argument('spider') cron = self.get_argument('cron') with session_scope() as session: project = session.query(Project).filter( Project.name == project).first() spider = session.query(Spider).filter( Spider.project_id == project.id, Spider.name == spider).first() try: self.scheduler_manager.add_schedule(project, spider, cron) response_data = { 'status': 'ok', } self.write(json.dumps(response_data)) except SpiderNotFound: response_data = { 'status': 'error', 'errormsg': 'spider not found', } self.write(json.dumps(response_data)) except ProjectNotFound: response_data = { 'status': 'error', 'errormsg': 'project not found', } self.write(json.dumps(response_data)) except InvalidCronExpression: response_data = { 'status': 'error', 'errormsg': 'invalid cron expression.', } self.write(json.dumps(response_data))
def _clear_running_jobs(self): with session_scope() as session: jobs = list( session.query(SpiderExecutionQueue).filter( SpiderExecutionQueue.status.in_([0, 1]))) for job in jobs: self._remove_histical_job(job)
def update_services(visit_id: str, services: list) -> bool: """ Adds and or removes services for a given visit based on a provided list of services :param visit_id: :param services: :return: """ try: with session_scope() as session: old_services = session.query(VisitsServices.service_id).filter( VisitsServices.visit_id == visit_id).all() old_services = [x[0] for x in old_services] for _id in old_services: if _id not in services: deletion = session.query(VisitsServices).filter( VisitsServices.visit_id == visit_id).filter( VisitsServices.service_id == _id) deletion.delete() session.commit() for _id in services: if _id not in old_services: session.add( VisitsServices(service_id=_id, visit_id=visit_id)) except Exception as e: print(e) return False return True
def riaParser(url, loops=1, timeout=2): while loops != 0: ria = requests.get(url, headers={'User-Agent': UserAgent().chrome}) processes = [] if ria.status_code == 200: result = [] ria_soup = bs(ria.content, features="html.parser") # Получаем следующую ссылку try: url = "https://ria.ru" + ria_soup.find( class_="list-items-loaded").get("data-next-url") except AttributeError: url = "https://ria.ru" + ria_soup.find( class_="list-more").get("data-url") news_block = ria_soup.findAll(class_="list-item") result = news_scraper(news_block) with session_scope() as session: for first in result: if first['Text'] is not None: instance = RiaNews(title=first['Title'], date=first['Date'], url=first['URL'], url_next=url, tags=first['Tags'], text=first['Text']) session.add(instance) loops = loops - 1 return
def _remove_histical_job(self, job): ''' @type job: HistoricalJob ''' with session_scope() as session: job = session.query(HistoricalJob).filter( HistoricalJob.id == job.id).first() if job.items_file: try: os.remove(job.items_file) except Exception as e: logger.warning(e.message) if job.log_file: try: os.remove(job.log_file) except Exception as e: logger.warning(e.message) original_log_file = os.path.join('logs', job.project_name, job.spider_name, '%s.log' % job.id) if os.path.exists(original_log_file): os.remove(original_log_file) original_items_file = os.path.join('items', job.project_name, job.spider_name, '%s.jl' % job.id) if os.path.exists(original_items_file): os.remove(original_items_file) session.delete(job)
def create_salon(salon_data: dict) -> dict: """ Create new salon based on provided data :param salon_data: JSON with salon data :return: Dict with info about the success or failure of the creation """ try: validate_salon(salon_data) except ValueError as e: return {"success": False, "error": str(e)} _id = uuid.uuid4().hex try: adress_id = create_adress(salon_data['adress'], _id) except Exception as e: return {"success": False, "error": str(e)} new_service = Salons(id=_id, adress_id=adress_id, opening_hour=salon_data["opening_hour"], closing_hour=salon_data["closing_hour"], created_at=datetime.datetime.utcnow()) try: with session_scope() as session: session.add(new_service) return {"success": True} except IntegrityError as e: return {"success": False, "error": str(e)}
def get_azure_id(self, immutable_id): """ Return Azure User ID, None is returned if a user with given ID is not found. """ with session_scope() as session: azure_id = session.query(User.azure_user_id).filter_by(immutable_id=immutable_id).first() return azure_id
def faucet(): update_counters() with session_scope() as session: bver = str(get_latest_version(session)) message = '' if request.method == 'POST': try: acct = request.form.get('acct') app.logger.info('acct: {}'.format(acct)) amount = float(request.form.get('amount')) app.logger.info('amount: {}'.format(amount)) if amount < 0: message = 'Amount must be >= 0' elif not is_valid_account(acct): message = 'Invalid account format' else: response = requests.get(config['FAUCET_HOST'], params={ 'address': acct, 'amount': format(amount * 1e6, '.0f') }) if response.status_code == 200: message = 'Sent {0} <small>Libra</small> to <a href="/account/{1}">{1}</a>'.format( amount, acct) except: message = 'Invalid request logged!' app.logger.exception(message) if message: message = faucet_alert_template.format(message) return faucet_template.format(bver, message)
async def add_rule_by_self( self, ctx, name: str, message: discord.Message, emoji: str, reward_sticker: Optional[int], reward_role: Optional[discord.Role], ): with session_scope() as session: rule = Rule( name=name, by_admin=False, message_id=message.id, emoji=str(emoji) ) if reward_sticker is not None: rule.reward_sticker = reward_sticker if reward_role is not None: rule.reward_role_id = reward_role.id session.add(rule) session.commit() self.bot.logger.info("rule add: " + repr(rule)) await message.add_reaction(emoji) await ctx.send(f"규칙 #{rule.id} 추가됨!")
def can_access_admin(session_id, account_id): if session_is_valid(session_id, account_id): with session_scope() as session: if session.query(Accounts.is_admin).filter( Accounts.id == account_id).first()[0]: return True return False
def all_objects_from_db(object_table): """ Gets all user objects from the database :return: a user object """ with session_scope() as session: for _object in session.query(object_table).all(): yield _object
def get(self, project, spider, jobid): with session_scope() as session: job = session.query(HistoricalJob).filter_by(id=jobid).first() log_file = job.log_file with open(log_file, 'r') as f: log = f.read() loader = get_template_loader() self.write(loader.load("log.html").generate(log=log))
def is_customer(account_id: str) -> bool: with session_scope() as session: user = session.query(Accounts).filter( Accounts.id == account_id).first() if user.account_type == "customer": return True else: return False
def user_exists(self, immutable_id): with session_scope() as session: user = session.query(User).filter_by(immutable_id=immutable_id).first() if user: return True else: return False
def get_next_job(self): with session_scope() as session: next_jobs = list( session.query(WebhookJob).filter( WebhookJob.status == 0).order_by(WebhookJob.id).slice( 0, 1)) if next_jobs: return next_jobs[0]
async def list_rules(self, ctx): with session_scope() as session: content = "\n".join(repr(rule) for rule in list_all_rules(session)) if not content: await ctx.send("규칙 없음") return await ctx.send(content, allowed_mentions=discord.AllowedMentions.none())
async def show_sticker(self, ctx): with session_scope() as session: user = get_user(session, ctx.author.id) if user is None: await ctx.send("아직 스티커를 받지 않았습니다.") return await ctx.send(f"스티커를 `{user.sticker}`장 갖고 있습니다.")
def poll_task_queue(self): if self.task_queue.empty(): with session_scope() as session: tasks_to_run = session.query(SpiderExecutionQueue).filter_by( status=0).order_by(SpiderExecutionQueue.update_time).slice( 0, 10) for task in tasks_to_run: self.task_queue.put(task)
def __init__(self): with session_scope() as session: # reset all running jobs to PENDING status on start for job in session.query(WebhookJob).filter( WebhookJob.status == 1): job.status = 0 session.add(job) session.commit()
async def reward_test(self, ctx, formlink: str): with session_scope() as session: user = search_by_formlink(session, formlink) if user is None: await ctx.send(f"{ctx.author.mention}, 해당하는 사용자를 찾지 못했습니다.") return await ctx.send(f"{ctx.author.mention}, {user}")
async def load_quiz(self, ctx): io = BytesIO() await ctx.message.attachments[0].save(io) with session_scope() as session: instances = [Quiz(**instance) for instance in json.load(io)] session.add_all(instances) await ctx.send(f"`{len(instances)}`개 데이터를 추가했습니다.")
def add_user_to_db(self, immutable_id, user_id, is_active): user = User() user.immutable_id = immutable_id user.azure_user_id = user_id user.is_active = is_active user.created = datetime.datetime.now() user.synced = datetime.datetime.now() with session_scope() as session: session.add(user)
def delete_users(self): """ Delete all the users that were not found on the CSV file but were still present in the database. """ for user in self.azure_user_list: with session_scope() as session: user = session.query(User).filter_by(immutable_id=user).first() self.remove_user_from_db(user.immutable_id) self.delete_user(user.azure_user_id)
def update_user_in_db(self, immutable_id): """ Updates the synced time with the current time """ with session_scope() as session: user = session.query(User).filter_by(immutable_id=immutable_id).first() if user: user.synced = datetime.datetime.now() session.add(user)
def user_exists(self, immutable_id): """ Returns True if user exists in the database """ with session_scope() as session: user_filter = exists().where(User.immutable_id==immutable_id) for user in session.query(User.immutable_id).filter(user_filter): if user: return True else: return False
def is_active(self, immutable_id): with session_scope() as session: for active in session.query(User.is_active).filter(User.immutable_id==immutable_id): return active
def remove_user_from_db(self, immutable_id): with session_scope() as session: user = session.query(User).filter_by(immutable_id=immutable_id).first() session.delete(user)