def run(): """ 后端爬虫入口 :return: """ loop = asyncio.get_event_loop() scheduler = AsyncIOScheduler() scheduler.add_listener(spider_listener, mask=EVENT_JOB_MAX_INSTANCES | EVENT_JOB_ERROR | EVENT_JOB_MISSED) asyncio.ensure_future(init_task(loop=loop), loop=loop) scheduler.add_job( func=refresh_task, args=(loop, scheduler), trigger="cron", second="*/10", misfire_grace_time=600, max_instances=2, coalesce=True, id="refresh-task", ) scheduler.start() asyncio.ensure_future(save.consume(loop, save.save_queue), loop=loop) try: loop.run_forever() except KeyboardInterrupt: logger.info(f"退出......") executor.shutdown(wait=False) scheduler.shutdown(wait=False) loop.close()
def setup_scheduler_application(): # noqa: WPS213 def shutdown_application(): # noqa: WPS430 logger.debug("Shutting down job scheduler") scheduler.shutdown(wait=True) logger.debug("Stopping event loop") asyncio.get_event_loop().stop() logger.debug("The application should shut down now") def on_job_error(event): # noqa: WPS430 logger.error( "Error when executing scheduled job. Shutting down application...", exc_info=event.exception, ) shutdown_application() def on_signal(signal_code): # noqa: WPS430 logger.info(f"Received signal {signal_code}. Shutting down application...") shutdown_application() scheduler = AsyncIOScheduler(logger=logger) scheduler.add_listener(on_job_error, mask=EVENT_JOB_ERROR) loop = asyncio.get_event_loop() loop.add_signal_handler(signal.SIGINT, on_signal, "SIGINT") loop.add_signal_handler(signal.SIGTERM, on_signal, "SIGTERM") return scheduler
async def initialize_scheduler(app, loop): logger.info("Starting job scheduler") global scheduler scheduler = AsyncIOScheduler({ "apscheduler.jobstores.default": { "type": "sqlalchemy", "url": DATABASE_URL, }, }) scheduler.start() scheduler.add_listener(partial(update_picker_rotation, scheduler), EVENT_JOB_EXECUTED)
def run(self, task=None, *args, **kwargs): if task is None: scheduler = AsyncIOScheduler() scheduler.add_job(self.stat_user, 'interval', minutes=1) scheduler.add_job(self.stat_post, 'interval', minutes=1) scheduler.add_listener(self.error_listener, EVENT_JOB_ERROR) scheduler.start() try: asyncio.get_event_loop().run_forever() except (KeyboardInterrupt, SystemExit): pass else: method = getattr(self, task) asyncio.get_event_loop().run_until_complete(method( *args, **kwargs))
spider() work_schedule() def execution_listener(event): if event.exception: _logger.error('The job crashed') else: # check that the executed job is the first job job = scheduler.get_job(event.job_id) if getattr(job, 'name', '') == 'spider': scheduler.add_job(work_schedule, name='work_schedule') if __name__ == '__main__': scheduler = AsyncIOScheduler() scheduler.add_job(spider, trigger='interval', hours=1, name='spider', next_run_time=datetime.now() + timedelta(seconds=4)) scheduler.add_listener(callback=execution_listener, mask=EVENT_JOB_EXECUTED | EVENT_JOB_ERROR) scheduler.start() # Execution will block here until Ctrl+C (Ctrl+Break on Windows) is pressed. try: asyncio.get_event_loop().run_forever() except (KeyboardInterrupt, SystemExit): pass
class Gallery(commands.Cog): """Handle the Gallery channels.""" delete_after = 15 compare = lambda x, y: collections.Counter(x) == collections.Counter(y) def __init__(self, bot): Gallery.bot = self self.bot = bot self.db = None self.cogset = dict() self.jobstore = SQLAlchemyJobStore(url=fr'sqlite:///{pathlib.Path.cwd() / "data" / "jobs" / "gallery.sqlite"}') jobstores = {"default": self.jobstore} self.scheduler = AsyncIOScheduler(jobstores=jobstores) self.scheduler.add_listener(self.job_missed, events.EVENT_JOB_MISSED) # -------------------- LOCAL COG EVENTS -------------------- async def cog_before_invoke(self, ctx): '''THIS IS CALLED BEFORE EVERY COG COMMAND, IT'S SOLE PURPOSE IS TO CONNECT TO THE DATABASE''' credentials = {"user": dblogin.user, "password": dblogin.pwrd, "database": dblogin.name, "host": dblogin.host} self.db = await asyncpg.create_pool(**credentials) return async def cog_after_invoke(self, ctx): '''THIS IS CALLED AFTER EVERY COG COMMAND, IT DISCONNECTS FROM THE DATABASE AND DELETES INVOKING MESSAGE IF SET TO.''' await self.db.close() await ctx.message.delete() return async def cog_command_error(self, ctx, error): if isinstance(error, discord.ext.commands.errors.NotOwner): try: owner = (self.bot.application_info()).owner except: owner = self.bot.get_guild(self.cogset['guild_id']).owner() await ctx.channel.send(content=f"```diff\n- {ctx.prefix}{ctx.invoked_with} is an owner only command, this will be reported to {owner.name}.") await owner.send(content=f"{ctx.author.mention} tried to use the owner only command{ctx.invoked_with}") return # -------------------- STATIC METHODS -------------------- @staticmethod def time_pat_to_hrs(t): ''' Converts a string in format <xdxh> (d standing for days and h standing for hours) to amount of hours. eg: 3d5h would be 77 hours Args: (str) or (int) Returns: (int) or (None) ''' try: timeinHours = int(t) return timeinHours except ValueError: valid = False #===== if input doesn't match basic pattern if (re.match(r"(\d+[DHdh])+", t)): #=== if all acsii chars in the string are unique letters = re.findall(r"[DHdh]", t) if len(letters) == len(set(letters)): #= if more then 1 letter side by side #= ie. if t was 2dh30m then after the split you'd have ['', 'dh', 'm', ''] if not ([i for i in re.split(r"[0-9]", t) if len(i) > 1]): # if letters are in order. if letters == sorted(letters, key=lambda letters: ["d", "h"].index(letters[0])): valid = True if valid: total_hours = int() for data in re.findall(r"(\d+[DHdh])", t): if data.endswith("d"): total_hours += int(data[:-1])*24 if data.endswith("h"): total_hours += int(data[:-1]) return total_hours return False @staticmethod async def split_list(arr, size=100): """Custom function to break a list or string into an array of a certain size""" arrs = [] while len(arr) > size: pice = arr[:size] arrs.append(pice) arr = arr[size:] arrs.append(arr) return arrs # -------------------- LISTENERS -------------------- @commands.Cog.listener() async def on_ready(self): self.cogset = await cogset.LOAD(cogname=self.qualified_name) if not self.cogset: self.cogset= dict( guild_id= 0, enable= False, channel_ids= [], text_expirein= None, rem_low= False, user_wl= [], allow_links= False, link_wl= [] ) await cogset.SAVE(self.cogset, cogname=self.qualified_name) await asyncio.sleep(120) # ===== SCHEDULER self.scheduler.start() @commands.Cog.listener() async def on_message(self, msg): # ===== RETURN IF GALLERYS ARE DISABLED or MESSAGE IS NOT FROM A GUILD if ( not self.cogset['enable'] or not msg.guild or not msg.type == discord.MessageType.default or msg.author.id in self.cogset['user_wl'] ): return if msg.channel.id in self.cogset['channel_ids']: valid = False ###=== IF MESSAGE HAS ATTACHMENTS ASSUME THE MESSAGE IS OF ART. if msg.attachments: if not self.cogset['rem_low']: valid = True else: toosmall = False for attch in msg.attachments: if attch.height < 300 and attch.width < 300: toosmall=True break valid = not toosmall ###=== IF LINKS ARE ALLOWED IN GALLERY CHANNELS if self.cogset['allow_links']: #- get the links from msg content links = re.findall(r"(?P<url>http[s]?://[^\s]+)", msg.content) ###= IF ONLY CERTAIN LINKS ARE ALLOWED if self.cogset['link_wl']: #= LOOP THROUGH THE LINKS FROM THE MESSAGE CONTENT AND THE WHITELISTED LINKS #= ASSUME VALID IF ONE LINK MATCHES. for link in links: for wl_link in self.cogset['link_wl']: if link.startswith(wl_link): valid = True break else: valid = False ###=== IF THE MESSAGE IS NOT VALID. if not valid: credentials = {"user": dblogin.user, "password": dblogin.pwrd, "database": dblogin.name, "host": dblogin.host} self.db = await asyncpg.create_pool(**credentials) await self.db.execute(pgCmds.ADD_GALL_MSG, msg.id, msg.channel.id, msg.guild.id, msg.author.id, msg.created_at) await self.db.close() # -------------------- COMMANDS -------------------- @commands.is_owner() @commands.command(pass_context=True, hidden=False, name='galenable', aliases=[]) async def cmd_galenable(self, ctx): """ [Bot Owner] Enables the gallery feature. Useage: [prefix]galenable """ # ===== SET LOCAL COG VARIABLE self.cogset['enable']= True # ===== ADD THE FUNCTION TO THE SCHEDULER self.scheduler.add_job(call_schedule, 'date', id="_delete_gallery_messages", run_date=get_next(hours=self.cogset['text_expirein']), kwargs={"func": "_delete_gallery_messages"} ) # ===== SAVE SETTINGS await cogset.SAVE(self.cogset, cogname=self.qualified_name) await ctx.channel.send(content="Galleries are **enabled**.") return @commands.is_owner() @commands.command(pass_context=True, hidden=False, name='galdisable', aliases=[]) async def cmd_galdisable(self, ctx): """ [Bot Owner] Disables the gallery feature. Useage: [prefix]galdisable """ # ===== SET LOCAL COG VARIABLE self.cogset['enable']= False # ===== SAVE SETTINGS await cogset.SAVE(self.cogset, cogname=self.qualified_name) # ===== DELETE THE JOB IF IT EXISTS for job in self.jobstore.get_all_jobs(): if ["_delete_gallery_messages"] == job.id.split(" "): self.scheduler.remove_job(job.id) await ctx.channel.send(content="Galleries are disabled.") return @commands.is_owner() @commands.command(pass_context=True, hidden=False, name='galtogglechannel', aliases=[]) async def cmd_galtogglechannel(self, ctx, channel): """ [Bot Owner] Add or remove a channel to the list of active gallery channels Useage: [prefix]galaddchannel <channelid/mention> """ # ===== GET CHANNEL ID try: ch_id = int(channel.lower().replace('<').replace('>').replace('#').strip()) except ValueError: ctx.send_help('galtogglechannel', delete_after=Gallery.delete_after) ret_msg="" # ===== REMOVE CHANNEL ID FROM LIST if ch_id in self.cogset['channel_ids']: self.cogset['channel_ids'].remove(ch_id) ret_msg = f"<#{ch_id}> is no longer a gallery channel." ###=== DELETE LOGGED MESSAGES FROM DATABASE await self.db.execute(pgCmds.DEL_GALL_MSGS_FROM_CH, ch_id, self.cogset['guild_id']) # ===== ADD CHANNEL ID TO LIST else: self.cogset['channel_ids'] = list(set(self.cogset['channel_ids']) + {ch_id}) ret_msg = f"<#{ch_id}> has been made a gallery channel." # ===== SAVE SETTINGS await cogset.SAVE(self.cogset, cogname=self.qualified_name) # ===== END await ctx.channel.send(content=ret_msg, delete_after=Gallery.delete_after) return @commands.is_owner() @commands.command(pass_context=True, hidden=False, name='galsetexpirehours', aliases=[]) async def cmd_galsetexpirehours(self, ctx, timepat): """ [Bot Owner] Sets how long the bot should wait to delete text only messages from gallery channels. Can be provided as <XdXh> Useage: [prefix]galsetexpirehours <hours> """ # ===== CHECK IF VALID new_time = Gallery.time_pat_to_hrs(timepat) if not new_time: await ctx.send_help("galsetexpirehours", delete_after=15) return # ===== SAVE COG SETTINGS self.cogset['text_expirein'] = new_time await cogset.SAVE(self.cogset, cogname=self.qualified_name) resetJob = False # ===== RESET THE JOB IF IT EXISTS for job in self.jobstore.get_all_jobs(): if ["_delete_gallery_messages"] == job.id.split(" "): self.scheduler.remove_job(job.id) resetJob = True if resetJob: # ===== ADD THE FUNCTION TO THE SCHEDULER self.scheduler.add_job(call_schedule, 'date', id="_delete_gallery_messages", run_date=get_next(hours=self.cogset['text_expirein']), kwargs={"func": "_delete_gallery_messages"} ) await ctx.channel.send(content=f"Text message expirey time has been set to {new_time} hours and the scheduler was reset.") return await ctx.channel.send(content=f"Text message expirey time has been set to {new_time} hours.") return @commands.is_owner() @commands.command(pass_context=True, hidden=False, name='galtoguserwl', aliases=[]) async def cmd_galtoguserwl(self, ctx, user_id): """ [Bot Owner] Toggles a user in the gallery user whitelist. If they are in the whitelist then their messages are persistent and not deleted retroactivly. Useage: [prefix]galtoguserwl <userid/mention> """ # ===== CHECK IF INPUT IS VALID try: user_id = int(user_id.replace("<", '').replace("@", '').replace("!", '').replace(">", '')) except (IndexError, ValueError): await ctx.send_help('galtoguserwl', delete_after=Gallery.delete_after) return # ===== REMOVE OR ADD USER TO THE WHITELIST ret_msg = "" if user_id in self.cogset['user_wl']: self.cogset['user_wl'].remove(user_id) ret_msg = f'<@{user_id} has been **removed** from the gallery whitelist.' else: self.cogset['user_wl'].append(user_id) ret_msg = f'<@{user_id} has been **added** to the gallery whitelist.' # ===== WRITE TO THE DATABASE await cogset.SAVE(self.cogset, cogname=self.qualified_name) # ===== RETURN await ctx.channel.send(content=ret_msg, delete_after=Gallery.delete_after) return @commands.is_owner() @commands.command(pass_context=True, hidden=False, name='galltogglelinks', aliases=[]) async def cmd_galtogglelinks(self, ctx, tog=None): """ [Bot Owner] Toggle links in the gallery channels or you can set if links are allowed with true or false. Useage: [prefix]galltogglelinks [] """ update = not self.cogset['allow_links'] # ===== IF EXPLICITLY SETTING LINK STATUS if tog is not None: if tog.lower() in ['y', 'true', 'ture', 't']: update = True if self.cogset['allow_links']: await ctx.channel.send("Galleries are already **enabled**.", delete_after=Gallery.delete_after) elif tog.lower() in ['n', 'false', 'flase', 'f']: update = False if not self.cogset['allow_links']: await ctx.channel.send("Galleries are already **disabled**.", delete_after=Gallery.delete_after) self.cogset['allow_links']=update # ===== WRITE TO THE DATABASE await cogset.SAVE(self.cogset, cogname=self.qualified_name) # ===== RETURN await ctx.channel.send(content=f"Links in the gallery channels is now set to{update}.", delete_after=Gallery.delete_after) return ### ADD LINK WHITELIST @commands.is_owner() @commands.command(pass_context=True, hidden=False, name='galaddlinkuwl', aliases=[]) async def cmd_galaddlinkuwl(self, ctx): """ [Bot Owner] Adds a link from gallery link whitelist. Useage: [prefix]galaddlinkuwl <startoflink> """ links = re.findall(r"(?P<url>http[s]?://[^\s]+)", ctx.message.content) if not links: await ctx.channel.send('`Useage: [p]galaddlinkuwl <startoflink>, [Bot Owner] Adds a link from gallery link whitelist.`') # ===== ADD THE NEW LINKS TO THE WHITELIST new_gal_link_wl = list(set(self.cogset['link_wl']) + set(links)) if Gallery.compare(new_gal_link_wl, self.cogset['link_wl']): await ctx.channel.send(content="{}\n are already in the gallery link whitelist.".format('\n'.join(links)), delete_after=Gallery.delete_after) return else: self.cogset['link_wl'] = new_gal_link_wl # ===== WRITE TO THE DATABASE await cogset.SAVE(self.cogset, cogname=self.qualified_name) # ===== RETURN await ctx.channel.send(content="{}\n have been added to the gallery link whitelist.".format('\n'.join(links)), delete_after=Gallery.delete_after) return ### REM LINK WHITELIST @commands.is_owner() @commands.command(pass_context=True, hidden=False, name='galremlinkuwl', aliases=[]) async def cmd_galremlinkuwl(self, ctx): """ [Bot Owner] Removes a link from gallery link whitelist. Useage: [prefix]galremlinkuwl <startoflink> """ links = re.findall(r"(?P<url>http[s]?://[^\s]+)", ctx.message.content) if not links: await ctx.channel.send('Useage: [p]galremlinkuwl <startoflink>, [Bot Owner] Removes a link from gallery link whitelist.') # ===== REMOVE THE LINKS FROM THE LIST new_gal_link_wl = list(set(self.cogset['link_wl']) - set(links)) if Gallery.compare(new_gal_link_wl, self.cogset['link_wl']): await ctx.channel.send(content="{}\n are not in the gallery link whitelist.".format('\n'.join(links)), delete_after=Gallery.delete_after) return else: self.cogset['link_wl'] = new_gal_link_wl # ===== WRITE TO THE DATABASE await cogset.SAVE(self.cogset, cogname=self.qualified_name) # ===== RETURN await ctx.channel.send(content="{}\n have been removed from the gallery link whitelist.".format('\n'.join(links)), delete_after=Gallery.delete_after) return ### SPECIAL @commands.is_owner() @commands.command(pass_context=True, hidden=False, name='galloadsettings', aliases=[]) async def cmd_galloadsettings(self, ctx): """ [Bot Owner] Loads gallery settings from the setup.ini file Useage: [prefix]galloadsettings """ config = Config() # ===== UPDATE THE SETTINGS IN THE LOCAL COG self.cogset['guild_id'] = config.target_guild_id self.cogset['enable']= config.galEnable self.cogset['channel_ids'] = config.gallerys["chls"] self.cogset['text_expirein']= config.gallerys['expire_in'] self.cogset['rem_low']= config.gallerys['rem_low'] self.cogset['user_wl']= config.gallerys["user_wl"] self.cogset['allow_links']= config.gallerys["links"] self.cogset['link_wl']= config.gallerys['link_wl'] # ===== SAVE COG SETTING await cogset.SAVE(self.cogset, cogname=self.qualified_name) # ===== RETURN await ctx.channel.send(content="Gallery information has been updated from the setup.ini file", delete_after=15) return @commands.is_owner() @commands.command(pass_context=True, hidden=False, name='galsettings', aliases=[]) async def cmd_galsettings(self, ctx, showlinks=False): sched = None for job in self.jobstore.get_all_jobs(): if ["_delete_gallery_messages"] == job.id.split(" "): sched = job embed=discord.Embed( title= "Gallery Channel Settings.", description=f"**Enabled:** {self.cogset['enable']}\n" f"**Expire time:** {self.cogset['text_expirein']} hours", colour= RANDOM_DISCORD_COLOR(), type= "rich", timestamp= datetime.datetime.utcnow() ) embed.add_field( name= "Gallery Channels", value= "\n".join([f"<#{ch_id}>" for ch_id in self.cogset['channel_ids']]) or "None", inline= False ) embed.add_field( name= "Whitelisted Members", value= "\n".join([f"<#{user_id}>" for user_id in self.cogset['user_wl']]) or "None", inline= False ) if showlinks: links = "\n".join(self.cogset["link_wl"]) embed.add_field( name= "Gallery Links", value= f'**Allowed:** {self.cogset["allow_links"]}\n' f'**Links:** {links}', inline= False ) if sched: run_time = sched.next_run_time.__str__() else: run_time = "never" embed.add_field( name= "Scheduler", value= f"**Next run time:** {run_time}", inline= False ) embed.set_footer( icon_url= GUILD_URL_AS(ctx.guild) if ctx.guild else AVATAR_URL_AS(self.bot.user), text= "Gallery Settings" ) await ctx.channel.send(embed=embed) return # -------------------- SCHEDULING -------------------- def job_missed(self, event): """ This exists too """ asyncio.ensure_future(call_schedule(*event.job_id.split(" "))) @staticmethod def get_id_args(func, arg): """ I have no damn idea what this does """ return "{} {}".format(func.__name__, arg) @commands.is_owner() @commands.command(pass_context=True, hidden=False, name='galinitiateschedule', aliases=[]) async def cmd_galinitiateschedule(self, ctx): # ===== DELETE THE JOB IF IT ALREADY EXISTS for job in self.jobstore.get_all_jobs(): if ["_delete_gallery_messages"] == job.id.split(" "): self.scheduler.remove_job(job.id) # ===== ADD THE FUNCTION TO THE SCHEDULER self.scheduler.add_job(call_schedule, 'date', id="_delete_gallery_messages", run_date=get_next(hours=self.cogset['text_expirein']), kwargs={"func": "_delete_gallery_messages"} ) # ===== RETURN await ctx.channel.send(content=f"Gallery schedule has been set for {get_next(hours=self.cogset['text_expirein'])}") return async def _delete_gallery_messages(self, *args): # ===== QUIT ID GALLERIES ARE DISABLED. if not self.cogset['enable']: return # ===== CONNECT TO THE DATABASE credentials = {"user": dblogin.user, "password": dblogin.pwrd, "database": dblogin.name, "host": dblogin.host} self.db = await asyncpg.create_pool(**credentials) after = (datetime.datetime.utcnow() - datetime.timedelta(hours=self.cogset['text_expirein'])) t = await self.db.fetch(pgCmds.GET_GALL_MSG_BEFORE, after) ch_ids = await self.db.fetch(pgCmds.GET_GALL_CHIDS_BEFORE, after) await self.db.execute(pgCmds.DEL_GALL_MSGS_BEFORE, self.cogset['guild_id'], after) await self.db.close() # ===== TURNING THE DATA INTO SOMETHING MORE USEFUL ch_ids = [ch_id['ch_id'] for ch_id in ch_ids] fast_delete = dict() slow_delete = [] for ch_id in ch_ids: fast_delete[ch_id] = [] now = datetime.datetime.utcnow() delta = datetime.timedelta(days=13, hours=12) for record in t: ###=== IF MESSAGE IS OLDER THAN 13 DAYS AND 12 HOURS if bool((now - record['timestamp']) > delta): slow_delete.append(record) ###=== IF MESSAHE IS YOUNGER THAN 13 DAYS AND 12 HOURS else: fast_delete[record['ch_id']].append(record['msg_id']) # ===== IF THERE IS FAST DELETE DATA # WITH FAST DELETE MESSAGES WE CAN DELETE MESSAGES IN BULK OF 100 if fast_delete: for ch_id in fast_delete.keys(): msgs_ids = await Gallery.split_list(fast_delete[ch_id], 100) for msg_ids in msgs_ids: if len(msg_ids) > 1: await self.bot.http.delete_messages(ch_id, msg_ids, reason="Deleting Gallery Messages") await asyncio.sleep(0.5) else: # SOMETIMES AN EMPTY LIST MAKES IT HERE if msg_ids: msg_id = msg_ids[0] await self.bot.http.delete_message(ch_id, msg_id, reason="Deleting Gallery Messages") await asyncio.sleep(0.5) # ===== IF THERE IS SLOW DELETE DATA # WE CANNOT DELETE THESE MESSAGES IN BULK, ONLY ONE BY ONE. if slow_delete: for record in slow_delete: await self.bot.http.delete_message(record['ch_id'], record['msg_id'], reason="Deleting Gallery Messages") await asyncio.sleep(0.5) ###==== LOOP THE SCHEDULER self.scheduler.add_job( call_schedule, 'date', id="_delete_gallery_messages", run_date=get_next(hours=self.cogset['text_expirein']), kwargs={"func": "_delete_gallery_messages"} ) return
# EVENT HANDLERS # def jobExecHandler(event: JobExecutionEvent): try: db = SessionLocal() job = AfishScheduler.get_job(event.job_id) output = str(event.exception) if event.exception else event.retval execution = Execution( eid=str(uuid.uuid4()), job_id=event.job_id, name=job.name, module=job.kwargs["module"], project=job.kwargs["project"], output=output, state=EventCodes(event.code).name, scheduled_time=event.scheduled_run_time, ) db.add(execution) db.commit() if event.exception: AfishScheduler.pause_job(event.job_id) except Exception as e: logger.error(e) AfishScheduler.add_listener( jobExecHandler, EventCodes.EVENT_JOB_EXECUTED | EventCodes.EVENT_JOB_ERROR)
class NewMembers(commands.Cog): """Private feedback system.""" config = None delete_after = 15 def __init__(self, bot): self.bot = bot NewMembers.bot = self self.cogset = dict() self.roles = dict() self.db = None self.tguild = None self.jobstore = SQLAlchemyJobStore( url= fr'sqlite:///{pathlib.Path.cwd() / "data" / "jobs" / "newmembers_jobstore.sqlite"}' ) jobstores = {"default": self.jobstore} self.scheduler = AsyncIOScheduler(jobstores=jobstores) self.scheduler.add_listener(self.job_missed, events.EVENT_JOB_MISSED) # -------------------- LOCAL COG STUFF -------------------- async def connect_db(self): """ Connects to the database using variables set in the dblogin.py file. """ credentials = { "user": dblogin.user, "password": dblogin.pwrd, "database": dblogin.name, "host": dblogin.host } self.db = await asyncpg.create_pool(**credentials) return async def disconnet_db(self): """ Closes the connection to the database. """ await self.db.close() return @asyncio.coroutine async def cog_command_error(self, ctx, error): print('Ignoring exception in {}'.format(ctx.invoked_with), file=sys.stderr) print(error) return def cog_unload(self): pass # -------------------- STATIC METHODS -------------------- @staticmethod def time_pat_to_secs(t): ''' Converts a string in format <xDxHxMxS> (d for days, h for hours, M for minutes, S for seconds) to amount of seconds. eg: 3d5h would be 77 hours Args: (str) or (int) Returns: (int) or (None) ''' try: total_seconds = int(t) return total_seconds except ValueError: valid = False #===== if input doesn't match basic pattern if (re.match(r"(\d+[DHMSdhms])+", t)): #=== if all acsii chars in the string are unique letters = re.findall(r"[DHMSdhms]", t) if len(letters) == len(set(letters)): #= if more then 1 letter side by side #= ie. if t was 2dh30m then after the split you'd have ['', 'dh', 'm', ''] if not ([i for i in re.split(r"[0-9]", t) if len(i) > 1]): # if letters are in order. if letters == sorted( letters, key=lambda letters: ["d", "h", "m", "s"].index( letters[0])): valid = True if valid: total_seconds = int() for data in re.findall(r'(\d+[DHMSdhms])', t): if data.endswith("d"): total_seconds += int(data[:-1]) * 86400 if data.endswith("h"): total_seconds += int(data[:-1]) * 3600 if data.endswith("m"): total_seconds += int(data[:-1]) * 60 if data.endswith("s"): total_seconds += int(data[:-1]) return total_seconds return False # -------------------- LISTENERS -------------------- @commands.Cog.listener() async def on_ready(self): # ---------- LOAD COGSET ---------- self.cogset = await cogset.LOAD(cogname=self.qualified_name) if not self.cogset: self.cogset = dict(NMlastmsgid=0, NMlastchid=0, guildclosed=False, agreeoff=False) await cogset.SAVE(self.cogset, cogname=self.qualified_name) # ---------- WAIT FOR BOT TO RUN ON_READY ---------- await asyncio.sleep(5) # ---------- LOG INVITES ---------- inviteLog = await self.__get_invite_info() if inviteLog is not None: await self.bot.db.execute(pgCmds.ADD_INVITES, json.dumps(inviteLog)) self.bot.safe_print("[Log] Invite information has been logged.") else: self.bot.safe_print("[Log] No invite information to log.") # ---------- GET IMPORTANT ROLES READY ---------- self.tguild = self.bot.get_guild(self.bot.config.target_guild_id) self.roles['member'] = discord.utils.get( self.tguild.roles, id=self.bot.config.roles['member']) self.roles['newmember'] = discord.utils.get( self.tguild.roles, id=self.bot.config.roles['newmember']) self.roles['gated'] = discord.utils.get( self.tguild.roles, id=self.bot.config.roles['gated']) self.roles['name_colour'] = discord.utils.get( self.tguild.roles, id=self.bot.config.name_colors[0]) # ---------- SCHEDULER ---------- self.scheduler.start() self.scheduler.print_jobs() # ---------- CHECK NEW MEMBERS ---------- await self.check_new_members() # ---------- START TASK LOOPS ---------- #self.updateNewMembers.start() not this @commands.Cog.listener() async def on_resume(self): # ===== WAIT FOR BOT TO FINISH SETTING UP await self.bot.wait_until_ready() # ===== LOG INVITES inviteLog = await self.__get_invite_info() if inviteLog is not None: await self.bot.db.execute(pgCmds.ADD_INVITES, json.dumps(inviteLog)) self.bot.safe_print("[Log] Invite information has been logged.") else: self.bot.safe_print("[Log] No invite information to log.") @commands.Cog.listener() async def on_member_join(self, m): ###===== WAIT FOR THE BOT TO BE FINISHED SETTING UP await self.bot.wait_until_ready() # ---------- GET INVITE INFO ---------- invite = await self.__get_invite_used() # ---------- LOG NEW MEMBER ---------- embed = await GenEmbed.getMemJoinStaff(member=m, invite=invite) await self.bot.send_msg_chid(self.bot.config.channels['bot_log'], embed=embed) # ---------- SEND WELCOME MESSAGE ---------- fmt = random.choice([ f'Oh {m.mention} steps up to my dinner plate, I mean to {m.guild.name}!', f"I'm so excited to have {m.mention} join us, that I think I'll tear up the couch!", f"Well dip me in batter and call me a nugget, {m.mention} has joined us at {m.guild.name}!", f"The gates of {m.guild.name} have opened to: {m.mention}.", f"Attention {m.mention}, all new members of {m.guild.name} must be approved by me and I approve of you *hugs*." ]) #fmt += "\nPlease give the rules in <#" + self.bot.config.channels['public_rules_id'] + "> a read and when you're ready make a post in <#" + self.bot.config.channels['entrance_gate'] + "> saying that you agreed to the rules." await asyncio.sleep(0.5) welMSG = await self.bot.send_msg_chid( self.bot.config.channels['bot_log'], content=fmt, guild_id=m.guild.id) # ---------- Update Database ---------- await self.bot.db.execute(pgCmds.ADD_WEL_MSG, welMSG.id, welMSG.channel.id, welMSG.guild.id, m.id) await self.bot.db.execute(pgCmds.ADD_MEMBER_FUNC, m.id, m.joined_at, m.created_at) # ---------- AUTO ROLES ---------- if self.bot.config.roles["autoroles"]: for r_id in self.bot.config.roles['autoroles']: await asyncio.sleep(0.4) role = discord.utils.get(m.guild.roles, id=r_id) await m.add_roles(role, reason="Auto Roles") # ---------- Schedule a kick ---------- await self.schedule_kick(m, daysUntilKick=Days.gated, days=Days.gated) @commands.Cog.listener() async def on_member_remove(self, m): # ===== WAIT FOR THE BOT TO BE FINISHED SETTING UP await self.bot.wait_until_ready() # ===== IGNORE NON-TARGET GUILDS if m.guild.id != self.bot.config.target_guild_id: return # ---------- CANCEL SCHEDULED KICK ---------- await self.cancel_scheduled_kick(member=m) # ---------- IF MEMBER IS KICKED OR BANNED ---------- # ===== WAIT A BIT TO MAKE SURE THE GUILD AUDIT LOGS ARE UPDATED BEFORE READING THEM await asyncio.sleep(0.2) banOrKick = list() past_id = discord.utils.time_snowflake(datetime.datetime.utcnow() - datetime.timedelta(seconds=10), high=True) try: for i in [discord.AuditLogAction.ban, discord.AuditLogAction.kick]: async for entry in m.guild.audit_logs(limit=30, action=i, oldest_first=False): if entry.id >= past_id and entry.target.id == m.id: if banOrKick: if entry.id > banOrKick[4]: banOrKick = [ entry.action, entry.target, entry.user, entry.reason or "None", entry.id ] else: banOrKick = [ entry.action, entry.target, entry.user, entry.reason or "None", entry.id ] except discord.errors.Forbidden: self.bot.safe_print("[Info] Missing view_audit_log permission.") except discord.errors.HTTPException: self.bot.safe_print( "[Info] HTTP error occurred, likely being rate limited or blocked by CloudFlare. Restart recommended." ) # ---------- REMOVED MEMBER LOGGING ---------- # ===== STAFF ONLY LOGGING embed = await GenEmbed.getMemLeaveStaff(m, banOrKick) await self.bot.send_msg_chid(self.bot.config.channels['bot_log'], embed=embed) # ===== PUBLIC VISABLE LOGGING, ONLY APPLICABLE IF EXMEMBER WAS GIVEN THE CORE ROLE if discord.utils.get(m.roles, id=self.bot.config.roles['member']): wel_ch = self.bot.get_channel( self.bot.config.channels['public_bot_log']) async with wel_ch.typing(): # = GET THE USERS PFP AS BYTES avatar_bytes = await GET_AVATAR_BYTES(user=m, size=128) # = SAFELY RUN SOME SYNCRONOUS CODE TO GENERATE THE IMAGE final_buffer = await self.bot.loop.run_in_executor( None, partial(images.GenGoodbyeImg, avatar_bytes, m, banOrKick)) # = SEND THE RETURN IMAGE await wel_ch.send( file=discord.File(filename="goodbye.png", fp=final_buffer)) # ---------- REMOVE WELCOME MESSAGES ---------- await self.del_user_welcome(m) # ---------- UPDATE THE DATABASE ---------- await self.bot.db.execute(pgCmds.REMOVE_MEMBER_FUNC, m.id) # ===== END return @commands.Cog.listener() async def on_member_update(self, before, after): """When there is an update to a users user data""" await self.bot.wait_until_ready() # Wait for bot to finish setting up if before.guild.id != self.bot.config.target_guild_id: return # Ignore non-target guilds # ===== Make sure cog is finished setting up while True: if "member" in self.roles: break await asyncio.sleep(1) # ===== HANDLING FOR STAFF ADDING THE MEMBER ROLE TO NEW USERS MANUALLY if {self.roles['member'], self.roles['gated']}.issubset( set(after.roles)) and self.roles['gated'] in before.roles: await self.handle_gated2member(after) return @commands.Cog.listener() async def on_message(self, msg): await self.bot.wait_until_ready() # Wait for bot to finish setting up # ===== IF MESSAGE WAS NOT IN ENTRANCE GATE if msg.channel.id != self.bot.config.channels['entrance_gate']: return # ===== IF THE MESSAGE IS A BOT COMMAND if (msg.content[len(self.bot.command_prefix):].split(" ") )[0] in self.bot.all_cmds: return # ===== IF THE AUTHOR IS GATED, LOG THE MESSAGE. IGNORES STAFF SINCE THEY TEND TO MESS AROUND if (any(role.id == self.bot.config.roles['gated'] for role in msg.author.roles) and not any(role.id in self.bot.config.roles['any_staff'] for role in msg.author.roles)): await self.bot.db.execute(pgCmds.ADD_WEL_MSG, msg.id, msg.channel.id, msg.guild.id, msg.author.id) return # ===== CYCLE THROUGH ALL THE MEMBER'S MENTIONED IN THE MESSAGE for member in msg.mentions: # === IF MENTIONED MEMBER HAS THE GATED ROLE AND IS NOT STAFF if (any(role.id == self.bot.config.roles['gated'] for role in member.roles) and not any(role.id in self.bot.config.roles['any_staff'] for role in member.roles)): await self.bot.db.execute(pgCmds.ADD_WEL_MSG, msg.id, msg.channel.id, msg.guild.id, member.id) break return @commands.Cog.listener() async def on_guild_role_update(self, before, after): # ===== WAIT FOR THE BOT TO BE FINISHED SETTING UP await self.bot.wait_until_ready() # ===== IF STORED MEMBER ROLE WAS UPDATED if self.roles['member'] == before: self.roles['member'] = after # ===== IF STORED NEWMEMBER ROLE WAS UPDATED elif self.roles['newmember'] == before: self.roles['newmember'] = after # ===== IF STORED GATED ROLE WAS UPDATED elif self.roles['gated'] == before: self.roles['gated'] = after return # -------------------- COMMANDS -------------------- @checks.HIGHEST_STAFF() @commands.command(pass_context=True, hidden=False, name='clearEntranceGate', aliases=['clearentrancegate']) async def cmd_clearentrancegate(self, ctx): """ [Minister] Kick members who have sat in the entrance gate for 14 days or more. """ currDateTime = datetime.datetime.utcnow() oldFreshUsers = [ member for member in ctx.guild.members if (self.roles['gated'] in member.roles) and ( self.roles['member'] not in member.roles) and ( (currDateTime - member.joined_at).days > Days.gated) ] if len(oldFreshUsers) == 0: await ctx.send( content="No members need to be kicked at this time.", delete_after=10) return react = await self.bot.ask_yn( ctx, "{} gated users will be kicked.\nAre you sure you want to continue?" .format(len(oldFreshUsers)), timeout=120, expire_in=2) #===== if user says yes if react: try: for member in oldFreshUsers: await member.kick( reason= f"Manual clearing of the entrance gate by {ctx.author.id}" ) await asyncio.sleep(0.5) await ctx.send( content=f"Done, {len(oldFreshUsers)} members kicked", delete_after=30) except discord.errors.Forbidden: await ctx.send( content="Can't kick members due to lack of permissions.", delete_after=30) except discord.errors.HTTPException: await ctx.send( content= "Some error occurred. Go blame discord and try again later.", delete_after=30) return #===== Time out handing elif react == None: await ctx.send( content="You took too long respond. Canceling action.", delete_after=30) #===== if user says no else: await ctx.send(content="Alright then, no members kicked.", delete_after=30) return @checks.HIGHEST_STAFF() @commands.command(pass_context=True, hidden=False, name='closeGuild', aliases=['closeguild']) async def cmd_closeguild(self, ctx, timer=None): """ [Admins] Closes the guild either for a certain amount of time in seconds or until manually reopened. Useage: [p]closeguild <xDxHxMxS>/<S> (d for days, h for hours, M for minutes, S for seconds) eg: [p]closeguild 4D3H """ t = None if timer: t = NewMembers.time_pat_to_secs(timer) if not t: ctx.send_help('closeGuild') # ===== EDIT COGSET self.cogset['guildclosed'] = True await cogset.SAVE(self.cogset, cogname=self.qualified_name) if t: await self.schedule_reopen_guild(t) else: embed = await GenEmbed.genCloseGuild() await self.bot.send_msg_chid(self.bot.config.channels['bot_log'], embed=embed) return @checks.GATED() @commands.command(pass_context=False, hidden=False, name='agree', aliases=['iagree', 'letmein']) async def cmd_agree(self, ctx): """ [Gated] Lets a new member sitting in the gate into the rest of the guild. """ if self.cogset['agreeoff'] or self.cogset['guildclosed']: await ctx.send( f"```\nSorry <@{ctx.author.id}>, but the guild is not accepting new members at this time. This is most likely due to a raid.\nPlease ask staff for help or check back later.\n```" ) await ctx.author.add_roles(self.roles['member']) return # -------------------- FUNCTIONS -------------------- @asyncio.coroutine async def handle_gated2member(self, member): # ===== ADD NEW MEMBER AND REMOVE GATED ROLES await member.remove_roles(self.roles['gated'], reason="Removed Gated Role") await asyncio.sleep(0.2) await member.add_roles(self.roles['newmember'], reason="Added new member role") await asyncio.sleep(0.2) await member.add_roles(self.roles['name_colour'], reason="Added name colour") # ===== SCHEDULE REMOVAL OF NEW MEMBER ROLE await self.schedule_rem_newuser_role(member, daysUntilRemove=7, days=7) # ===== CANCEL EXISTING MEMBER KICK await self.cancel_scheduled_kick(member) # ===== TELL THE USERS A NEW MEMBER HAS JOINED wel_ch = self.bot.get_channel( self.bot.config.channels['public_bot_log']) async with wel_ch.typing(): # === GET THE USERS PFP AS BYTES avatar_bytes = await GET_AVATAR_BYTES(user=member, size=128) # === SAFELY RUN SOME SYNCRONOUS CODE TO GENERATE THE IMAGE final_buffer = await self.bot.loop.run_in_executor( None, partial(images.GenWelcomeImg, avatar_bytes, member)) # === SEND THE RETURN IMAGE await wel_ch.send( file=discord.File(filename="welcome.png", fp=final_buffer)) # ===== DELETE USER MESSAGES IN THE GATE await self.del_user_welcome(member) return @asyncio.coroutine async def del_user_welcome(self, user): """ Custom func to delete a users welcome message """ # ===== GRAB ALL THE WELCOME MESSAGES FROM THE DATABASE RELATED TO THE USER IN QUESTION. welcomeMessages = await self.bot.db.fetch(pgCmds.GET_MEM_WEL_MSG, user.id) bulkDelete = {} now = datetime.datetime.utcnow() # ===== DO NOTHING IF NOT DATA if not welcomeMessages: return # ===== CYCLE THROUGH OUR DATABASE DATA for MYDM in welcomeMessages: # === LOG MESSAGES INTO OUR DICT IF THEY CAN BE DELETED IN BULK if (now - MYDM["timestamp"]).days < 13: # = IF CHANNEL ID DOES NOT EXIST AS A KEY if MYDM['ch_id'] not in bulkDelete.keys(): bulkDelete[MYDM['ch_id']] = list() bulkDelete[MYDM['ch_id']].append(MYDM["msg_id"]) else: # = IF MESSAGE IS TOO OLD, DELETE ONE BY ONE. await self.bot.delete_msg_id(MYDM["msg_id"], MYDM["ch_id"], reason="Welcome message cleanup.") await asyncio.sleep(0.2) # ===== IF THERE ARE MESSAGES TO BE BULK DELETED. if bulkDelete: # === EVEN THOUGH ALL MESSAGES WILL MOST LIKELY BE FROM THE SAME CHANNEL, THIS ENSURES COMPATIBILTY WITH WELCOME MESSAGES FROM MULTIPLE CHANNELS for i in bulkDelete.keys(): await self.bot.delete_msgs_id( messages=bulkDelete[i], channel=i, reason="Welcome message cleanup.") # ===== DELETE WELCOME MESSAGES FROM THE DATABASE await self.bot.db.execute(pgCmds.REM_MEM_WEL_MSG, user.id) @asyncio.coroutine async def __get_invite_used(self): """ When called it tries to find the invite used by calling the equivalent handler. Will return none if previous history file is not found if history file could not be read if new invite info cannot be found It will try to update the the invite info file as long as the current info can be found. """ #===== Get current invite info inviteLog = await self.__get_invite_info() #=== if info cannot be gotten if inviteLog == None: invite = None #=== if info received else: invite = await self.__get_invite_used_handler(inviteLog) await self.bot.db.execute(pgCmds.ADD_INVITES, json.dumps(inviteLog)) return invite @asyncio.coroutine async def __get_invite_info(self, quiet=False): """Returns a dict with the information on the invites of selected guild""" try: invites = await self.bot.get_guild(self.bot.config.target_guild_id ).invites() except discord.Forbidden: if not quiet: await self.bot.send_msg_chid( self.bot.config.channels['bot_log'], content= "```css\nAn error has occurred```I do not have proper permissions to get the invite information." ) return None except discord.HTTPException: if not quiet: await self.bot.send_msg_chid( self.bot.config.channels['bot_log'], content= "```css\nAn error has occurred```An error occurred when getting the invite information." ) return None inviteLog = list() for invite in invites: inviteLog.append( dict( max_age=invite.max_age, created_at=invite.created_at.__str__(), uses=invite.uses, max_uses=invite.max_uses, code=invite.id, inviter=dict( name=invite.inviter.name, id=invite.inviter.id, discriminator=invite.inviter.discriminator, avatar_url=invite.inviter.avatar_url.__str__(), mention=invite.inviter.mention) if invite.inviter != None else dict( name="N/A", id="N/A", discriminator="N/A", avatar_url= "https://discordapp.com/assets/6debd47ed13483642cf09e832ed0bc1b.png?size=128", mention="N/A"), channel=dict(name=invite.channel.name, id=invite.channel.id, mention=invite.channel.mention))) if len(inviteLog) == 0: return None else: return inviteLog @asyncio.coroutine async def __get_invite_used_handler(self, current_invite_info): """ Tries to find which invite was used by a user joining. """ #===== Read old invite info past_invite_info = json.loads(await self.bot.db.fetchval( pgCmds.GET_INVITE_DATA)) #===== Testing the existing invites. for past_invite in past_invite_info: for curr_invite in current_invite_info: if past_invite["code"] == curr_invite["code"]: if past_invite["uses"] < curr_invite["uses"]: return curr_invite #===== testing the new invites. should work if new invite is made and a user joins with that invite. for curr_invite in [ curr_invite for curr_invite in current_invite_info if curr_invite not in past_invite_info ]: if curr_invite["uses"] == 1: return curr_invite #===== CHECKING THE AUDIT LOG FOR INVITE CREATIONS guild = self.bot.get_guild(self.bot.config.target_guild_id) try: logs = await guild.audit_logs( action=discord.AuditLogAction.invite_create, before=(datetime.datetime.utcnow() - datetime.timedelta(days=1))).flatten() if len(logs) == 1: log = logs[0] invite = { "inviter": { 'mention': "<@{}>".format(log.user.id), 'name': log.user.name, 'discriminator': log.user.discriminator }, 'code': "N/A", 'uses': "N/A", 'max_uses': "N/A" } return invite except discord.Forbidden: return None return None # -------------------- SCHEDULING STUFF -------------------- # -------------------- Task loops -------------------- @tasks.loop(hours=24.0) async def updateNewMembers(self): if self.cogset['NMlastmsgid']: await self.bot.delete_msg_id(self.cogset['NMlastmsgid'], self.cogset['NMlastchid']) newmems = await self.bot.fetchval(pgCmds.GET_ADDED_MEMBERS) @updateNewMembers.before_loop async def before_updateNewMembers(self): await self.bot.wait_until_ready() # -------------------- Auto Kick Members -------------------- async def check_new_members(self): """ [Called on_ready] Adds members with the fresh role and not the core role to the scheduler via self.schedule_kick with the warning for member already in the scheduler turned off. Really only useful if the scheduled data in the SQL file has been lost. """ # ===== WAIT FOR THE BOT TO BE FINISHED SETTING UP await self.bot.wait_until_ready() # ===== VARIABLE SETUP guild = self.bot.get_guild(self.bot.config.target_guild_id) now = datetime.datetime.utcnow() for member in guild.members: # === IF MEMBER HAS ONLY THE EVERYONE ROLE if len(member.roles) == 1: # = APPLY THE AUTO ROLES if self.bot.config.roles["autoroles"]: for r_id in self.bot.config.roles['autoroles']: role = discord.utils.get(guild.roles, id=r_id) await member.add_roles(role, reason="Auto Roles") await asyncio.sleep(0.4) # = WORK OUT THE TIME THE USER HAS LEFT TO REGISTER diff = Days.gated - int((now - member.joined_at).days) # = IF MEMBER HAS BEEN ON THE GUILD FOR GREATER THEN 14 DAYS if diff < 1: diff = 1 await self.schedule_kick(member, daysUntilKick=diff, quiet=True, days=diff) # === ELSE IF MEMBER HAS THE GATED ROLE BUT NOT THE MEMBER ROLE elif (self.roles['gated'] in member.roles) and (self.roles['member'] not in member.roles): # = WORK OUT THE TIME THE USER HAS LEFT TO REGISTER diff = Days.gated - int((now - member.joined_at).days) # = IF MEMBER HAS BEEN ON THE GUILD FOR GREATER THEN 14 DAYS if diff < 1: diff = 1 await self.schedule_kick(member, daysUntilKick=diff, quiet=True, days=diff) # === IF MEMBER HAS THE MEMBER ROLE BUT NOT THE NEW MEMBER ROLE elif (self.roles['member'] in member.roles) and (self.roles['newmember'] not in member.roles): days = (Days.newmember + 1) - int( (now - member.joined_at).days) # = IF MEMBER HAS BEEN ON GUILD FOR LONGER THAN THE TIME REQUIRED FOR NEW MEMBER ROLE TO EXPIRE if days < 1: continue # = GIVE THE MEMBER THE NEWMEMBER ROLE await member.add_roles(self.roles['newmember'], reason="Added new member role") # = SCHEDULE THE NEW MEMBER ROLE FOR REMOVAL await self.schedule_rem_newuser_role(member, days) #-------------------- Remove New User Role -------------------- @asyncio.coroutine async def schedule_rem_newuser_role(self, member: Union[discord.User, discord.Member], daysUntilRemove=Days.newmember, **kwargs): """ [Called on_member_update] Adds the removal of a new member's fresh role to the scheduler. Handles: If member is already scheduled. It passes on the time allotted for an automatic kick to self._rem_newuser_role via the scheduler in the form of **kwargs """ ###===== IF MEMBER IS ALREADY SCHEDULED TO HAVE NEW MEMBER ROLE REMOVED, QUIT for job in self.jobstore.get_all_jobs(): if ["_rem_newuser_role", str(member.id)] == job.id.split(" "): return ###===== SEND REPORT MESSAGE TO STAFF embed = await GenEmbed.getScheduleRemNewRole( member=member, daysUntilRemove=daysUntilRemove) await self.bot.send_msg_chid(self.bot.config.channels['bot_log'], embed=embed) ###===== ADD EVENT TO THE SCHEDULER self.scheduler.add_job(call_schedule, 'date', id=self.get_id_args(self._rem_newuser_role, member.id), run_date=get_next(**kwargs), kwargs={ "func": "_rem_newuser_role", "arg": str(member.id) }) return @asyncio.coroutine async def cancel_rem_newuser_role(self, member): """ Cancels the scheduled kick of a member """ for job in self.jobstore.get_all_jobs(): if ["_rem_newuser_role", str(member.id)] == job.id.split(" "): self.scheduler.remove_job(job.id) @asyncio.coroutine async def _rem_newuser_role(self, user_id): """ [Assumed to be called by the scheduler] Takes a user id and removes their new member role. Handles: If member is not on the guild. if bot lacks permission to edit roles """ ###===== WAIT FOR THE BOT TO BE FINISHED SETTING UP await self.bot.wait_until_ready() guild = self.bot.get_guild(self.bot.config.target_guild_id) member = guild.get_member(int(user_id)) ###===== QUIT IF MEMBER HAS LEFT THE GUILD if member == None: return try: await member.remove_roles(self.roles['newmember'], reason="Auto remove New Member role") embed = await GenEmbed.genRemNewRole(member=member) await self.bot.send_msg_chid(self.bot.config.channels['bot_log'], embed=embed) except discord.Forbidden: self.bot.safe_print( f"I could not remove {member.mention}'s New Member role due to Permission error." ) except discord.HTTPException: self.bot.safe_print( f"I could not remove {member.mention}'s New Member role due to generic error." ) return #-------------------- Kick new members -------------------- @asyncio.coroutine async def cancel_scheduled_kick(self, member: Union[discord.User, discord.Member]): """ Cancels the scheduled kick of a member Args: (discord.User/discord.Member) Member you want to cancel kicking """ for job in self.jobstore.get_all_jobs(): if ["_kick_entrance", str(member.id)] == job.id.split(" "): self.scheduler.remove_job(job.id) @asyncio.coroutine async def schedule_kick(self, member, daysUntilKick=Days.gated, quiet=False, **kwargs): """ [Called on_member_join and check_new_members] Adds the automatic kick of a member from entrance gate after 14 days to the scheduler. Handles: If member is already scheduled to be kicked. It passes on the time allotted for an automatic kick to self._kick_entrance via the scheduler in the form of **kwargs """ for job in self.jobstore.get_all_jobs(): if ["_kick_entrance", str(member.id)] == job.id.split(" "): if not quiet: await self.bot.send_msg_chid( self.bot.config.channels['bot_log'], content="{0.mention} already scheduled for a kick". format(member)) return embed = await GenEmbed.getScheduleKick( member=member, daysUntilKick=daysUntilKick, kickDate=(datetime.datetime.now() + datetime.timedelta(seconds=( (daysUntilKick * 24 * 60 * 60) + 3600)))) await self.bot.send_msg_chid(self.bot.config.channels['bot_log'], embed=embed) #===== add the kicking of member to the scheduler self.scheduler.add_job(call_schedule, 'date', id=self.get_id_args(self._kick_entrance, member.id), run_date=get_next(**kwargs), kwargs={ "func": "_kick_entrance", "arg": str(member.id) }) @asyncio.coroutine async def _kick_entrance(self, user_id): """ [Assumed to be called by the scheduler] Takes a user id and kicks them from entrance gate. Handles: If member is not on the guild. if bot lacks permission to kick members """ ###===== WAIT FOR THE BOT TO FINISH IT'S SETUP await self.bot.wait_until_ready() guild = self.bot.get_guild(self.bot.config.target_guild_id) member = guild.get_member(int(user_id)) ###===== IF MEMBER IS NO LONGER ON THE GUILD if member == None: return gatedRole = discord.utils.get(guild.roles, id=self.bot.config.roles['gated']) memberRole = discord.utils.get(guild.roles, id=self.bot.config.roles['member']) try: #=== if member has fresh role and not core role if (gatedRole in member.roles) and (memberRole not in member.roles): #= kick member await member.kick(reason="Waited in entrance for too long.") #= report event embed = await GenEmbed.genKickEntrance( member, self.bot.config.channels['entrance_gate']) await self.bot.send_msg_chid( self.bot.config.channels['bot_log'], embed=embed) #===== Error if bot lacks permission except discord.errors.Forbidden: self.bot.safe_print( "[Error] (Scheduled event) I do not have permissions to kick members" ) await self.bot.send_msg_chid( self.bot.config.channels['bot_log'], content= "I could not kick <@{0.id}> | {0.name}#{0.discriminator}, due to lack of permissions" .format(member)) #===== Error for generic error, eg discord api gateway down except discord.errors.HTTPException: self.bot.safe_print( "[Error] (Scheduled event) I could not kick a member") await self.bot.send_msg_chid( self.bot.config.channels['bot_log'], content= "I could not kick <@{0.id}> | {0.name}#{0.discriminator}, due to an error" .format(member)) return #-------------------- Close Guild -------------------- @asyncio.coroutine async def schedule_reopen_guild(self, secondsUntilReopen=3600, **kwargs): """ [Called Close Guild Command] Adds re-open guild func to the scheduler. """ # ===== for job in self.jobstore.get_all_jobs(): if ["_reopen_guild"] == job.id.split(" "): return # ===== SEND REPORT MESSAGE TO STAFF embed = await GenEmbed.genReopenGuild(secondsUntilReopen) await self.bot.send_msg_chid(self.bot.config.channels['bot_log'], embed=embed) # ===== ADD EVENT TO THE SCHEDULER self.scheduler.add_job(call_schedule, 'date', id=self._reopen_guild.__name__, run_date=get_next(**kwargs), kwargs={"func": "_reopen_guild"}) return @asyncio.coroutine async def _reopen_guild(self): # ===== WAIT FOR THE BOT TO FINISH IT'S SETUP await self.bot.wait_until_ready() # ===== EDIT COGSET self.cogset['guildclosed'] = False await cogset.SAVE(self.cogset, cogname=self.qualified_name) # ===== REPORT TO STAFF embed = discord.Embed( title='Guild is now open.', description="Users will now be able to join the guild.", type="rich", timestamp=datetime.datetime.utcnow(), color=RANDOM_DISCORD_COLOR()) await self.bot.send_msg_chid(self.bot.config.channels['bot_log'], embed=embed) return #-------------------- TRINKETS -------------------- def job_missed(self, event): """ This exists too """ asyncio.ensure_future(call_schedule(*event.job_id.split(" "))) @staticmethod def get_id_args(func, arg): """ I have no damn idea what this does """ return "{} {}".format(func.__name__, arg)
class TimerTask(object): """docstring for TimerTask""" def __init__(self): super(TimerTask, self).__init__() self.logger = get_log("timerTask") self.max_instances = 20 # 最大并发数 # self.scheduler = BlockingScheduler() self.scheduler = AsyncIOScheduler() self.common = Common() # 清盘-实时行情校验 async def CleanData(self, exchange, code, loop, sub_quote_type=sub_quote_type): """ 测试清盘 """ self.logger.debug("执行的参数为: exchange: {}, code: {}, sub_quote_type: {}".format(exchange, code, sub_quote_type)) exchange = exchange code = code frequence = None isSubKLineMin = True query_type = 0 direct = 0 start = 0 end = 0 vol = 0 start_time_stamp = int(time.time() * 1000) isSubTrade = True type = "BY_VOL" start_time = start_time_stamp end_time = None vol = 100 count = 50 # http = MarketHttpClient() # market_token = http.get_market_token( # http.get_login_token(phone=login_phone, pwd=login_pwd, device_id=login_device_id)) market_token = None asyncio.set_event_loop(loop) try: api = SubscribeApi(union_ws_url, loop, logger=self.logger) await api.client.ws_connect() await api.LoginReq(token=market_token, start_time_stamp=start_time_stamp, frequence=frequence) asyncio.run_coroutine_threadsafe(api.hearbeat_job(), loop) self.logger.debug("订阅手机图表行情, 不会返回前快照数据和前盘口数据, {}, {}".format(sub_quote_type, code)) app_rsp_list = await api.StartChartDataReqApi(exchange, code, start_time_stamp, recv_num=1, sub_quote_type=sub_quote_type) app_rsp = app_rsp_list[0] basic_json_list = app_rsp.get("basicData") # 静态数据 assert self.common.searchDicKV(app_rsp.get("snapshot"), "high") is None assert self.common.searchDicKV(app_rsp.get("snapshot"), "open") is None assert self.common.searchDicKV(app_rsp.get("snapshot"), "low") is None assert self.common.searchDicKV(app_rsp.get("snapshot"), "close") is None assert self.common.searchDicKV(app_rsp.get("snapshot"), "last") is None if exchange not in ["ASE", "NYSE", "NASDAQ"]: # 美股盘前 assert app_rsp.get("orderbook") is None self.logger.debug("查询并订阅分时, 查询为空, 订阅成功, 不会返回前数据, {}, {}".format(sub_quote_type, code)) final_rsp = await api.QueryKLineMinMsgReqApi(isSubKLineMin, exchange, code, query_type, direct, start, end, vol, start_time_stamp, sub_quote_type=sub_quote_type) if sub_quote_type == "REAL_QUOTE_MSG": assert self.common.searchDicKV(final_rsp["query_kline_min_rsp_list"][0], 'retCode') == 'INITQUOTE_TIME' elif sub_quote_type == "DELAY_QUOTE_MSG": assert self.common.searchDicKV(final_rsp["query_kline_min_rsp_list"][0], 'data') is None assert self.common.searchDicKV(final_rsp["sub_kline_min_rsp_list"][0], 'retCode') == 'SUCCESS' assert final_rsp.get("before_kline_min_list") is None self.logger.debug("查询五日分时") fiveday_rsp = await api.QueryFiveDaysKLineMinReqApi(isSubKLineMin, exchange, code, start, start_time_stamp) query_5day_klinemin_rsp_list = fiveday_rsp['query_5day_klinemin_rsp_list'] sub_kline_min_rsp_list = fiveday_rsp['sub_kline_min_rsp_list'] assert self.common.searchDicKV(query_5day_klinemin_rsp_list[0], 'retCode') == 'SUCCESS' assert self.common.searchDicKV(sub_kline_min_rsp_list[0], 'retCode') == 'SUCCESS' # self.logger.debug(u'校验五日分时清盘时的数据, {}, {}'.format(sub_quote_type, code)) day_data_list = self.common.searchDicKV(query_5day_klinemin_rsp_list[0], 'dayData') assert day_data_list.__len__() == 5 # 获取五个交易日 fiveDateList = self.common.get_fiveDays(exchange) self.logger.debug("合约 {} , 五个交易日时间 : {}".format(code, fiveDateList)) for i in range(len(day_data_list)): # 校验五日date依次递增, 遇到节假日无法校验 assert day_data_list[i].get("date") == fiveDateList[i] info_list = self.common.searchDicKV(day_data_list[i], 'data') if info_list.__len__() > 0: if exchange == "HKFE": assert day_data_list[i].get("date") == info_list[-1].get("updateDateTime")[:8] else: assert day_data_list[i].get("date") == info_list[0].get("updateDateTime")[:8] self.logger.debug("查询并订阅逐笔, 查询为空, 订阅成功, 不会返回前数据, {}, {}".format(sub_quote_type, code)) final_rsp = await api.QueryTradeTickMsgReqApi(isSubTrade, exchange, code, type, direct, start_time, end_time, vol, start_time_stamp, sub_quote_type=sub_quote_type) try: assert final_rsp["query_trade_tick_rsp_list"] == [] except AssertionError: assert self.common.searchDicKV(final_rsp["query_trade_tick_rsp_list"], "data") is None assert self.common.searchDicKV(final_rsp["sub_trade_tick_rsp_list"][0], 'retCode') == 'SUCCESS' assert final_rsp.get("before_tickdata_list") is None # 港股才有经济席位 if exchange == "SEHK": # 只有港股有经济席位 self.logger.debug("订阅经济席位快照, 不会返回前数据, {}, {}".format(sub_quote_type, code)) final_rsp = await api.SubscribeBrokerSnapshotReqApi(exchange, code, start_time_stamp, sub_quote_type=sub_quote_type) assert self.common.searchDicKV(final_rsp["first_rsp_list"][0], 'retCode') == 'SUCCESS' assert final_rsp["before_broker_snapshot_json_list"] == [] # 查询指数成分股 self.logger.debug("查询港股指数成分股, {}, {}".format(sub_quote_type, code)) IndexShare = await api.QueryIndexShareMsgReqApi(isSubTrade=isSubTrade, exchange=exchange, sort_direct="DESCENDING_ORDER", indexCode="0000100", count=count, start_time_stamp=int(time.time() * 1000)) for indexData in self.common.searchDicKV(IndexShare, "snapshotData"): assert indexData.get("last") is None assert indexData.get("riseFall") is None # 按版块 self.logger.debug("查询港股版块信息, {}, {}".format(sub_quote_type, code)) for plate_type in ["MAIN", "LISTED_NEW_SHARES", "RED_SHIPS", "ETF", "GME"]: PlateSort = await api.QueryPlateSortMsgReqApi(isSubTrade=isSubTrade, zone="HK", plate_type=plate_type, sort_direct="DESCENDING_ORDER", count=count, start_time_stamp=int(time.time() * 1000)) if self.common.searchDicKV(PlateSort, "snapshotData"): for quoteData in self.common.searchDicKV(PlateSort, "snapshotData"): assert quoteData.get("last") is None assert quoteData.get("riseFall") is None if exchange in [ASE_exchange, NYSE_exchange, NASDAQ_exchange]: self.logger.debug("查询美股中概版和明星版, {}, {}".format(sub_quote_type, code)) for plate_type in ["START_STOCK", "CHINA_CONCEPT_STOCK"]: PlateSort = await api.QueryPlateSortMsgReqApi(isSubTrade=isSubTrade, zone="US", plate_type=plate_type, sort_direct="DESCENDING_ORDER", count=count, start_time_stamp=int(time.time() * 1000)) if self.common.searchDicKV(PlateSort, "snapshotData"): for quoteData in self.common.searchDicKV(PlateSort, "snapshotData"): assert quoteData.get("last") is None assert quoteData.get("riseFall") is None self.logger.debug("查询美股交易所排序--按涨跌排序, {}, {}".format(sub_quote_type, code)) ExchangeSort = await api.QueryExchangeSortMsgReqApi(isSubTrade=isSubTrade, exchange=exchange, sortFiled="R_F_RATIO", count=count, start_time_stamp=int(time.time() * 1000)) for ex_sort in self.common.searchDicKV(ExchangeSort, "snapshotData"): assert ex_sort.get("last") is None assert ex_sort.get("riseFall") is None finally: api.client.disconnect() # 验证状态改变后的推送通知 async def push_TradeStasut(self, exchange, code, loop): exchange = exchange product_list = [code] frequence = None start_time_stamp = int(time.time() * 1000) market_token = None asyncio.set_event_loop(loop) try: api = SubscribeApi(union_ws_url, loop, logger=self.logger) await api.client.ws_connect() await api.LoginReq(token=market_token, start_time_stamp=start_time_stamp, frequence=frequence) asyncio.run_coroutine_threadsafe(api.hearbeat_job(), loop) self.logger.debug(u'查询代码:{} 的交易状态, curtime : {}'.format(code, str(datetime.datetime.now()))) first_rsp_list = await api.QueryTradeStatusMsgReqApi(exchange=exchange, productList=product_list, recv_num=2) cur_status = self.common.searchDicKV(first_rsp_list, "status") self.logger.info("cur_status : {}".format(cur_status)) _start = time.time() PUSH_TRADE_STATUS = False while time.time() - _start < 120: # 循环120秒 rsp = await api.client.recv(recv_timeout_sec=5) if rsp: rev_data = QuoteMsgCarrier() rev_data.ParseFromString(rsp[0]) # self.logger.debug(rev_data) if rev_data.type == QuoteMsgType.PUSH_TRADE_STATUS: PUSH_TRADE_STATUS = True tradeStatus = TradeStatusData() tradeStatus.ParseFromString(rev_data.data) self.logger.info(tradeStatus) assert tradeStatus.status != cur_status # 确认校验状态只变化的一次 assert PUSH_TRADE_STATUS self.logger.debug("代码 : {} 有推送交易状态".format(code)) finally: api.client.disconnect() async def Liquidation(self, exchange, code): # loop = asyncio.new_event_loop() # loop.run_until_complete(future=self.CleanData(exchange, code, loop, "REAL_QUOTE_MSG")) loop = asyncio.get_event_loop() await self.CleanData(exchange, code, loop, "REAL_QUOTE_MSG") # 清盘-延时行情校验 async def Liquidation_DELAY(self, exchange, code): """ 延迟清盘 """ # loop = asyncio.new_event_loop() # loop.run_until_complete(future=self.CleanData(exchange, code, loop, "DELAY_QUOTE_MSG")) loop = asyncio.get_event_loop() await self.CleanData(exchange, code, loop, "DELAY_QUOTE_MSG") async def check_TradeStatus(self, exchange, code): loop = asyncio.get_event_loop() await self.push_TradeStasut(exchange, code, loop) # 定时任务回调 def Listener(self, event): # 监听器, 输出对应的错误信息 if event.exception: self.logger.error("{} 异常, 错误信息为 : \n{}".format(event.job_id, event.traceback)) # 创建清盘定时任务 def run_CleanData(self): self.logger.debug("runner 定时任务验证清盘") HK_stock = [ [SEHK_exchange, SEHK_code1], [SEHK_exchange, SEHK_indexCode1], [SEHK_exchange, SEHK_TrstCode1], [SEHK_exchange, SEHK_WarrantCode1], [SEHK_exchange, SEHK_CbbcCode1], [SEHK_exchange, SEHK_InnerCode1], ] US_stock = [ [ASE_exchange, ASE_code1], [NYSE_exchange, NYSE_code1], [NASDAQ_exchange, NASDAQ_code1], [BATS_exchange, BATS_code1], ] # 实时订阅清盘定时任务 [self.scheduler.add_job(self.Liquidation, 'cron', day_of_week='mon-fri', hour="08", minute="55-59", args=product, id='CleanData>>{}'.format('-'.join(product)), max_instances=self.max_instances) for product in HK_stock] [self.scheduler.add_job(self.Liquidation, 'cron', day_of_week='mon-fri', hour="22", minute="25-29", args=product, id='CleanData>>{}'.format('-'.join(product)), max_instances=self.max_instances) for product in US_stock] [self.scheduler.add_job(self.Liquidation_DELAY, 'cron', day_of_week='mon-fri', hour="08", minute="55-59", args=product, id='DELAY_CleanData>>{}'.format('-'.join(product)), max_instances=self.max_instances) for product in HK_stock] [self.scheduler.add_job(self.Liquidation_DELAY, 'cron', day_of_week='mon-fri', hour="22", minute="25-29", args=product, id='DELAYCleanData>>{}'.format('-'.join(product)), max_instances=self.max_instances) for product in US_stock] # 从exchangeTradeTime遍历, 添加定时任务 curDate = datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d") for key, value in exchangeTradeTime.items(): if key in ["HK_Stock", "US_Stock", "Grey"]: continue arg = key.split('_') arg[1] = arg[1] + "main" # print(arg) _time = datetime.datetime.strptime(curDate + value[0], '%Y-%m-%d%H:%M') # 开盘时间 if arg[0] not in ["HKFE", "SGX", "SEHK", "Grey"] and not isSummerTime: # 冬令时加一个小时 _time = datetime.datetime.strptime(curDate + value[0], '%Y-%m-%d%H:%M') + datetime.timedelta(hours=1) start_date = _time # copy一个变量 delay_endTime = _time start_date = start_date - datetime.timedelta(minutes=10) delay_endTime = delay_endTime + datetime.timedelta(minutes=15) # s_time = datetime.datetime.strftime(start_date, "%H%M") job_id = "CleanData_{}>>>start_date:{}, end_date:{}".format(arg, start_date, _time) # print(job_id) # self.scheduler.add_job(self.Liquidation, 'interval', minutes=1, start_date=start_date, end_date=_time, args=arg, id=job_id) self.scheduler.add_job(self.Liquidation, 'cron', day_of_week='mon-fri', hour=s_time[:-2], minute=s_time[-2:], args=arg, max_instances=self.max_instances, id=job_id) # 延时行情 delay_job_id = "DELAY_CleanData_{}>>>start_date:{}, end_date:{}".format(arg, start_date, delay_endTime) # print(delay_job_id) # self.scheduler.add_job(self.Liquidation_DELAY, 'interval', minutes=1, start_date=start_date, end_date=delay_endTime, args=arg, id=delay_job_id) self.scheduler.add_job(self.Liquidation_DELAY, 'cron', day_of_week='mon-fri', hour=s_time[:-2], minute=s_time[-2:], args=arg, max_instances=self.max_instances, id=delay_job_id) # 创建交易状态推送定时任务 def run_TradeStatus(self): self.logger.debug("交易交易状态推送通知") # 从exchangeTradeTime遍历, 添加定时任务 curDate = datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d") for key, value in exchangeTradeTime.items(): if key in ["HK_Stock", "US_Stock", "Grey"]: continue arg = key.split('_') # 每个时间段, 交易状态都会发生变化 for val in value: _time = datetime.datetime.strptime(curDate + val, '%Y-%m-%d%H:%M') if arg[0] not in ["HKFE", "SGX", "SEHK", "Grey"] and not isSummerTime: # 冬令时加一个小时 _time = datetime.datetime.strptime(curDate + val, '%Y-%m-%d%H:%M') + datetime.timedelta(hours=1) _time = _time - datetime.timedelta(seconds=30) s_time = datetime.datetime.strftime(_time, "%H%M") job_id = "push_status_{}>>>BeginTime:{}".format('-'.join(arg), s_time) # print(job_id) self.scheduler.add_job(self.check_TradeStatus, 'cron', hour=s_time[:-2], minute=s_time[-2:], second="00", args=arg, max_instances=self.max_instances, id=job_id) # 港股 self.scheduler.add_job(self.check_TradeStatus, 'cron', day_of_week='mon-fri', hour="09", minute="29", second="00", args=["SEHK", "00700"], max_instances=self.max_instances, id="SEHK_pushStatus_open") self.scheduler.add_job(self.check_TradeStatus, 'cron', day_of_week='mon-fri', hour="11-12,15", minute="59", second="00", args=["SEHK", "00700"], max_instances=self.max_instances, id="SEHK_pushStatus") # 美股 self.scheduler.add_job(self.check_TradeStatus, 'cron', day_of_week='mon-fri', hour="22", minute="29", second="00", args=[NASDAQ_exchange, NASDAQ_code1], max_instances=self.max_instances, id="NASDAQ_pushStatus_open") self.scheduler.add_job(self.check_TradeStatus, 'cron', day_of_week='mon-fri', hour="04", minute="59", second="00", args=[NASDAQ_exchange, NASDAQ_code1], max_instances=self.max_instances, id="NASDAQ_pushStatus") self.scheduler.add_job(self.check_TradeStatus, 'cron', day_of_week='sat', hour="04", minute="59", second="00", args=[ NASDAQ_exchange, NASDAQ_code1], max_instances=self.max_instances, id="NASDAQ_pushStatus_sat") # 定时任务运行入口 def run_Scheduler(self): self.run_CleanData() self.run_TradeStatus() # 更新合约 self.scheduler.add_job(start, 'cron', hour="08", minute="10", args=['SYNC_INSTR_REQ', codegenerate_dealer_address], id="dealer_instr") self.scheduler.add_listener(self.Listener, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR) self.scheduler.start() try: asyncio.get_event_loop().run_forever() except (KeyboardInterrupt, SystemExit): pass
from apscheduler.schedulers.asyncio import AsyncIOScheduler from loguru import logger from pytz import utc from sentry_sdk import capture_exception from telegram_bot.settings import redis_config jobstores = { 'default': RedisJobStore(host=redis_config.host, port=redis_config.port, password=redis_config.password) } job_defaults = {} def my_listener(event: JobExecutionEvent): if isinstance(event, JobExecutionEvent): logger.info(f"my_listener: {event.scheduled_run_time}") if event.exception: capture_exception(error=event.exception) logger.exception(str(event.exception), 'The job crashed :(') else: logger.info(f"my_listener: {event}") scheduler = AsyncIOScheduler(jobstores=jobstores, job_defaults=job_defaults, timezone=utc) scheduler.add_listener(my_listener, EVENT_ALL)
class AlamoScheduler(object): message_queue = None loop = handler = None def __init__(self, loop=None): kw = dict() if loop: kw['event_loop'] = loop self.scheduler = AsyncIOScheduler(**kw) def setup(self, loop=None): if loop is None: loop = asyncio.get_event_loop() asyncio.set_event_loop(loop) self.loop = loop self.message_queue = ZeroMQQueue( settings.ZERO_MQ_HOST, settings.ZERO_MQ_PORT ) self.message_queue.connect() self.scheduler.add_listener( self.event_listener, EVENT_JOB_ERROR | EVENT_JOB_MISSED | EVENT_JOB_MAX_INSTANCES ) @aiostats.increment() def _schedule_check(self, check): """Schedule check.""" logger.info( 'Check `%s:%s` scheduled!', check['uuid'], check['name'] ) check['scheduled_time'] = datetime.now(tz=pytz_utc).isoformat() self.message_queue.send(check) def remove_job(self, job_id): """Remove job.""" try: logger.info('Removing job for check id=`%s`', job_id) self.scheduler.remove_job(str(job_id)) except JobLookupError: pass def schedule_check(self, check): """Schedule check with proper interval based on `frequency`. :param dict check: Check definition """ try: frequency = check['fields']['frequency'] = int( check['fields']['frequency'] ) logger.info( 'Scheduling check `%s` with id `%s` and interval `%s`', check['name'], check['id'], frequency ) jitter = random.randint(0, frequency) first_run = datetime.now() + timedelta(seconds=jitter) kw = dict( seconds=frequency, id=str(check['uuid']), next_run_time=first_run, args=(check,) ) self.schedule_job(self._schedule_check, **kw) except KeyError as e: logger.exception('Failed to schedule check: %s. Exception: %s', check, e) def schedule_job(self, method, **kwargs): """Add new job to scheduler. :param method: reference to method that should be scheduled :param kwargs: additional kwargs passed to `add_job` method """ try: self.scheduler.add_job( method, 'interval', misfire_grace_time=settings.JOBS_MISFIRE_GRACE_TIME, max_instances=settings.JOBS_MAX_INSTANCES, coalesce=settings.JOBS_COALESCE, **kwargs ) except ConflictingIdError as e: logger.error(e) def event_listener(self, event): """React on events from scheduler. :param apscheduler.events.JobExecutionEvent event: job execution event """ if event.code == EVENT_JOB_MISSED: aiostats.increment.incr('job.missed') logger.warning("Job %s scheduler for %s missed.", event.job_id, event.scheduled_run_time) elif event.code == EVENT_JOB_ERROR: aiostats.increment.incr('job.error') logger.error("Job %s scheduled for %s failed. Exc: %s", event.job_id, event.scheduled_run_time, event.exception) elif event.code == EVENT_JOB_MAX_INSTANCES: aiostats.increment.incr('job.max_instances') logger.warning( 'Job `%s` could not be submitted. ' 'Maximum number of running instances was reached.', event.job_id ) @aiostats.increment() def get_jobs(self): return [job.id for job in self.scheduler.get_jobs()] async def checks(self, request=None): uuid = request.match_info.get('uuid', None) if uuid is None: jobs = self.get_jobs() return json_response(data=dict(count=len(jobs), results=jobs)) job = self.scheduler.get_job(uuid) if job is None: return json_response( data={'detail': 'Check does not exists.'}, status=404 ) check, = job.args return json_response(data=check) @aiostats.timer() async def update(self, request=None): check = await request.json() check_uuid = check.get('uuid') check_id = check.get('id') message = dict(status='ok') if not check_id or not check_uuid: return json_response(status=400) if check_id % settings.SCHEDULER_COUNT != settings.SCHEDULER_NR: return json_response(data=message, status=202) job = self.scheduler.get_job(str(check_uuid)) if job: scheduled_check, = job.args timestamp = scheduled_check.get('timestamp', 0) if timestamp > check['timestamp']: return json_response(data=message, status=202) message = dict(status='deleted') self.remove_job(check_uuid) if any([trigger['enabled'] for trigger in check['triggers']]): self.schedule_check(check) message = dict(status='scheduled') return json_response(data=message, status=202) def wait_and_kill(self, sig): logger.warning('Got `%s` signal. Preparing scheduler to exit ...', sig) self.scheduler.shutdown() self.loop.stop() def register_exit_signals(self): for sig in ['SIGQUIT', 'SIGINT', 'SIGTERM']: logger.info('Registering handler for `%s` signal ' 'in current event loop ...', sig) self.loop.add_signal_handler( getattr(signal, sig), self.wait_and_kill, sig ) def start(self, loop=None): """Start scheduler.""" self.setup(loop=loop) self.register_exit_signals() self.scheduler.start() logger.info( 'Press Ctrl+%s to exit.', 'Break' if os.name == 'nt' else 'C' ) try: self.loop.run_forever() except KeyboardInterrupt: pass logger.info('Scheduler was stopped!')
EVENT_JOB_SUBMITTED, JobSubmissionEvent) from apscheduler.executors.pool import ThreadPoolExecutor from apscheduler.jobstores.redis import RedisJobStore from apscheduler.schedulers.asyncio import AsyncIOScheduler from pytz import utc jobstores = {"default": RedisJobStore(host="127.0.0.1", port=6379)} executors = { "default": ThreadPoolExecutor(20), } job_defaults = {"max_instances": 4} scheduler = AsyncIOScheduler( executors=executors, job_defaults=job_defaults, timezone=utc ) submitted_jobs = {} def events_processor(event): print(f">>>>> {event.__class__.__name__} <<<<<") if isinstance(event, JobSubmissionEvent): if event.code == EVENT_JOB_SUBMITTED: print(f"{event.job_id} - Submitted") submitted_jobs[event.job_id] = event elif event.code == EVENT_JOB_MAX_INSTANCES: print(f"{event.job_id} - Denied") scheduler.add_listener(events_processor, EVENT_ALL)
2**3: "EVENT_SCHEDULER_RESUMED", 2**4: "EVENT_EXECUTOR_ADDED", 2**5: "EVENT_EXECUTOR_REMOVED", 2**6: "EVENT_JOBSTORE_ADDED", 2**7: "EVENT_JOBSTORE_REMOVED", 2**8: "EVENT_ALL_JOBS_REMOVED", 2**9: "EVENT_JOB_ADDED", 2**10: "EVENT_JOB_REMOVED", 2**11: "EVENT_JOB_MODIFIED", 2**12: "EVENT_JOB_EXECUTED", 2**13: "EVENT_JOB_ERROR", 2**14: "EVENT_JOB_MISSED", 2**15: "EVENT_JOB_SUBMITTED", 2**16: "EVENT_JOB_MAX_INSTANCES", } def log_event(event): s = ["Scheduler event:"] for method in dir(event): if method.startswith("_") or method == "traceback": continue if method == "code": s.append(f"code={EVENTS[event.code]}") else: s.append(f"{method}={getattr(event, method)}") print(" ".join(s)) scheduler.add_listener(log_event, EVENT_ALL)
class TaskManager(AsyncRunnable): _scheduler: AsyncIOScheduler def __init__(self): self._scheduler = AsyncIOScheduler() def add_interval(self, weeks=0, days=0, hours=0, minutes=0, seconds=0, start_date=None, end_date=None, timezone=None, jitter=None): """decorator, add a interval type task""" trigger = IntervalTrigger(weeks=weeks, days=days, hours=hours, minutes=minutes, seconds=seconds, start_date=start_date, end_date=end_date, timezone=timezone, jitter=jitter) return lambda func: self._scheduler.add_job(func, trigger) def add_cron(self, year=None, month=None, day=None, week=None, day_of_week=None, hour=None, minute=None, second=None, start_date=None, end_date=None, timezone=None, jitter=None): """decorator, add a cron type task""" trigger = CronTrigger(year=year, month=month, day=day, week=week, day_of_week=day_of_week, hour=hour, minute=minute, second=second, start_date=start_date, end_date=end_date, timezone=timezone, jitter=jitter) return lambda func: self._scheduler.add_job(func, trigger) def add_date(self, run_date=None, timezone=None): """decorator, add a date type task""" trigger = DateTrigger(run_date=run_date, timezone=timezone) return lambda func: self._scheduler.add_job(func, trigger) async def start(self): self._scheduler.configure({'event_loop': self.loop}, '') self._scheduler.add_listener(lambda e: log.exception(f'error raised during task', exc_info=e.exception), EVENT_JOB_ERROR) self._scheduler.start() # reminder: this is not blocking
class Gallery(commands.Cog): """Handle the Gallery channels.""" config = None delete_after = 15 compare = lambda x, y: collections.Counter(x) == collections.Counter(y) def __init__(self, bot): Gallery.bot = self self.bot = bot self.db = None #Gallery.config = Config() self.gal_guild_id= 0 self.gal_enable= False self.gal_channel_ids= [] self.gal_channels= [] self.gal_text_expirein= None self.gal_user_wl= [] self.gal_allow_links= False self.gal_link_wl= [] self.jobstore = SQLAlchemyJobStore(url='sqlite:///gallery.sqlite') jobstores = {"default": self.jobstore} self.scheduler = AsyncIOScheduler(jobstores=jobstores) self.scheduler.add_listener(self.job_missed, events.EVENT_JOB_MISSED) #-------------------- LOCAL COG EVENTS -------------------- async def cog_before_invoke(self, ctx): '''THIS IS CALLED BEFORE EVERY COG COMMAND, IT'S SOLE PURPOSE IS TO CONNECT TO THE DATABASE''' credentials = {"user": dblogin.user, "password": dblogin.pwrd, "database": dblogin.name, "host": dblogin.host} self.db = await asyncpg.create_pool(**credentials) return async def cog_after_invoke(self, ctx): '''THIS IS CALLED AFTER EVERY COG COMMAND, IT DISCONNECTS FROM THE DATABASE AND DELETES INVOKING MESSAGE IF SET TO.''' await self.db.close() if Gallery.config.delete_invoking: await ctx.message.delete() return async def on_cog_command_error(self, ctx, error): if isinstance(error, discord.ext.commands.errors.NotOwner): try: owner = (self.bot.application_info()).owner except: owner = self.bot.get_guild(self.gal_guild_id).owner() await ctx.channel.send(content=f"```diff\n- {ctx.prefix}{ctx.invoked_with} is an owner only command, this will be reported to {owner.name}.") await owner.send(content=f"{ctx.author.mention} tried to use the owner only command{ctx.invoked_with}") return #-------------------- STATIC METHODS -------------------- @staticmethod async def get_channel_id(content): try: args= content.split(" ") if len(args) > 2: return False #=== SPLIT, REMOVE MENTION WRAPPER AND CONVERT TO INT ch_id = args[1] ch_id = ch_id.replace("<", "").replace("#", "").replace(">", "") ch_id = int(ch_id) return ch_id except (IndexError, ValueError): return False @staticmethod async def get_user_id(content): try: args= content.split(" ") if len(args) > 2: return False #=== SPLIT, REMOVE MENTION WRAPPER AND CONVERT TO INT user_id = args[1] user_id = user_id.replace("<", "").replace("@", "").replace("!", "").replace(">", "") user_id = int(user_id) return user_id except (IndexError, ValueError): return False @staticmethod def time_pat_to_hrs(content): ''' Converts a string in format <xdxh> (d standing for days and h standing for hours) to amount of hours. eg: 3d5h would be 77 hours Args: (str) or (int) Returns: (int) or (None) ''' args= content.split(" ") if len(args) > 2: return False t = args[1] timeinHours = int() try: timeinHours = int(t) return timeinHours except ValueError: valid = False #===== if input doesn't match basic pattern if (re.match(r"(\d+[DHdh])+", t)): #=== if all acsii chars in the string are unique letters = re.findall(r"[DHdh]", t) if len(letters) == len(set(letters)): #= if more then 1 letter side by side #= ie. if t was 2dh30m then after the split you'd have ['', 'dh', 'm', ''] if not ([i for i in re.split(r"[0-9]", t) if len(i) > 1]): # if letters are in order. if letters == sorted(letters, key=lambda letters: ["d", "h"].index(letters[0])): valid = True if valid: total_hours = int() for data in re.findall(r"(\d+[DHdh])", t): if data.endswith("d"): total_hours += int(data[:-1])*24 if data.endswith("h"): total_hours += int(data[:-1]) return total_hours return False @staticmethod async def split_list(arr, size=100): """Custom function to break a list or string into an array of a certain size""" arrs = [] while len(arr) > size: pice = arr[:size] arrs.append(pice) arr = arr[size:] arrs.append(arr) return arrs @staticmethod async def oneline_valid(content): try: args = content.split(" ") if len(args) > 1: return False return True except (IndexError, ValueError): return False #-------------------- LISTENERS -------------------- @commands.Cog.listener() async def on_ready(self): self.cogset = await cogset.LOAD(cogname=self.qualified_name) if not self.cogset: self.cogset= dict( enablelogging=False ) await cogset.SAVE(self.cogset, cogname=self.qualified_name) #@commands.Cog.listener() async def on_ready_old(self): credentials = {"user": dblogin.user, "password": dblogin.pwrd, "database": dblogin.name, "host": dblogin.host} self.db = await asyncpg.create_pool(**credentials) dbconfig = await self.db.fetchrow(pgCmds.GET_GUILD_GALL_CONFIG) await self.db.close() self.gal_guild_id= dbconfig['guild_id'] self.gal_enable= dbconfig['gall_nbl'] self.gal_channel_ids= dbconfig['gall_ch'] guild = self.bot.get_guild(self.gal_guild_id) self.gal_channels= [channel for channel in guild.channels if channel.id in dbconfig['gall_ch']] self.gal_text_expirein= dbconfig['gall_text_exp'] self.gal_user_wl= dbconfig['gall_user_wl'] self.gal_allow_links= dbconfig['gall_nbl_links'] self.gal_link_wl= dbconfig['gall_links'] ###===== SCHEDULER self.scheduler.start() @commands.Cog.listener() async def on_message(self, msg): ###===== RETURN IF GALLERYS ARE DISABLED if not self.gal_enable: return ###===== RETURN IF MESSAGE IS NOT FROM A GUILD if not msg.guild: return ###===== RETURN IF MESSAGE TYPE IS ANYTHING OTHER THAN A NORMAL MESSAGE. if not bool(msg.type == discord.MessageType.default): return if msg.channel in self.gal_channels: ###=== IF AUTHOR IS ALLOWED TO POST MESSAGES FREELY IN GALLERY CHANNELS if msg.author.id in self.gal_user_wl: return valid = False ###=== IF MESSAGE HAS ATTACHMENTS ASSUME THE MESSAGE IS OF ART. if msg.attachments: valid = True ###=== IF LINKS ARE ALLOWED IN GALLERY CHANNELS if self.gal_allow_links: #- get the links from msg content links = re.findall(r"(?P<url>http[s]?://[^\s]+)", msg.content) ###= IF ONLY CERTAIN LINKS ARE ALLOWED if self.gal_link_wl: #= LOOP THROUGH THE LINKS FROM THE MESSAGE CONTENT AND THE WHITELISTED LINKS #= ASSUME VALID IF ONE LINK MATCHES. for link in links: for wl_link in self.gal_link_wl: if link.startswith(wl_link): valid = True break else: valid = True ###=== IF THE MESSAGE IS NOT VALID. if not valid: credentials = {"user": dblogin.user, "password": dblogin.pwrd, "database": dblogin.name, "host": dblogin.host} self.db = await asyncpg.create_pool(**credentials) self.db.execute(pgCmds.ADD_GALL_MSG, msg.id, msg.channel.id, msg.guild.id, msg.author.id, msg.created_at) await self.db.close() #regex = r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?" #urls = re.findall( regex, text ) #re.findall("(?P<url>http[s]?://[^\s]+)", t) #re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\), ]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', t) #credentials = {"user": dblogin.user, "password": dblogin.pwrd, "database": dblogin.name, "host": dblogin.host} #self.db = await asyncpg.create_pool(**credentials) #-------------------- COMMANDS -------------------- @commands.is_owner() @commands.command(pass_context=True, hidden=True, name='galenable', aliases=[]) async def cmd_galenable(self, ctx): """ [Bot Owner] Enables the gallery feature. Useage: [prefix]galenable """ ###===== Write to database await self.db.execute(pgCmds.SET_GUILD_GALL_ENABLE, self.gal_guild_id) ###===== SET LOCAL COG VARIABLE self.gal_enable= True ###===== DELETE THE JOB IF IT EXISTS for job in self.jobstore.get_all_jobs(): if ["_delete_gallery_messages"] == job.id.split(" "): self.scheduler.remove_job(job.id) ###===== ADD THE FUNCTION TO THE SCHEDULER self.scheduler.add_job(call_schedule, 'date', id="_delete_gallery_messages", run_date=get_next(hours=self.gal_text_expirein), kwargs={"func": "_delete_gallery_messages"} ) await ctx.channel.send(content="Galleries are disabled.") return @commands.is_owner() @commands.command(pass_context=True, hidden=True, name='galdisable', aliases=[]) async def cmd_galdisable(self, ctx): """ [Bot Owner] Disables the gallery feature. Useage: [prefix]galdisable """ ###===== Write to database await self.db.execute(pgCmds.SET_GUILD_GALL_DISABLE, self.gal_guild_id) ###===== SET LOCAL COG VARIABLE self.gal_enable= False ###===== DELETE THE JOB IF IT EXISTS for job in self.jobstore.get_all_jobs(): if ["_delete_gallery_messages"] == job.id.split(" "): self.scheduler.remove_job(job.id) await ctx.channel.send(content="Galleries are disabled.") return @commands.is_owner() @commands.command(pass_context=True, hidden=True, name='enablegalleries', aliases=[]) async def cmd_galaddchannel(self, ctx): """ [Bot Owner] Add a channel to the list of active gallery channels Useage: [prefix]galaddchannel <channelid/mention> """ ###===== VALIDATE INPUT ch_id = await Gallery.get_channel_id(ctx.message.content) if not ch_id: ctx.channel.send(content="`Useage: [p]galaddchannel <channelid/mention>, [Bot Owner] Add a channel to the list of active gallery channels.`", delete_after=Gallery.delete_after) ###===== ADD NEW CHANNEL ID TO LIST new_channel_ids = list(set(self.gal_channel_ids) + {ch_id}) if Gallery.compare(self.gal_channel_ids, new_channel_ids): await ctx.channel.send(content=f"<#{ch_id}> is already a gallery channel.") return else: self.gal_channel_ids = new_channel_ids ###===== GET THE ACTUAL CHANNEL FROM THE GUILD if ctx.guild: guild = ctx.guild else: guild = self.bot.get_guild(self.gal_guild_id) self.gal_channels = [channel for channel in guild.channels if channel.id in self.gal_channel_ids] ###===== WRITE DATA TO DATABASE await self.db.execute(pgCmds.SET_GUILD_GALL_CHLS, self.gal_channel_ids) ###===== END await ctx.channel.send(content=f"<#{ch_id}> has been made a gallery channel.") return @commands.is_owner() @commands.command(pass_context=True, hidden=True, name='galremchannel', aliases=[]) async def cmd_galremchannel(self, ctx): """ [Bot Owner] Removes a channel to the list of active gallery channels Useage: [prefix]galremchannel <channelid/mention> """ ch_id = await Gallery.get_channel_id(ctx.message.content) if not ch_id: ctx.channel.send(content="`Useage: [p]galremchannel <channelid/mention>, [Bot Owner] Removes a channel to the list of active gallery channels.`", delete_after=Gallery.delete_after) ###===== REMOVE CHANNEL ID FROM LIST try: self.gal_channel_ids.remove(ch_id) except ValueError: await ctx.channel.send(content=f"<#{ch_id}> isn't a gallery channel.") return ###===== GET THE ACTUAL CHANNEL FROM THE GUILD if ctx.guild: guild = ctx.guild else: guild = self.bot.get_guild(self.gal_guild_id) self.gal_channels = [channel for channel in guild.channels if channel.id in self.gal_channel_ids] ###===== WRITE DATA TO DATABASE await self.db.execute(pgCmds.SET_GUILD_GALL_CHLS, self.gal_channel_ids) await self.db.execute(pgCmds.DEL_GALL_MSGS_FROM_CH, ch_id, self.gal_guild_id) ###===== END await ctx.channel.send(content=f"<#{ch_id}> is no longer a gallery channel.") return @commands.is_owner() @commands.command(pass_context=True, hidden=True, name='galsetexpirehours', aliases=[]) async def cmd_galsetexpirehours(self, ctx): """ [Bot Owner] Sets how long the bot should wait to delete text only messages from gallery channels Useage: [prefix]galsetexpirehours <hours> """ new_time = Gallery.time_pat_to_hrs(ctx.message.content) await self.db.execute(pgCmds.SET_GUILD_GALL_EXP, new_time) resetJob = False ###===== RESET THE JOB IF IT EXISTS for job in self.jobstore.get_all_jobs(): if ["_delete_gallery_messages"] == job.id.split(" "): self.scheduler.remove_job(job.id) resetJob = True if resetJob: ###===== ADD THE FUNCTION TO THE SCHEDULER self.scheduler.add_job(call_schedule, 'date', id="_delete_gallery_messages", run_date=get_next(hours=self.gal_text_expirein), kwargs={"func": "_delete_gallery_messages"} ) await ctx.channel.send(content=f"Text message expirey time has been set to {new_time} hours and the scheduler was reset.") return await ctx.channel.send(content=f"Text message expirey time has been set to {new_time} hours.") return @commands.is_owner() @commands.command(pass_context=True, hidden=True, name='galadduserwl', aliases=[]) async def cmd_galadduserwl(self, ctx): """ [Bot Owner] Adds a user to the gallery user whitelist. Allowing them to post in Gallery channels. Useage: [prefix]galadduserwl <userid/mention> """ ###===== CHECK IF INPUT IS VALID user_id = Gallery.get_user_id(ctx.message.content) if not user_id: return ###===== ADD USER ID TO THE WHITELIST new_user_whitelist = list(set(self.gal_user_wl) + {user_id}) if Gallery.compare(self.gal_user_wl, new_user_whitelist): await ctx.channel.send(content=f"<@{user_id}> is alreadt in the gallery whitelist.", delete_after=Gallery.delete_after) return else: self.gal_user_wl = new_user_whitelist ###===== WRITE TO THE DATABASE await self.db.execute(pgCmds.SET_GUILD_GALL_USER_WL, self.gal_user_wl, self.gal_guild_id) ###===== RETURN await ctx.channel.send(content=f"<@{user_id}> has been added to the gallery whitelist.", delete_after=Gallery.delete_after) return @commands.is_owner() @commands.command(pass_context=True, hidden=True, name='galremuserwl', aliases=[]) async def cmd_galremuserwl(self, ctx): """ [Bot Owner] Remomes a user to the gallery user whitelist. Useage: [prefix]galremuserwl <userid/mention> """ ###===== CHECK IF INPUT IS VALID user_id = Gallery.get_user_id(ctx.message.content) if not user_id: return ###===== REMOVE USER FROM WHITELIST try: self.gal_user_wl.remove(user_id) except ValueError: #=== IF USER IS NOT ON THE WHITELIST await ctx.channel.send(content=f"<@{user_id}> was not on the gallery whitelist.", delete_after=Gallery.delete_after) return ###===== WRITE TO DATABASE await self.db.execute(pgCmds.SET_GUILD_GALL_USER_WL, self.gal_user_wl, self.gal_guild_id) ###===== RETURN await ctx.channel.send(content=f"<@{user_id}> has been removed from the gallery whitelist.", delete_after=Gallery.delete_after) return ### ENABLE LINKS @commands.is_owner() @commands.command(pass_context=True, hidden=True, name='galenablelinks', aliases=[]) async def cmd_galenablelinks(self, ctx): """ [Bot Owner] Allow links in the gallery channels. Useage: [prefix]galenablelinks <channelid/mention> """ valid = Gallery.oneline_valid(ctx.message.content) if not valid: return self.gal_allow_links=True ###===== WRITE TO THE DATABASE await self.db.execute(pgCmds.SET_GUILD_GALL_LINK_ENABLE) ###===== RETURN await ctx.channel.send(content="Links are now allowed in the gallery channels.", delete_after=Gallery.delete_after) return ### BLOCK LINKS @commands.is_owner() @commands.command(pass_context=True, hidden=True, name='galdisablelinks', aliases=[]) async def cmd_galdisablelinks(self, ctx): """ [Bot Owner] Block links in the gallery channels. Useage: [prefix]galdisablelinks <channelid/mention> """ valid = Gallery.oneline_valid(ctx.message.content) if not valid: return self.gal_allow_links=False ###===== WRITE TO THE DATABASE await self.db.execute(pgCmds.SET_GUILD_GALL_LINK_DISABLE) ###===== RETURN await ctx.channel.send(content="Links are no longer allowed in the gallery channels.", delete_after=Gallery.delete_after) return ### ADD LINK WHITELIST @commands.is_owner() @commands.command(pass_context=True, hidden=True, name='galaddlinkuwl', aliases=[]) async def cmd_galaddlinkuwl(self, ctx): """ [Bot Owner] Adds a link from gallery link whitelist. Useage: [prefix]galaddlinkuwl <startoflink> """ links = re.findall(r"(?P<url>http[s]?://[^\s]+)", ctx.message.content) if not links: await ctx.channel.send('`Useage: [p]galaddlinkuwl <startoflink>, [Bot Owner] Adds a link from gallery link whitelist.`') ###===== ADD THE NEW LINKS TO THE WHITELIST new_gal_link_wl = list(set(self.gal_link_wl) + set(links)) if Gallery.compare(new_gal_link_wl, self.gal_link_wl): await ctx.channel.send(content="{}\n are already in the gallery link whitelist.".format('\n'.join(links)), delete_after=Gallery.delete_after) return else: self.gal_link_wl = new_gal_link_wl ###===== WRITE TO THE DATABASE await self.db.execute(pgCmds.SET_GUILD_GALL_LINKS, self.gal_link_wl, self.gal_guild_id) ###===== RETURN await ctx.channel.send(content="{}\n have been added to the gallery link whitelist.".format('\n'.join(links)), delete_after=Gallery.delete_after) return ### REM LINK WHITELIST @commands.is_owner() @commands.command(pass_context=True, hidden=True, name='galremlinkuwl', aliases=[]) async def cmd_galremlinkuwl(self, ctx): """ [Bot Owner] Removes a link from gallery link whitelist. Useage: [prefix]galremlinkuwl <startoflink> """ links = re.findall(r"(?P<url>http[s]?://[^\s]+)", ctx.message.content) if not links: await ctx.channel.send('Useage: [p]galremlinkuwl <startoflink>, [Bot Owner] Removes a link from gallery link whitelist.') ###===== REMOVE THE LINKS FROM THE LIST new_gal_link_wl = list(set(self.gal_link_wl) - set(links)) if Gallery.compare(new_gal_link_wl, self.gal_link_wl): await ctx.channel.send(content="{}\n are not in the gallery link whitelist.".format('\n'.join(links)), delete_after=Gallery.delete_after) return else: self.gal_link_wl = new_gal_link_wl ###===== WRITE TO THE DATABASE await self.db.execute(pgCmds.SET_GUILD_GALL_LINKS, self.gal_link_wl, self.gal_guild_id) ###===== RETURN await ctx.channel.send(content="{}\n have been removed from the gallery link whitelist.".format('\n'.join(links)), delete_after=Gallery.delete_after) return ### SPECIAL @commands.is_owner() @commands.command(pass_context=True, hidden=True, name='galloadsettings', aliases=[]) async def cmd_galloadsettings(self, ctx): ###===== OPEN THE SETUP.INI FILE config = Config() ###===== WRITE DATA FROM THE SETUP.INI FILE TO THE DATABASE await self.db.execute(pgCmds.SET_GUILD_GALL_CONFIG, config.galEnable, config.gallerys["chls"], config.gallerys['expire_in'], config.gallerys["user_wl"], config.gallerys["links"], config.gallerys['link_wl']) ###===== UPDATE THE SETTINGS IN THE LOCAL COG self.gal_enable= config.galEnable guild = self.bot.get_guild(self.gal_guild_id) self.gal_channels= [channel for channel in guild.channels if channel.id in config.gallerys["chls"]] self.gal_text_expirein= config.gallerys['expire_in'] self.gal_user_wl= config.gallerys["user_wl"] self.gal_allow_links= config.gallerys["links"] self.gal_link_wl= config.gallerys['link_wl'] ###===== RETURN await ctx.channel.send(content="Gallery information has been updated from the setup.ini file", delete_after=15) return #-------------------- SCHEDULING -------------------- def job_missed(self, event): """ This exists too """ asyncio.ensure_future(call_schedule(*event.job_id.split(" "))) @staticmethod def get_id_args(func, arg): """ I have no damn idea what this does """ return "{} {}".format(func.__name__, arg) @commands.is_owner() @commands.command(pass_context=True, hidden=True, name='galinitiateschedule', aliases=[]) async def cmd_galinitiateschedule(self, ctx): ###===== DELETE THE JOB IF IT ALREADY EXISTS for job in self.jobstore.get_all_jobs(): if ["_delete_gallery_messages"] == job.id.split(" "): self.scheduler.remove_job(job.id) ###===== ADD THE FUNCTION TO THE SCHEDULER self.scheduler.add_job(call_schedule, 'date', id="_delete_gallery_messages", run_date=get_next(hours=self.gal_text_expirein), kwargs={"func": "_delete_gallery_messages"} ) ###===== RETURN ctx.channel.send(content=f"Gallery schedule has been set for {get_next(hours=self.gal_text_expirein)}") return async def _delete_gallery_messages(self): ###===== QUIT ID GALLERIES ARE DISABLED. if not self.gal_enable: return ###===== CONNECT TO THE DATABASE credentials = {"user": dblogin.user, "password": dblogin.pwrd, "database": dblogin.name, "host": dblogin.host} self.db = await asyncpg.create_pool(**credentials) after = datetime.datetime.utcnow() - datetime.timedelta(hours=self.gal_text_expirein) t = await self.db.fetch(pgCmds.GET_GALL_MSG_AFTER, after) ch_ids = await self.db.fetch(pgCmds.GET_GALL_CHIDS_AFTER, after) await self.db.close() ###===== TURNING THE DATA INTO SOMETHING MORE USEFUL ch_ids = [ch_id['ch_id'] for ch_id in ch_ids] fast_delete = dict() slow_delete = [] for ch_id in ch_ids: fast_delete[ch_id] = [] now = datetime.datetime.utcnow() delta = datetime.timedelta(days=13, hours=12) for record in t: ###=== IF MESSAGE IS OLDER THAN 13 DAYS AND 12 HOURS if bool((now - record['timestamp']) > delta): slow_delete.append(record) ###=== IF MESSAHE IS YOUNGER THAN 13 DAYS AND 12 HOURS else: fast_delete[record['ch_id']].append(record['msg_id']) ###===== IF THERE IS FAST DELETE DATA # WITH FAST DELETE MESSAGES WE CAN DELETE MESSAGES IN BULK OF 100 if fast_delete: for ch_id in fast_delete.keys(): msgs_ids = Gallery.split_list(fast_delete[ch_id], 100) for msg_ids in msgs_ids: if len(msg_ids) > 1: await self.bot.http.delete_messages(ch_id, msg_ids, reason="Deleting Gallery Messages") await asyncio.sleep(0.5) else: msg_id = msg_ids[0] await self.bot.http.delete_message(ch_id, msg_id, reason="Deleting Gallery Messages") await asyncio.sleep(0.5) ###===== IF THERE IS SLOW DELETE DATA # WE CANNOT DELETE THESE MESSAGES IN BULK, ONLY ONE BY ONE. if slow_delete: for record in slow_delete: await self.bot.http.delete_message(record['ch_id'], record['msg_id'], reason="Deleting Gallery Messages") await asyncio.sleep(0.5) ###==== LOOP THE SCHEDULER self.scheduler.add_job( call_schedule, 'date', id="_delete_gallery_messages", run_date=get_next(hours=self.gal_text_expirein), kwargs={"func": "_delete_gallery_messages"} ) return