async def send_welcome(self, member): guild = self.bot.get_guild(Configuration.get_var("guild_id")) if member.guild.id != guild.id or self.is_member_verified(member): return False try: welcome_channel = self.bot.get_config_channel( guild.id, Utils.welcome_channel) rules_channel = self.bot.get_config_channel( guild.id, Utils.rules_channel) # Send welcome message in configured language. default to english if welcome_channel and rules_channel: txt = Lang.get_string( "welcome/welcome_msg", user=member.mention, rules_channel=rules_channel.mention, accept_emoji=Emoji.get_chat_emoji('PAINTBRUSH')) if self.mute_new_members: # add mute notification if mute for new members is on mute_txt = Lang.get_string("welcome/welcome_mute_msg") txt = f"{txt}\n{mute_txt}" await welcome_channel.send(txt) return True except Exception as ex: Logging.info(f"failed to welcome {member.id}") Logging.error(ex) raise ex return False
async def startup_cleanup(self): Logging.info("starting bugs") # TODO: find out what the condition is we need to wait for instead of just sleep # await asyncio.sleep(20) # for name, cid in Configuration.get_var("channels").items(): reporting_channel_ids = [] for row in BugReportingChannel.select(): cid = row.channelid name = f"{row.platform.platform}_{row.platform.branch}" guild_id = row.guild.serverid channel = self.bot.get_channel(cid) shutdown_key = f"{guild_id}_{name}_shutdown" shutdown_id = Configuration.get_persistent_var(shutdown_key) if shutdown_id is not None: Configuration.del_persistent_var(shutdown_key) try: message = await channel.fetch_message(shutdown_id) await message.delete() except (NotFound, HTTPException) as e: pass reporting_channel_ids.append(cid) try: await self.send_bug_info(*reporting_channel_ids) except Exception as e: await Utils.handle_exception("bug startup failure", self.bot, e)
async def handle_reaction_change(self, t, reaction, user_id): roles = Configuration.get_var("roles") if reaction in roles: guild = self.bot.get_guild(Configuration.get_var("guild_id")) role = guild.get_role(roles[reaction]) member_role = guild.get_role(Configuration.get_var("member_role")) nonmember_role = guild.get_role( Configuration.get_var("nonmember_role")) member = guild.get_member(user_id) if member is None: return action = getattr(member, f"{t}_roles") try: await action(role) # if acting on member role, toggle corresponding nonmember role if role is member_role: if t == 'add': await member.remove_roles(nonmember_role) else: await member.add_roles(nonmember_role) except Exception as ex: Logging.info("failed") Logging.error(ex) raise ex
async def ping_unverified(self, ctx): guild = self.bot.get_guild(Configuration.get_var("guild_id")) try: nonmember_role = guild.get_role( Configuration.get_var("nonmember_role")) welcome_channel = self.bot.get_config_channel( guild.id, Utils.welcome_channel) rules_channel = self.bot.get_config_channel( guild.id, Utils.rules_channel) if welcome_channel and rules_channel: txt = Lang.get_string( "welcome/welcome_msg", user=nonmember_role.mention, rules_channel=rules_channel.mention, accept_emoji=Emoji.get_chat_emoji('CANDLE')) await nonmember_role.edit(mentionable=True) await welcome_channel.send(txt) await nonmember_role.edit(mentionable=False) return True except Exception as ex: Logging.info(f"failed to welcome unverified role.") Logging.error(ex) raise ex return False
async def periodic_task(self): # periodic task to run while cog is loaded # remove expired cooldowns now = datetime.now().timestamp() cooldown = Configuration.get_persistent_var(f"mischief_cooldown", dict()) try: # key for loaded dict is a string updated_cooldown = {} for str_uid, member_last_access_time in cooldown.items(): if (now - member_last_access_time) < self.cooldown_time: updated_cooldown[str_uid] = member_last_access_time Configuration.set_persistent_var(f"mischief_cooldown", updated_cooldown) except: Logging.info("can't clear cooldown") # update role count storage (because it's slow) try: guild = Utils.get_home_guild() for role_id in self.role_map.values(): my_role = guild.get_role(role_id) if my_role is not None: self.role_counts[str(role_id)] = len(my_role.members) except: Logging.info("can't update role counts")
def list(path): ''' List existing repository. ''' os.chdir(path) for name in os.listdir(path): if check_repo_validity(name): repo_url = wrap_repo_url(path, name) name = name.replace(REPO_POSTFIX, '') Logging.info('%s %80s' % (name, repo_url))
async def load(self, ctx, cog: str): if os.path.isfile(f"cogs/{cog}.py"): self.bot.load_extension(f"cogs.{cog}") if cog not in Configuration.MASTER_CONFIG["cogs"]: Configuration.MASTER_CONFIG["cogs"].append(cog) Configuration.save() await ctx.send(f"**{cog}** has been loaded!") await Logging.bot_log( f"**{cog}** has been loaded by {ctx.author.name}.") Logging.info(f"{cog} has been loaded") else: await ctx.send( f"{Emoji.get_chat_emoji('NO')} I can't find that cog.")
async def unload(self, ctx, cog: str): if cog in ctx.bot.cogs: self.bot.unload_extension(f"cogs.{cog}") if cog in Configuration.MASTER_CONFIG["cogs"]: Configuration.get_var("cogs").remove(cog) Configuration.save() await ctx.send(f'**{cog}** has been unloaded.') await Logging.bot_log( f'**{cog}** has been unloaded by {ctx.author.name}') Logging.info(f"{cog} has been unloaded") else: await ctx.send( f"{Emoji.get_chat_emoji('NO')} I can't find that cog.")
async def startup_cleanup(self): await self.bot.wait_until_ready() Logging.info("starting DropBox") for guild in self.bot.guilds: # fetch dropbox channels per server self.init_guild(guild.id) for row in DropboxChannel.select().where( DropboxChannel.serverid == guild.id): self.dropboxes[guild.id][row.sourcechannelid] = row self.loaded = True self.deliver_to_channel.start() self.clean_channels.start()
async def on_ready(self): if not self.loaded: Logging.BOT_LOG_CHANNEL = self.get_channel( Configuration.get_var("log_channel")) Emoji.initialize(self) for cog in Configuration.get_var("cogs"): try: self.load_extension("cogs." + cog) except Exception as e: await Utils.handle_exception(f"Failed to load cog {cog}", self, e) Logging.info("Cogs loaded") self.loop.create_task(self.keepDBalive()) self.loaded = True await Logging.bot_log("Journey bot surfing through the dunes!")
def init_root_repos(path): grm_path = os.path.join(path, BASE_GRM_DIR) if os.path.exists(grm_path): Logging.error('Reinitialized existing remote root directory in %s' % os.path.abspath(path)) return False if not os.path.exists(path): os.mkdir(path) Logging.info('Initialized remote root directory in %s' % os.path.abspath(path)) os.chdir(path) os.mkdir(BASE_GRM_DIR) init_config('.') p = Process(target=init_public_ip) p.start()
async def on_ready(self): if self.loaded: Logging.info(f"{self.my_name} reconnect") return Logging.BOT_LOG_CHANNEL = self.get_channel( Configuration.get_var("log_channel")) Emoji.initialize(self) for cog in Configuration.get_var("cogs"): try: self.load_extension("cogs." + cog) except Exception as e: await Utils.handle_exception(f"Failed to load cog {cog}", self, e) Logging.info("Cogs loaded") self.db_keepalive = self.loop.create_task(self.keepDBalive()) self.loaded = True await Logging.bot_log(f"{self.my_name} has started. Time to bot!")
async def send_bug_info(self, *args): for channel_id in args: channel = self.bot.get_channel(channel_id) if channel is None: await Logging.bot_log(f"can't send bug info to nonexistent channel {channel_id}") continue bug_info_id = Configuration.get_persistent_var(f"{channel.guild.id}_{channel_id}_bug_message") ctx = None tries = 0 while not ctx and tries < 5: tries += 1 # this API call fails on startup because connection is not made yet. # TODO: properly wait for connection to be initialized try: last_message = await channel.send('preparing bug reporting...') ctx = await self.bot.get_context(last_message) if bug_info_id is not None: try: message = await channel.fetch_message(bug_info_id) except (NotFound, HTTPException): pass else: await message.delete() if message.id in self.bug_messages: self.bug_messages.remove(message.id) bugemoji = Emoji.get_emoji('BUG') message = await channel.send(Lang.get_locale_string("bugs/bug_info", ctx, bug_emoji=bugemoji)) self.bug_messages.add(message.id) await message.add_reaction(bugemoji) Configuration.set_persistent_var(f"{channel.guild.id}_{channel_id}_bug_message", message.id) Logging.info(f"Bug report message sent in channel #{channel.name} ({channel.id})") await last_message.delete() except Exception as e: await self.bot.guild_log(channel.guild.id, f'Having trouble sending bug message in {channel.mention}') await Utils.handle_exception( f"Bug report message failed to send in channel #{channel.name} ({channel.id})", self.bot, e) await asyncio.sleep(0.5)
def run_db_migrations(): dbv = int(Configuration.get_persistent_var('db_version', 0)) Logging.info(f"db version is {dbv}") dbv_list = [f for f in glob.glob("db_migrations/db_migrate_*.py")] dbv_pattern = re.compile(r'db_migrations/db_migrate_(\d+)\.py', re.IGNORECASE) migration_count = 0 for filename in sorted(dbv_list): # get the int version number from filename version = int(re.match(dbv_pattern, filename)[1]) if version > dbv: try: Logging.info( f"--- running db migration version number {version}") spec = importlib.util.spec_from_file_location( f"migrator_{version}", filename) dbm = importlib.util.module_from_spec(spec) spec.loader.exec_module(dbm) Configuration.set_persistent_var('db_version', version) migration_count = migration_count + 1 except Exception as e: # throw a fit if it doesn't work raise e Logging.info( f"--- {migration_count if migration_count else 'no'} db migration{'' if migration_count == 1 else 's'} run" )
def get_locale_string(key, ctx='', **arg_dict): global LANG, locales_loaded locale = get_defaulted_locale(ctx) if not locale: return L_ERR if not locales_loaded: load_locales() output = [] # locale is a list or tuple. may be a single item or multiple for item in locale: locale_lang = LANG[item] key_list = key.split("/") # Check that keys point to a valid path in base keys obj = LANG['keys'] if get_by_path( obj, key_list[:-1]) is None or key_list[-1] not in get_by_path( obj, key_list[:-1]): raise KeyError(f"Lang key is not in lang_keys: {key}") if get_by_path(obj, key_list) is not None: raise KeyError(f"Lang key is not terminal: {key}") obj = get_by_path(locale_lang, key_list) # keys were found. Now check locale for value: if isinstance(obj, str): try: output.append(obj.format(**arg_dict)) except KeyError as e: output.append(obj) else: # Maybe string is not defined in lang file. Logging.info( f"localized lang string failed for key {key} in locale {item}") output.append(L_ERR) return '\n'.join(output)
async def send_welcome(self, member): guild = self.bot.get_guild(Configuration.get_var("guild_id")) if member.guild.id != guild.id or self.is_member_verified(member): return False try: welcome_channel = self.bot.get_config_channel( guild.id, Utils.welcome_channel) rules_channel = self.bot.get_config_channel( guild.id, Utils.rules_channel) if welcome_channel and rules_channel: txt = Lang.get_string( "welcome/welcome_msg", user=member.mention, rules_channel=rules_channel.mention, accept_emoji=Emoji.get_chat_emoji('CANDLE')) await welcome_channel.send(txt) return True except Exception as ex: Logging.info(f"failed to welcome {member.id}") Logging.error(ex) raise ex return False
async def close(self): Logging.info("Shutting down?") if not self.shutting_down: Logging.info("Shutting down...") self.shutting_down = True self.db_keepalive.cancel() temp = [] for cog in self.cogs: temp.append(cog) for cog in temp: Logging.info(f"unloading cog {cog}") c = self.get_cog(cog) if hasattr(c, "shutdown"): await c.shutdown() self.unload_extension(f"cogs.{cog}") return await super().close()
def add(path, name): ''' Create a repository. ''' os.chdir(path) name = wrap_repo_name(name) repo_name = os.path.join(path, name) if os.path.exists(repo_name): Logging.error('the repository is existent') return False run_command('git init --bare %s' % name) run_command('chown -R git:git %s' % name) repo_url = wrap_repo_url(path, name) Logging.info("repository url: %s" % (repo_url)) Logging.info("git clone: git clone %s" % (repo_url)) Logging.info("git remote: git remote add origin %s" % (repo_url))
async def hotreload(self, ctx): message = await ctx.send("Hot reloading...") importlib.reload(Reloader) for c in Reloader.components: importlib.reload(c) Emoji.initialize(self.bot) Logging.info("Reloading all cogs...") temp = [] for cog in self.bot.cogs: temp.append(cog) for cog in temp: self.bot.unload_extension(f"cogs.{cog}") Logging.info(f'{cog} has been unloaded.') self.bot.load_extension(f"cogs.{cog}") Logging.info(f'{cog} has been loaded.') await message.edit(content="Hot reload complete")
def get_defaulted_locale(ctx): locale = 'en_US' if isinstance(ctx, Context): # TODO: move guild/channel checks to LangConfig, store in dict, update there on guild events and config changes cid = ctx.channel.id if ctx.guild is None: # DM - default the language locale = Configuration.get_var('broadcast_locale', 'en_US') if locale == ALL_LOCALES: return locales return [locale] # TODO: create lookup table so we don't hit database every time # github issue #91 gid = ctx.guild.id guild_row = Guild.get_or_none(serverid=gid) chan_locale = Localization.get_or_none(channelid=cid) # Bot default is English if guild_row is not None and guild_row.defaultlocale in locales: # server locale overrides bot default locale = guild_row.defaultlocale if chan_locale is not None and chan_locale.locale in locales: # channel locale overrides server locale = chan_locale.locale elif isinstance(ctx, str): # String assumes caller knows better and is overriding all else if ctx == ALL_LOCALES: return locales if ctx not in locales: if ctx != '': Logging.info( f"Locale string override '{ctx}' not found. Defaulting.") else: locale = ctx else: Logging.info(f"Cannot derive locale from context: {ctx}") locale = False if locale not in locales: Logging.info(f"Missing locale {locale} - defaulting to English") locale = 'en_US' return [locale]
async def on_ready(self): Logging.info(f"Skybot... {'RECONNECT!' if self.loaded else 'STARTUP!'}") if self.loaded: Logging.info("Skybot reconnect") return Logging.BOT_LOG_CHANNEL = self.get_channel(Configuration.get_var("log_channel")) Emoji.initialize(self) for cog in Configuration.get_var("cogs"): try: self.load_extension("cogs." + cog) except Exception as e: await Utils.handle_exception(f"Failed to load cog {cog}", self, e) Logging.info("Cogs loaded") self.db_keepalive = self.loop.create_task(self.keepDBalive()) self.loaded = True await Logging.bot_log("Skybot soaring through the skies!")
import pymongo from config import * from utils import Logging logger = Logging(log_name="collateral_str").getLogger() conn = pymongo.MongoClient(mongodb_params['host'], mongodb_params['port']) conn_db = conn[mongodb_params['db_name']] count = 1 for object in conn_db.account_collateral.find(): logger.info(object) object["collateral"] = str(object["collateral"]) logger.info(object) logger.info("--------------- {} ---------------------\n".format(count)) count += 1 conn_db.account_collateral.save(object)
class Aggregator: """ Aggregator object Establishes connection and joins MUCs. Registers handlers """ def __init__(self): config = Configuration() self.db = Database() self.results = self.db.get_table('results') self.job_map = {} self.job_pool = [] self.failed_jobs = [] self.evals = {} self.notifier = Notifier() self.sched = MessageScheduler(self.message_handler) conn = Connection('aggregator', 'roflcake') entity_prefix, entity_suffix = conn.get_entity_name() self.entity_name = entity_prefix + entity_suffix self.log = Logging(conn) conn.join_muc('aggregators') self.conn = conn.get_conn() self.roster = self.conn.getRoster() self.conn.RegisterHandler('iq',self.set_handler,'set') self.conn.RegisterHandler('iq',self.get_handler,'get') self.conn.RegisterHandler('iq',self.result_handler,'result') self.conn.RegisterHandler('presence',self.presence_handler) self.temp_messages = [] self.parser = Parser() self.go_on() """ Handler for scheduler """ def message_handler(self, message, retry=False): #print 'Sending message.' if retry == True: self.log.error('Timed out, attempting to resend.') self.conn.send(message) # HANDLERS """ IQ result handler """ def result_handler(self, conn, iq_node): if self.sched.is_managed(int(iq_node.getID())): self.sched.received_response(iq_node) raise NodeProcessed """ Presence handler """ def presence_handler(self, conn, presence_node): if len(self.job_map) > 0: sender = presence_node.getFrom() if presence_node.getAttr('type') == 'unavailable': failed_poller = None for poller in self.job_map: if poller == sender: failed_poller = poller break if failed_poller != None: # Only used if Controller has gone offline self.log.info('Poller %s has gone offline.' % failed_poller) raise NodeProcessed """ IQ get handler """ def get_handler(self, conn, iq_node): if iq_node.getQueryNS() == NS_DISCO_INFO: reply = iq_node.buildReply('result') identity = Node('identity', {'category':'skynet', 'type':'aggregator'}) reply.setQueryPayload([identity]) conn.send(reply) raise NodeProcessed """ IQ set handler. Permits RPC calls in the whitelist, else returns error message """ def set_handler(self, conn, iq_node): sender = iq_node.getFrom() iq_id = iq_node.getAttr('id') if sender == '[email protected]/skynet': query_node = iq_node.getQueryChildren() for node in query_node: try: method = node.getTagData('methodName') method_whitelist = ['run_job', 'add_poller', 'remove_poller', 'remove_job', 'move_job'] if method in method_whitelist: method = getattr(self, method) try: try: params = node.getTag('params').getChildren() args = self.parser.get_args(params) except AttributeError: args = [] status, parameters = apply(method, args) message = self.parser.rpc_response(iq_node.getFrom(), iq_node.getID(), status, parameters) conn.send(message) except TypeError: #print sys.exc_info() conn.send(iq_node.buildReply('error')) else: conn.send(iq_node.buildReply('error')) self.log.error('Method called not in whitelist') except AttributeError: #print sys.exc_info() conn.send(iq_node.buildReply('error')) if len(self.job_map) > 0: if sender in self.job_map: query_node = iq_node.getQueryChildren() for node in query_node: try: method = node.getTagData('methodName') method_whitelist = ['add_result'] if method in method_whitelist: method = getattr(self, method) try: try: params = node.getTag('params').getChildren() args = self.parser.get_args(params) except AttributeError: args = [] status, parameters = apply(method, args) message = self.parser.rpc_response(iq_node.getFrom(), iq_node.getID(), status, parameters) conn.send(message) except TypeError: #print sys.exc_info() conn.send(iq_node.buildReply('error')) else: conn.send(iq_node.buildReply('error')) except AttributeError: #print sys.exc_info() conn.send(iq_node.buildReply('error')) raise NodeProcessed # This stanza is fully processed """ Establish evaluations """ def set_evals(self, job, evaluations): for evaluation in evaluations: if evaluation.string != None: self.evals[job].append((evaluation.comparison, str(evaluation.string))) elif evaluation.float != None: self.evals[job].append((evaluation.comparison, float(evaluation.float))) elif evaluation.int != None: self.evals[job].append((evaluation.comparison, int(evaluation.int))) else: self.log.info('Evaluation contains no comparison value.') """ Callback handler used by scheduler.add_message when poller replies on successful job assignment """ def assign_job(self, sender, query_node): if query_node.getNamespace() == NS_RPC: params = query_node.getTag('methodResponse').getTag('params').getChildren() job_id = self.parser.get_args_no_sender(params)[0] job_id = int(job_id) self.job_map[JID(sender)].append(job_id) self.evals[job_id] = [] evaluations = self.db.get_evaluations(job_id) self.set_evals(job_id, evaluations) else: pass # print 'Receieved iq message with incorrect namespace' # END HANDLERS # RPC METHODS """ Retains details of a job, allocates to Poller with least currently assinged jobs """ def run_job(self, sender, poller, job, addr, proto, freq, dom, resource): # Checks a poller is assigned if len(self.job_map) > 0: # Determines which poller has least assigned jobs # job_comp = None # least_loaded = None # for poller, jobs in self.job_map.items(): # num_jobs = len(jobs) # if job_comp != None: # if len(jobs) < job_comp: # least_loaded = poller # else: # least_loaded = poller # job_comp = num_jobs message = self.parser.rpc_call(poller, 'run_job', [self.entity_name, job, addr, proto, freq, dom, resource]) self.sched.add_message(message, self.assign_job) return 'success', [str(poller), int(job)] else: return 'failure', ['There are no pollers connected'] """ Called when a job is moved to this Aggregator, sets up evals and details """ def move_job(self, sender, poller, job, addr, proto, freq, dom, resource, segment): # Checks a poller is assigned if len(self.job_map) > 0: job_id = int(job) self.job_map[JID(poller)].append(job_id) self.evals[job_id] = [] evaluations = self.db.get_evaluations(job_id) self.set_evals(job_id, evaluations) return 'success', ['Successfully moved job'] else: return 'failure', ['There are no pollers connected'] """ Removes job from the Aggregator, cleans up state """ def remove_job(self, sender, job_id): job_id = int(job_id) parent_poller = None for poller, jobs in self.job_map.items(): for i in range(len(jobs)): if jobs[i] == job_id: jobs.pop(i) parent_poller = poller break try: self.evals.pop(job_id) except KeyError: pass if parent_poller != None: message = self.parser.rpc_call(parent_poller, 'remove_job', [job_id]) self.sched.add_message(message) return 'success', ['Successfully removed job'] else: return 'failure', ['Failed to remove job'] """ Called by child Poller to deliver result """ def add_result(self, sender, id, recorded, val): status = self.insert_result(id, recorded, val) if status != 'failure': return 'success', ['Sucessfully added result'] else: messages = self.temp_messages self.temp_messages = [] return 'failure', messages """ Called by Controller when Poller is assigned """ def add_poller(self, sender, poller): # Subscribe to poller for presence updates poller_jid = JID(poller) self.roster.Subscribe(poller_jid) self.job_map[poller_jid] = [] self.sched.add_message(self.parser.rpc_call(poller_jid, 'set_aggregator', [self.entity_name])) return 'success', ['Successfully added %s' % poller_jid] """ Called by Controller to remove references to Poller """ def remove_poller(self, sender, poller): poller_jid = JID(poller) try: unassigned_jobs = self.job_map.pop(poller_jid) # If controller has also failed # for job in unassigned_jobs: # self.job_pool.append(job) self.roster.Unsubscribe(poller_jid) return 'success', [poller] except KeyError: return 'failure', ['Failed to remove poller %s' % poller] # END RPC METHODS """ Used when inserting results supplied by add_result. Peforms evaluations, casts type, makes notifications and then stores into the database. """ def insert_result(self, id, recorded, val, list_id=None): val_type = type(val).__name__ try: evals = self.evals.get(int(id)) if evals: for comparison, comp_val in evals: if val_type == 'int' or val_type == 'float': eval_statement = str(val) + str(comparison) + str(comp_val) elif val_type == 'str': eval_statement = str('\'' + val + '\'') + str(comparison) + str('\'' + comp_val + '\'') # print 'Eval statement: %s' % eval_statement result = eval(eval_statement) # print 'Eval result: %s' % result if result != True: message = 'Job %s has caused an error! The value %s failed an evaluation.' % (id, comp_val) self.log.error(message) if id not in self.failed_jobs: self.log.info('Sending notifications') self.notifier.send_email(message) self.notifier.send_sms(message) self.failed_jobs.append(id) else: if id in self.failed_jobs: message = 'Job %s is back within normal parameters' % id self.notifier.send_email(message) self.log.info(message) self.failed_jobs.remove(id) except: # traceback.print_exc() self.temp_messages = ['Failed to evaluate returned result'] return 'failure' if val_type == 'int': if list_id != None: self.results.insert().execute(job=id, int=val, recorded=recorded, list=list_id) else: self.results.insert().execute(job=id, int=val, recorded=recorded) elif val_type == 'str': if list_id != None: self.results.insert().execute(job=id, string=val, recorded=recorded, list=list_id) else: self.results.insert().execute(job=id, string=val, recorded=recorded) elif val_type == 'float': if list_item != None: self.results.insert().execute(job=id, float=val, recorded=recorded, list=list_id) else: self.results.insert().execute(job=id, float=val, recorded=recorded) elif val_type == 'list': self.results.insert().execute(job=id, recorded=recorded, list=0) where = and_(self.results.c.recorded == recorded, self.results.c.list == 0) list_id = self.results.select(where).execute().fetchone().id #print "Retrieved list id %s" % list_id for element in val: self.insert_result(id, recorded, element, list_id) else: self.temp_messages = ['Unexpected data type receieved'] return 'failure' """ Setup listener """ def step_on(self): try: self.conn.Process(1) except KeyboardInterrupt: server = 'quae.co.uk' features.unregister(self.conn, server) #print 'Unregistered from %s' % server return 0 return 1 def go_on(self): while self.step_on(): pass
async def drop_message_impl(self, source_message, drop_channel): ''' handles copying to dropbox, sending confirm message in channel, sending dm receipt, and deleting original for each message in any dropbox ''' guild_id = source_message.channel.guild.id source_channel_id = source_message.channel.id source_message_id = source_message.id # get the ORM row for this dropbox. drop = None if source_channel_id in self.dropboxes[guild_id]: drop = self.dropboxes[guild_id][source_channel_id] else: # should only return one entry because of how rows are added drop = DropboxChannel.select().where( DropboxChannel.serverid == guild_id and DropboxChannel.sourcechannelid == source_channel_id) # the embed to display who was the author in dropbox channel embed = Embed(timestamp=source_message.created_at, color=0x663399) embed.set_author( name=f"{source_message.author} ({source_message.author.id})", icon_url=source_message.author.avatar_url_as(size=32)) embed.add_field(name="Author link", value=source_message.author.mention) ctx = await self.bot.get_context(source_message) pages = Utils.paginate(source_message.content) page_count = len(pages) if source_message.author.dm_channel is None: await source_message.author.create_dm() dm_channel = source_message.author.dm_channel attachment_names = [] delivery_success = None try: # send embed and message to dropbox channel for attachment in source_message.attachments: try: buffer = io.BytesIO() await attachment.save(buffer) await drop_channel.send( file=discord.File(buffer, attachment.filename)) attachment_names.append(attachment.filename) except Exception as attach_e: await drop_channel.send( Lang.get_locale_string( 'dropbox/attachment_fail', ctx, author=source_message.author.mention)) if len(pages) == 0: # means no text content included if len(attachment_names) < 1: # if there isn't any attachments, the dropbox might end up having a floating embed so include a helpful message too last_drop_message = await drop_channel.send( embed=embed, content=Lang.get_locale_string('dropbox/msg_blank', ctx)) else: last_drop_message = await drop_channel.send(embed=embed) else: # deliver all the pages of text content for i, page in enumerate(pages[:-1]): if len(pages) > 1: page = f"**{i+1} of {page_count}**\n{page}" await drop_channel.send(page) last_page = pages[ -1] if page_count == 1 else f"**{page_count} of {page_count}**\n{pages[-1]}" last_drop_message = await drop_channel.send(embed=embed, content=last_page) # TODO: try/ignore: add reaction for "claim" "flag" "followup" "delete" msg = Lang.get_locale_string('dropbox/msg_delivered', ctx, author=source_message.author.mention) await ctx.send(msg) delivery_success = True except Exception as e: msg = Lang.get_locale_string('dropbox/msg_not_delivered', ctx, author=source_message.author.mention) await ctx.send(msg) await self.bot.guild_log(guild_id, "broken dropbox...? Call alex, I guess") await Utils.handle_exception("dropbox delivery failure", self.bot, e) delivery_success = False try: # delete original message, the confirmation of sending is deleted in clean_channels loop await source_message.delete() del self.drop_messages[guild_id][source_channel_id][ source_message_id] set(self.delivery_in_progress[guild_id][source_channel_id]).remove( source_message_id) except discord.errors.NotFound as e: # ignore missing message pass #give senders a moment before spam pinging them the copy await asyncio.sleep(1) try: # try sending dm receipts and report in dropbox channel if it was sent or not if drop and drop.sendreceipt: # get the locale versions of the messages for status, receipt header, and attachments ready to be sent status_msg = Lang.get_locale_string( 'dropbox/msg_delivered' if delivery_success else 'dropbox/msg_not_delivered', ctx, author="") receipt_msg_header = Lang.get_locale_string( 'dropbox/msg_receipt', ctx, channel=ctx.channel.mention) if len(attachment_names) == 0: attachment_msg = "" else: attachment_msg_key = 'dropbox/receipt_attachment_plural' if len( attachment_names ) > 1 else 'dropbox/receipt_attachment_singular' attachment_msg = Lang.get_locale_string( attachment_msg_key, ctx, number=len(attachment_names), attachments=", ".join(attachment_names)) # might as well try to stuff in as few pages as possible dm_header_pages = Utils.paginate( f"{status_msg}\n{receipt_msg_header}\n{attachment_msg}") for page in dm_header_pages: await dm_channel.send(page) if len(pages) == 0: # no text content if len(attachment_names) < 1: #if no text and no attachments, then send a response that there wasn't any text content await dm_channel.send(content=Lang.get_locale_string( 'dropbox/msg_blank', ctx)) else: # send the page(s) in code blocks to dm. for i, page in enumerate(pages[:-1]): if len(pages) > 1: page = f"**{i+1} of {page_count}**\n```{page}```" await dm_channel.send(page) last_page = f'```{pages[-1]}```' if page_count == 1 else f"**{page_count} of {page_count}**\n```{pages[-1]}```" await dm_channel.send(last_page) if delivery_success: embed.add_field(name="receipt status", value="sent") # this is used if drop first before dms to add status to embed await last_drop_message.edit(embed=embed) except Exception as e: Logging.info( "Dropbox DM receipt failed, not an issue so ignoring exception and giving up" ) if drop.sendreceipt and delivery_success: embed.add_field(name="receipt status", value="failed") # this is used if drop first before dms to add status to embed await last_drop_message.edit(embed=embed)
def init_public_ip(): set_config('.', 'public_ip', get_public_ip()) Logging.info('Initialized config : [public_ip]')
def version(): ''' Show version and exit. ''' from . import __version__ Logging.info(__version__)
def before_send(event, hint): if event['level'] == "error" and 'logger' in event.keys( ) and event['logger'] == 'gearbot': return None # we send errors manually, in a much cleaner way if 'exc_info' in hint: exc_type, exc_value, tb = hint['exc_info'] for t in [ConnectionClosed, ClientOSError, ServerDisconnectedError]: if isinstance(exc_value, t): return return event if __name__ == '__main__': Logging.init() Logging.info("Launching thatskybot!") dsn = Configuration.get_var('SENTRY_DSN', '') if dsn != '': sentry_sdk.init(dsn, before_send=before_send) Database.init() loop = asyncio.get_event_loop() skybot = Skybot(command_prefix=Configuration.get_var("bot_prefix"), case_insensitive=True, loop=loop) skybot.remove_command("help") Utils.BOT = skybot
class Controller: """ Controller object. Initializes various data structures used by object. Establishes connection with XMPP server, connects to poller, aggregator and logging Multi-User Chats and registers stanza handlers. """ def __init__(self): self.db = Database() entity_prefix = 'controller' conn = Connection(entity_prefix, static=True) self.entity_name, self.entity_suffix = conn.get_entity_name() # List of nodes known to the controller self.poller_map = {} self.poller_pool = {} self.job_map = {} self.job_pool = [] # Message scheduler self.sched = MessageScheduler(self.message_handler) self.log = Logging(conn) conn.join_muc('pollers') conn.join_muc('aggregators') self.parser = Parser() self.establish_jobs() self.conn = conn.get_conn() self.conn.RegisterHandler('iq',self.result_handler,'result') self.conn.RegisterHandler('iq',self.set_handler,'set') self.conn.RegisterHandler('presence',self.presence_handler) self.go_on() """ Called by the presence handler when an entity connects to the aggregator or poller MUCs. Used to inspect retreive service information from an entity, required in the XEP Jabber-RPC standard. """ def disco_lookup(self, recipient): self.log.info('Performing discovery lookup.') message = Iq('get', queryNS=NS_DISCO_INFO, to=recipient) self.sched.add_message(message, self.disco_handler) """ Method passed and used as handler for messages by MessageScheduler. Sends messages and logs an error if message send is a retry """ # Handler used by message scheduling class def message_handler(self, message, retry=False): # print 'Sending message.' if retry == True: self.log.error('Message timed out, resending.') self.conn.send(message) # # MESSAGE HANDLERS # """ Handler for presence stanzas recieved by the XMPP listener. If """ def presence_handler(self, conn, presence_node): sender = presence_node.getFrom() presence_type = presence_node.getAttr('type') # Ignore self and presence announcements from logging MUC if sender.getResource() != 'controller': if presence_type == 'unavailable': if sender.getNode() == 'aggregators' or sender.getNode() == 'pollers': self.remove_entity(sender.getNode(), sender) elif sender.getNode() == 'aggregators' or sender.getNode() == 'pollers': # Check the service discovery details for a connecting node. self.disco_lookup(sender) raise NodeProcessed """ IQ set handler, runs RPC methods in whitelist """ def set_handler(self, conn, iq_node): query_node = iq_node.getQueryChildren() for node in query_node: try: method = node.getTagData('methodName') method_whitelist = ['get_group', 'get_groups', 'create_group', 'update_group', 'remove_group', 'get_monitor', 'get_monitors', 'create_monitor', 'update_monitor', 'remove_monitor', 'get_monitors_by_gid', 'get_job', 'get_jobs', 'create_job', 'update_job', 'remove_job', 'get_evaluation', 'get_evaluations', 'create_evaluation', 'update_evaluation', 'remove_evaluation', 'get_results', 'get_results_day', 'get_results_week', 'get_results_hour', 'poller_failure', 'get_aggregator'] if method in method_whitelist: method = getattr(self, method) try: try: params = node.getTag('params').getChildren() args = self.parser.get_args(params, iq_node.getFrom()) except AttributeError: args = [] status, parameters = apply(method, args) message = self.parser.rpc_response(iq_node.getFrom(), iq_node.getID(), status, parameters) self.conn.send(message) except TypeError: # print sys.exc_info() conn.send(iq_node.buildReply('error')) else: conn.send(iq_node.buildReply('error')) self.log.error('Method not in whitelist') except AttributeError: traceback.print_exc() conn.send(iq_node.buildReply('error')) raise NodeProcessed def result_handler(self, conn, iq_node): # Check if the reponse is managed by scheduler if self.sched.is_managed(int(iq_node.getAttr('id'))): self.sched.received_response(iq_node) raise NodeProcessed # END MESSAGE HANDLERS # # BEGIN SCHEDULER RESPONSE HANDLERS # def disco_handler(self, sender, query_node): if query_node.getNamespace() == NS_DISCO_INFO: entity_type = query_node.getTagAttr('identity', 'type') if entity_type == 'aggregator' or entity_type == 'poller': adjusted_jid = JID(sender.getResource() + '@quae.co.uk/skynet') category = query_node.getTagAttr('identity', 'category') self.log.info('Registering node %s' % adjusted_jid) self.add_entity(entity_type, category, adjusted_jid) else: self.log.error('Receieved iq message with incorrect namespace') def assign_job(self, sender, query_node): if query_node.getNamespace() == NS_RPC: poller, job_id = self.parser.get_args_no_sender(query_node.getTag('methodResponse').getTag('params').getChildren()) poller_jid = JID(poller) job_id = int(job_id) job = None for i in range(len(self.job_pool)): if self.job_pool[i]['id'] == job_id: job = self.job_pool.pop(i) self.log.info('Removing job %s from the job pool' % job_id) break if job != None: self.job_map[poller_jid].append(job) self.log.info('Job %s successfully assigned to %s' % (job_id, poller_jid)) else: self.log.error('Receieved iq message with incorrect namespace') def poller_removed(self, sender, query_node): if query_node.getNamespace() == NS_RPC: args = self.parser.get_args_no_sender(query_node.getTag('methodResponse').getTag('params').getChildren()) adjusted_jid = JID(args[0]) unassigned_jobs = self.job_map.pop(adjusted_jid) for job in unassigned_jobs: self.log.info('Adding job %s to the job pool' % job['id']) self.job_pool.append(job) parent_poller = None for aggregator, pollers in self.poller_map.items(): for poller, segment in pollers: if poller == adjusted_jid: parent_aggregator = aggregator pollers.remove((poller, segment)) break self.log.info('Removed %s from %s' % (adjusted_jid, parent_aggregator)) self.assign_pooled_jobs() else: self.log.error('Receieved iq message with incorrect namespace') # END SCHEDULER HANDLERS # # BEGIN RPC METHODS # # Requested by an aggregator when an assigned poller has failed/disconnected. def poller_failure(self, sender, previous_poller): pollers = self.poller_map[JID(sender)] poller_jid = JID('[email protected]/' + JID(previous_poller).getNode()) try: pollers.remove(poller_jid) message = 'Removed failed poller' try: self.rebalance_pollers() except: print sys.exc_info() return 'success', [message] except: return 'failure', ['Failed to remove poller'] # Group operations def get_group(self, sender, name): group = self.db.get_group_by_name(name) if group != None: return 'success', [group] return 'failure', ['No such group exists'] def get_groups(self, sender): groups = self.db.get_groups() if groups != None: return 'success', [groups] return 'failure', ['Failed to retreieve groups'] def create_group(self, sender, name, desc): existing = self.db.get_group_by_name(name) if existing == None: self.db.create_group(name, desc) return 'success', ['Sucessfully created group %s' % name] return 'failure', ['failure'] def update_group(self, sender, id, name, desc): group = self.db.get_group_by_id(id) if group != None: self.db.update_group(id, name, desc) return 'success', ['Successfully update group %s' % name] return 'failure', ['Failed to update group'] def remove_group(self, sender, name): self.db.remove_group_by_name(name) if self.db.get_group_by_name(name) == None: return 'success', ['Successfully remove group %s' % name] else: return 'failure', ['Failed to remove group %s' % name] # Monitor operations def get_monitor(self, sender, name): try: monitor = self.db.get_monitor(name) if monitor != False: return 'success', [monitor] except TypeError: return 'failure', ['No such monitor exists'] return 'failure', ['No such monitor exists'] def get_monitors(self, sender, group=None): try: if group != None: monitors = self.db.get_monitors(group) else: monitors = self.db.get_monitors() return 'success', [monitors] except AttributeError: return 'failure', ['Failed to retrieve monitors'] def get_monitors_by_gid(self, sender, group_id): try: if group_id != None: monitors = self.db.get_monitors_by_gid(group_id) return 'success', [monitors] except AttributeError: pass return 'failure', ['Failed to retrieve monitors'] def create_monitor(self, sender, name, description, group): if self.db.create_monitor(name, description, group) == True: return 'success', ['Successfully create monitor %s' % name] return 'failure', ['Failed to create monitor'] def update_monitor(self, sender, name, description, group): group = self.db.get_group_by_id(id) if group != None: self.db.update_group(id, name, desc) return 'success', ['Successfully update monitor %s' % name] return 'failure', ['Failed to update monitor'] def remove_monitor(self, sender, name): self.db.remove_monitor_by_name(name) if self.db.get_monitor_by_name(name) == None: return 'success', ['Successfully removed monitor %s' % name] else: return 'failure', ['Failed to remove monitor %s' % name] # job operations def get_job(self, sender, mon, id): try: job = self.db.get_job(id, mon) return 'success', [job] except TypeError: return 'failure', ['No such job exists'] def get_jobs(self, sender, mon): try: jobs = self.db.get_jobs(mon) return 'success', [jobs] except AttributeError: return 'failure', ['Failed to retreieve jobs'] def create_job(self, sender, mon, address, protocol, frequency, interface, resource): if self.db.get_monitor(mon) != None: if self.db.create_job(address, protocol, frequency, interface, resource, mon) == True: return 'success', ['Successfully created a job for %s' % mon] return 'failure', ['Failed to create job'] def update_job(self, sender, mon, id, address, protocol, frequency, interface, resource): existing = self.db.get_job(id, mon) if existing != None: self.db.update_job(id, address, protocol, frequency, interface, resource) return 'success', ['Successfully updated job'] else: return 'failure', ['failure'] def remove_job(self, sender, mon, id): self.db.remove_job(id) if self.db.get_job(id) == None: return 'success', ['Successfully removed job'] else: return 'failure', ['failure'] # Result read operations def get_results(self, sender, monitor, job, start_datetime, end_datetime): results = self.db.get_results(monitor, job, start_datetime, end_datetime) if results == None: return 'failure', ['No such results exist'] elif results != False: return 'success', results return 'failure', ['Failed to retreive specificied results'] def get_results_day(self, sender, monitor, job, start_datetime): results = self.db.get_results_day(monitor, job, start_datetime) if results == []: return 'failure', ['No such results exist'] elif results != False: job_details = self.db.get_job(job, monitor) return 'success', [job_details, results] return 'failure', ['Failed to retreive specificied results'] def get_results_week(self, sender, monitor, job, start_datetime): pass def get_results_hour(self, sender, monitor, job, start_datetime): results = self.db.get_results_hour(monitor, job, start_datetime) if results == []: return 'failure', ['No such results exist'] elif results != False: job_details = self.db.get_job(job, monitor) return 'success', [job_details, results] return 'failure', ['Failed to retreive specificied results'] # END RPC METHODS # # BEGIN PRIVATE METHODS # """ Called on successful DISCO request. Will register Poller or Aggregator, send jobs or balance jobs. """ def add_entity(self, entity_type, segment, entity): if entity_type == 'aggregator': self.poller_map[entity] = [] # If pollers have been added, but there were no aggregators running self.assign_pooled_pollers() if len(self.poller_map) > 1: self.rebalance_pollers() self.assign_pooled_jobs() elif entity_type == 'poller': self.job_map[entity] = [] self.poller_pool[entity] = segment self.assign_pooled_pollers() if len(self.poller_map) > 1: self.rebalance_pollers() self.assign_pooled_jobs() if len(self.job_map) > 1: self.rebalance_jobs() # Give poller to appropriate aggregator print 'Added %s to %ss' % (entity, entity_type) self.log.info('Node %s was successfully registered with the controller' % entity) """ Removing Poller or Aggregator """ def remove_entity(self, entity_type, entity): try: if entity_type == 'aggregators': adjusted_jid = JID(JID(entity).getResource() + '@quae.co.uk/skynet') unassigned_pollers = self.poller_map.pop(adjusted_jid) for poller, segment in unassigned_pollers: self.poller_pool[poller] = segment self.log.info('Removed %s' % adjusted_jid) if len(self.poller_pool) > 0: # Try and assign pooled pollers if not self.assign_pooled_pollers(): for poller, segment in unassigned_pollers: message = self.parser.rpc_call(poller, 'aggregator_failure', []) self.sched.add_message(message) elif entity_type == 'pollers': adjusted_jid = JID(JID(entity).getResource() + '@quae.co.uk/skynet') if len(self.poller_map) > 0: parent_aggregator = None for aggregator, pollers in self.poller_map.items(): for poller, segment in pollers: if adjusted_jid == poller: parent_aggregator = aggregator break if parent_aggregator != None: remove_call = self.parser.rpc_call(parent_aggregator, 'remove_poller', [str(adjusted_jid)]) self.sched.add_message(remove_call, self.poller_removed) else: try: self.job_map.pop(adjusted_jid) self.poller_pool.pop(adjusted_jid) self.log.info('Poller not assigned, sucessfully removed') except: self.log.error('Failed to remove poller') traceback.print_exc() except ValueError: self.log.error('Failed to remove %s' % entity) """ Assign unassigned pollers """ def assign_pooled_pollers(self): if len(self.poller_map) > 0: while len(self.poller_pool) > 0: unassigned_poller, segment = self.poller_pool.popitem() chosen_aggregator = None poller_comp = None for aggregator, pollers in self.poller_map.items(): # If first loop or number of pollers assigned to aggregator is less than comp, make this agg the comp if (chosen_aggregator == None and poller_comp == None) or len(pollers) < poller_comp: chosen_aggregator = aggregator poller_comp = len(pollers) if chosen_aggregator != None: # Assign Poller to the Aggregtor with least assigned Pollers message = self.parser.rpc_call(chosen_aggregator, 'add_poller', [str(unassigned_poller)]) self.sched.add_message(message) for job in self.job_map[unassigned_poller]: message = self.parser.rpc_call(chosen_aggregator, 'move_job', [str(unassigned_poller), job['id'], job['address'], job['protocol'], job['frequency'], job['interface'], job['resource'], job['segment']]) self.sched.add_message(message) self.poller_map[chosen_aggregator].append((unassigned_poller, segment)) return True else: self.log.info('No aggregators available for poller assignment') return False """ Get Pollers for a given network segment """ def get_segment_pollers(self, segment): segment_pollers = {} for aggregator, pollers in self.poller_map.items(): for poller, poller_segment in pollers: if poller_segment == segment: segment_pollers[poller] = self.job_map[poller] return segment_pollers """ Called to allocate unassigned jobs """ def assign_pooled_jobs(self): if len(self.job_map) > 0 and len(self.poller_map) > 0: for job in self.job_pool: unassigned_job = job least_loaded = None job_comp = None pollers = self.get_segment_pollers(unassigned_job['segment']) for poller, jobs in pollers.items(): # for poller, jobs in self.job_map.items(): if (least_loaded == None and job_comp == None) or len(jobs) < job_comp: least_loaded = poller job_comp = len(jobs) if least_loaded != None: chosen_aggregator = None for aggregator, pollers in self.poller_map.items(): for poller, segment in pollers: if poller == least_loaded: chosen_aggregator = aggregator break if chosen_aggregator != None: self.send_job(unassigned_job, least_loaded, chosen_aggregator) else: self.log.info('No assigned pollers available for job assignment') #print 'Job map %s' % self.job_map #print 'Job pool %s' % self.job_pool """ Rebalance pollers, compares amount assigned to each Aggregator, and moves across to another Poller if there's at least 2 more than another Aggregator """ def rebalance_pollers(self): self.log.info('Attempting to rebalance pollers') poller_comp = None least_pollers = None most_pollers = None # Retrieve aggregators with least and most pollers for aggregator, pollers in self.poller_map.items(): if poller_comp == None: least_pollers = aggregator most_pollers = aggregator elif len(pollers) < poller_comp: least_pollers = aggregator elif len(pollers) > poller_comp: most_pollers = aggregator poller_comp = len(pollers) if least_pollers != None and most_pollers != None: # If the difference between the two pollers is worth balancing if (len(self.poller_map[most_pollers]) - len(self.poller_map[least_pollers])) > 1: poller, segment = self.poller_map[most_pollers].pop() self.poller_map[least_pollers].append((poller, segment)) self.sched.add_message(self.parser.rpc_call(least_pollers, 'add_poller', [str(poller)])) self.sched.add_message(self.parser.rpc_call(most_pollers, 'remove_poller', [str(poller)])) self.rebalance_pollers() else: self.log.info('Pollers balanced') return True """ Similar to above, checks number of assigned jobs through the system, and will level them across all Pollers """ def rebalance_jobs(self): self.log.info('Attempting to rebalance jobs') job_comp = None least_jobs = None most_jobs = None network_segment = 'skynet' pollers = self.get_segment_pollers(network_segment) # for poller, jobs in self.job_map.items(): for poller, jobs in pollers.items(): print poller if job_comp == None: least_jobs = poller most_jobs = poller elif len(jobs) < job_comp: least_jobs = poller elif len(jobs) > job_comp: most_jobs = poller job_comp = len(jobs) if least_jobs != None and most_jobs != None: if (len(self.job_map[most_jobs]) - len(self.job_map[least_jobs])) > 1: job = self.job_map[most_jobs].pop() self.job_map[least_jobs].append(job) least_parent = None most_parent = None for aggregator, pollers in self.poller_map.items(): for poller, node_segment in pollers: if network_segment == node_segment: if poller == least_jobs: least_parent = aggregator if poller == most_jobs: most_parent = aggregator if least_parent != None and most_parent != None: break if least_parent != None and most_parent != None: self.log.info('Moving job %s to %s' % (job['id'], least_jobs)) self.sched.add_message(self.parser.rpc_call(most_parent, 'remove_job', [job['id']])) self.sched.add_message(self.parser.rpc_call(least_parent, 'run_job', [str(least_jobs), job['id'], job['address'], job['protocol'], job['frequency'], job['interface'], job['resource']]), offset=True) self.rebalance_jobs() else: self.log.info('Jobs balanced') return True """ Retrieve jobs on startup """ def establish_jobs(self): monitors = self.db.get_monitors() self.log.info('Retrieving jobs') for monitor in monitors: jobs = self.db.get_jobs(monitor.name) for job in jobs: job = dict(job) segment_name = self.db.get_segment_name(job['segment']) job['segment'] = segment_name # Make the poll freq stored every minute minimum job['frequency'] = job['frequency'] * 60 self.job_pool.append(job) self.log.info('%s jobs added to the pool' % len(self.job_pool)) """ Send job to Aggregator to forward to Poller """ def send_job(self, job, poller, aggregator): message = self.parser.rpc_call(aggregator, 'run_job', [str(poller), job['id'], job['address'], job['protocol'], job['frequency'], job['interface'], job['resource']]) self.log.info('Sending job %s to %s' % (job['id'], aggregator)) self.sched.add_message(message, self.assign_job, offset=True) def step_on(self): try: self.conn.Process(1) except KeyboardInterrupt: return 0 return 1 def go_on(self): while self.step_on(): pass
def before_send(event, hint): if event['level'] == "error" and 'logger' in event.keys( ) and event['logger'] == 'gearbot': return None # we send errors manually, in a much cleaner way if 'exc_info' in hint: exc_type, exc_value, tb = hint['exc_info'] for t in [ConnectionClosed, ClientOSError, ServerDisconnectedError]: if isinstance(exc_value, t): return return event if __name__ == '__main__': Logging.init() Logging.info("Launching journeybot!") dsn = Configuration.get_var('SENTRY_DSN', '') env = Configuration.get_var('SENTRY_ENV', '') if dsn != '' and env != '': sentry_sdk.init(dsn, before_send=before_send, environment=env) Database.init() loop = asyncio.get_event_loop() journeybot = Journeybot(command_prefix=Configuration.get_var("bot_prefix"), case_insensitive=True, loop=loop) journeybot.remove_command("help")
async def on_message(self, message: discord.Message): if message.author.bot or not hasattr(message.author, "guild"): return welcome_channel = self.bot.get_config_channel(message.guild.id, Utils.welcome_channel) rules_channel = self.bot.get_config_channel(message.guild.id, Utils.rules_channel) log_channel = self.bot.get_config_channel(message.guild.id, Utils.log_channel) member_role = message.guild.get_role( Configuration.get_var("member_role")) nonmember_role = message.guild.get_role( Configuration.get_var("nonmember_role")) if message.author.id == 349977940198555660: # is gearbot pattern = re.compile( r'\(``(\d+)``\) has re-joined the server before their mute expired' ) match = re.search(pattern, message.content) if match: user_id = int(match[1]) # gearbot is handling it. never unmute this user self.remove_member_from_cooldown(message.guild.id, user_id) muted_member = message.guild.get_member(user_id) muted_member_name = Utils.get_member_log_name(muted_member) await log_channel.send(f''' Gearbot re-applied mute when member re-joined: {muted_member_name} I won't try to unmute them later. ''') return if message.author.guild_permissions.mute_members or \ await self.member_verify_action(message) or \ member_role in message.author.roles: # is a mod or # verification flow triggered. no further processing or # message from regular member. no action for welcomer to take. return # ignore when channels not configured if not welcome_channel or not rules_channel or message.channel.id != welcome_channel.id: if member_role not in message.author.roles: # nonmember speaking somewhere other than welcome channel? Maybe we're not using the # welcome channel anymore? or something else went wrong... give them member role. try: await message.author.add_roles(member_role) if nonmember_role in message.author.roles: Logging.info( f"{Utils.get_member_log_name(message.author)} - had shadow role when speaking. removing it!" ) await message.author.remove_roles(nonmember_role) except Exception as e: try: Logging.info(f"message: {message.content}") Logging.info(f"author id: {message.author.id}") except Exception as ee: pass await Utils.handle_exception("member join exception", self.bot, e) return # Only act on messages in welcome channel from here on # Nonmember will only be warned once every 10 minutes that they are speaking in welcome channel now = datetime.now().timestamp() then = 0 grace_period = 10 * 60 # 10 minutes try: was_welcomed = self.welcome_talkers[message.guild.id][ message.author.id] then = was_welcomed + grace_period except Exception as ex: # TODO: refine exception. KeyError only? pass if then > now: # grace period has not expired. Do not warn member again yet. # print("it hasn't been 10 minutes...") return # record the time so member won't be pinged again too soon if they keep talking self.welcome_talkers[message.guild.id][message.author.id] = now await welcome_channel.send( Lang.get_string("welcome/welcome_help", author=message.author.mention, rules_channel=rules_channel.mention)) # ping log channel with detail if log_channel: await log_channel.send( f"{Utils.get_member_log_name(message.author)} " f"spoke in {welcome_channel.mention} ```{message.content}```")
return event def can_help(ctx): return ctx.author.guild_permissions.mute_members def can_admin(): async def predicate(ctx): return await ctx.bot.permission_manage_bot(ctx) return commands.check(predicate) if __name__ == '__main__': Logging.init() Logging.info("Launching Skybot!") dsn = Configuration.get_var('SENTRY_DSN', '') dsn_env = Configuration.get_var('SENTRY_ENV', 'Dev') Logging.info(f"DSN info - dsn:{dsn} env:{dsn_env}") if dsn != '': sentry_sdk.init(dsn, before_send=before_send, environment=dsn_env, integrations=[AioHttpIntegration()]) # TODO: exception handling for db migration error run_db_migrations() Logging.info('dg migrations go') Database.init() Logging.info('db init go') intents = Intents(members=True, messages=True, guilds=True, bans=True, emojis=True, presences=True, reactions=True) loop = asyncio.get_event_loop()
dest="size", type=size_limit, default=20000, help= "enter a value from 10 to 1000 (default = 20, less than 20kB will not be downloaded)", ) parser.add_argument( "-e", dest="ext", metavar="exclude", default=False, help="exclude image type/extension, i.e., exclude gif, jpg, webp, etc.", ) parser.add_argument("-j", dest="hash", action="store_true", help="create json record of hashed image files") args = parser.parse_args() # remove dot from extension if present if args.ext: args.ext = args.ext.replace(".", "") # account for variation in jpg extension format if args.ext in ("jpg", ".jpg"): args.ext = "jpeg" log.info(f"{'Initiating connection':>15}") main(args.url, args.size, args.ext, args.hash)
class Poller: """ Poller object, establish connection, MUCs and handlers """ def __init__(self, segment='skynet'): config = Configuration() self.jobs = {} self.sched = MessageScheduler(self.message_handler) self.parser = Parser() self.aggregator = None self.failed_aggregator = False self.query_queue = [] conn = Connection('poller', 'roflcake') self.entity_prefix, entity_suffix = conn.get_entity_name() # self.entity_name = entity_prefix + entity_suffix conn.join_muc('pollers') self.segment = segment self.conn = conn.get_conn() self.log = Logging(conn) self.roster = self.conn.getRoster() self.conn.RegisterHandler('presence',self.presence_handler) self.conn.RegisterHandler('iq', self.result_handler, 'result') self.conn.RegisterHandler('iq', self.error_handler, 'error') self.conn.RegisterHandler('iq', self.get_handler, 'get') self.conn.RegisterHandler('iq', self.set_handler, 'set') self.go_on() """ Message scheduling handler """ def message_handler(self, message, retry=False): if retry == True: self.log.error('Timed out, attempting to resend.') self.conn.send(message) # # Handlers for node communication # """ Presence stanza handler """ def presence_handler(self, conn, presence_node): sender = presence_node.getFrom() if presence_node.getAttr('type') == 'subscribe': self.roster.Authorize(sender) # if sender != self.entity_name: # print presence_node.getRole() # conn.send(Iq('get', NS_DISCO_INFO, to=presence_node.getFrom())) raise NodeProcessed """ IQ result handler, acknowledges messages with scheduler """ def result_handler(self, conn, iq_node): iq_id = iq_node.getAttr('id') sender = iq_node.getFrom() if sender.getNode() != self.entity_prefix: # pass self.sched.received_response(iq_node) raise NodeProcessed # This stanza is fully processed """ IQ error handler """ def error_handler(self, conn, iq_node): if iq_node.getFrom() == self.aggregator: pass # print 'Erk!' raise NodeProcessed """ IQ get handler """ def get_handler(self, conn, iq_node): if iq_node.getQueryNS() == NS_DISCO_INFO: reply = iq_node.buildReply('result') if self.segment != None: category = self.segment else: category = 'skynet' identity = Node('identity', {'category':category, 'type':'poller'}) reply.setQueryPayload([identity]) conn.send(reply) else: conn.send(iq_node.buildReply('error')) raise NodeProcessed """ IQ set handler, used for RPC calls, permits methods in whitelist """ def set_handler(self, conn, iq_node): sender = iq_node.getFrom() if sender.getNode() != self.entity_prefix: query_node = iq_node.getQueryChildren() for node in query_node: try: method = node.getTagData('methodName') method_whitelist = ['run_job', 'set_aggregator', 'remove_job', 'aggregator_failure'] if method in method_whitelist: method = getattr(self, method) try: try: params = node.getTag('params').getChildren() args = self.parser.get_args(params) except AttributeError: args = [] status, parameters = apply(method, args) message = self.parser.rpc_response(iq_node.getFrom(), iq_node.getID(), status, parameters) conn.send(message) except TypeError: # print sys.exc_info() conn.send(iq_node.buildReply('error')) else: #print 'Method not in whitelist' #print sys.exc_info() conn.send(iq_node.buildReply('error')) except AttributeError: #print sys.exc_info() conn.send(iq_node.buildReply('error')) raise NodeProcessed # # RPC METHODS (job setup and scheduling) # """ Called by Aggregator to establish job """ def run_job(self, sender, aggregator, id, addr, proto, freq, dom, resource): try: job = Job(self, id, addr, proto, freq, dom, resource, self.conn, self.sched) self.jobs[id] = job job.start() return 'success', [int(id)] except: return 'failure', ['Failed to schedule job'] """ Controller calls to notify of parent Aggregator failure """ def aggregator_failure(self, sender): self.aggregator = None self.failed_aggregator = True return 'success', ['Removed parent aggregator'] """ Called when job moved off this Poller """ def remove_job(self, sender, job_id): try: job = self.jobs[int(job_id)] job.end() return 'success', ['Stopped job %s' % job_id] except: return 'failure', ['Failed to stop job %s' % job_id] """ Called by parent Aggregator, sets where add_results will be sent """ def set_aggregator(self, sender, aggregator): self.log.info('Setting aggregator %s' % aggregator) self.aggregator = aggregator if self.failed_aggregator == True: self.failed_aggregator = False for job in self.jobs.values(): job.send_cached_results() return 'success', ['Successfully set aggregator'] # # PRIVATE METHODS # """ Provides parent Aggregator, used by Job instances """ def get_aggregator(self): return self.aggregator """ Setup listener """ def step_on(self): try: self.conn.Process(1) except KeyboardInterrupt: server = 'quae.co.uk' features.unregister(self.conn, server) # Stop all running jobs for job in self.jobs.values(): job.end() self.sched.end() print 'Unregistered from %s' % server return 0 return 1 def go_on(self): while self.step_on(): pass