async def ping_unverified(self, ctx): guild = self.bot.get_guild(Configuration.get_var("guild_id")) try: nonmember_role = guild.get_role( Configuration.get_var("nonmember_role")) welcome_channel = self.bot.get_config_channel( guild.id, Utils.welcome_channel) rules_channel = self.bot.get_config_channel( guild.id, Utils.rules_channel) if welcome_channel and rules_channel: txt = Lang.get_string( "welcome/welcome_msg", user=nonmember_role.mention, rules_channel=rules_channel.mention, accept_emoji=Emoji.get_chat_emoji('CANDLE')) await nonmember_role.edit(mentionable=True) await welcome_channel.send(txt) await nonmember_role.edit(mentionable=False) return True except Exception as ex: Logging.info(f"failed to welcome unverified role.") Logging.error(ex) raise ex return False
async def handle_reaction_change(self, t, reaction, user_id): roles = Configuration.get_var("roles") if reaction in roles: guild = self.bot.get_guild(Configuration.get_var("guild_id")) role = guild.get_role(roles[reaction]) member_role = guild.get_role(Configuration.get_var("member_role")) nonmember_role = guild.get_role( Configuration.get_var("nonmember_role")) member = guild.get_member(user_id) if member is None: return action = getattr(member, f"{t}_roles") try: await action(role) # if acting on member role, toggle corresponding nonmember role if role is member_role: if t == 'add': await member.remove_roles(nonmember_role) else: await member.add_roles(nonmember_role) except Exception as ex: Logging.info("failed") Logging.error(ex) raise ex
async def send_welcome(self, member): guild = self.bot.get_guild(Configuration.get_var("guild_id")) if member.guild.id != guild.id or self.is_member_verified(member): return False try: welcome_channel = self.bot.get_config_channel( guild.id, Utils.welcome_channel) rules_channel = self.bot.get_config_channel( guild.id, Utils.rules_channel) # Send welcome message in configured language. default to english if welcome_channel and rules_channel: txt = Lang.get_string( "welcome/welcome_msg", user=member.mention, rules_channel=rules_channel.mention, accept_emoji=Emoji.get_chat_emoji('PAINTBRUSH')) if self.mute_new_members: # add mute notification if mute for new members is on mute_txt = Lang.get_string("welcome/welcome_mute_msg") txt = f"{txt}\n{mute_txt}" await welcome_channel.send(txt) return True except Exception as ex: Logging.info(f"failed to welcome {member.id}") Logging.error(ex) raise ex return False
def wrapper(*args, **kwargs): path = args[0] grm_path = os.path.join(path, BASE_GRM_DIR) if not os.path.exists(grm_path): Logging.error("fatal: not a remote root directory(%s)" % BASE_GRM_DIR) return return func(*args, **kwargs)
async def handle_exception(exception_type, bot, exception, event=None, message=None, ctx=None, *args, **kwargs): embed = get_embed_and_log_exception(exception_type, bot, exception, event, message, ctx, *args, **kwargs) try: await Logging.bot_log(embed=embed) except Exception as ex: Logging.error( f"Failed to log to botlog, either Discord broke or something is seriously wrong!\n{ex}") Logging.error(traceback.format_exc())
def load_guild(self, guild): my_channels = dict() for row in ConfigChannel.select().where(ConfigChannel.serverid == guild.id): if validate_channel_name(row.configname): my_channels[row.configname] = row.channelid else: Logging.error(f"Misconfiguration in config channel: {row.configname}") self.bot.config_channels[guild.id] = my_channels
def load(): global MASTER_CONFIG, MASTER_LOADED try: with open('config.json', 'r') as json_file: MASTER_CONFIG = json.load(json_file) MASTER_LOADED = True except FileNotFoundError: Logging.error("Unable to load config, running with defaults.") except Exception as e: Logging.error("Failed to parse configuration.") print(e) raise e
async def startup_cleanup(self): # Load channels self.bot.config_channels = dict() for guild in self.bot.guilds: my_channels = dict() for row in ConfigChannel.select().where( ConfigChannel.serverid == guild.id): if validate_channel_name(row.configname): my_channels[row.configname] = row.channelid else: Logging.error( f"Misconfiguration in config channel: {row.configname}" ) self.bot.config_channels[guild.id] = my_channels
def add(path, name): ''' Create a repository. ''' os.chdir(path) name = wrap_repo_name(name) repo_name = os.path.join(path, name) if os.path.exists(repo_name): Logging.error('the repository is existent') return False run_command('git init --bare %s' % name) run_command('chown -R git:git %s' % name) repo_url = wrap_repo_url(path, name) Logging.info("repository url: %s" % (repo_url)) Logging.info("git clone: git clone %s" % (repo_url)) Logging.info("git remote: git remote add origin %s" % (repo_url))
def init_root_repos(path): grm_path = os.path.join(path, BASE_GRM_DIR) if os.path.exists(grm_path): Logging.error('Reinitialized existing remote root directory in %s' % os.path.abspath(path)) return False if not os.path.exists(path): os.mkdir(path) Logging.info('Initialized remote root directory in %s' % os.path.abspath(path)) os.chdir(path) os.mkdir(BASE_GRM_DIR) init_config('.') p = Process(target=init_public_ip) p.start()
async def send_welcome(self, member): guild = self.bot.get_guild(Configuration.get_var("guild_id")) if member.guild.id != guild.id or self.is_member_verified(member): return False try: welcome_channel = self.bot.get_config_channel( guild.id, Utils.welcome_channel) rules_channel = self.bot.get_config_channel( guild.id, Utils.rules_channel) if welcome_channel and rules_channel: txt = Lang.get_string( "welcome/welcome_msg", user=member.mention, rules_channel=rules_channel.mention, accept_emoji=Emoji.get_chat_emoji('CANDLE')) await welcome_channel.send(txt) return True except Exception as ex: Logging.info(f"failed to welcome {member.id}") Logging.error(ex) raise ex return False
class Poller: """ Poller object, establish connection, MUCs and handlers """ def __init__(self, segment='skynet'): config = Configuration() self.jobs = {} self.sched = MessageScheduler(self.message_handler) self.parser = Parser() self.aggregator = None self.failed_aggregator = False self.query_queue = [] conn = Connection('poller', 'roflcake') self.entity_prefix, entity_suffix = conn.get_entity_name() # self.entity_name = entity_prefix + entity_suffix conn.join_muc('pollers') self.segment = segment self.conn = conn.get_conn() self.log = Logging(conn) self.roster = self.conn.getRoster() self.conn.RegisterHandler('presence',self.presence_handler) self.conn.RegisterHandler('iq', self.result_handler, 'result') self.conn.RegisterHandler('iq', self.error_handler, 'error') self.conn.RegisterHandler('iq', self.get_handler, 'get') self.conn.RegisterHandler('iq', self.set_handler, 'set') self.go_on() """ Message scheduling handler """ def message_handler(self, message, retry=False): if retry == True: self.log.error('Timed out, attempting to resend.') self.conn.send(message) # # Handlers for node communication # """ Presence stanza handler """ def presence_handler(self, conn, presence_node): sender = presence_node.getFrom() if presence_node.getAttr('type') == 'subscribe': self.roster.Authorize(sender) # if sender != self.entity_name: # print presence_node.getRole() # conn.send(Iq('get', NS_DISCO_INFO, to=presence_node.getFrom())) raise NodeProcessed """ IQ result handler, acknowledges messages with scheduler """ def result_handler(self, conn, iq_node): iq_id = iq_node.getAttr('id') sender = iq_node.getFrom() if sender.getNode() != self.entity_prefix: # pass self.sched.received_response(iq_node) raise NodeProcessed # This stanza is fully processed """ IQ error handler """ def error_handler(self, conn, iq_node): if iq_node.getFrom() == self.aggregator: pass # print 'Erk!' raise NodeProcessed """ IQ get handler """ def get_handler(self, conn, iq_node): if iq_node.getQueryNS() == NS_DISCO_INFO: reply = iq_node.buildReply('result') if self.segment != None: category = self.segment else: category = 'skynet' identity = Node('identity', {'category':category, 'type':'poller'}) reply.setQueryPayload([identity]) conn.send(reply) else: conn.send(iq_node.buildReply('error')) raise NodeProcessed """ IQ set handler, used for RPC calls, permits methods in whitelist """ def set_handler(self, conn, iq_node): sender = iq_node.getFrom() if sender.getNode() != self.entity_prefix: query_node = iq_node.getQueryChildren() for node in query_node: try: method = node.getTagData('methodName') method_whitelist = ['run_job', 'set_aggregator', 'remove_job', 'aggregator_failure'] if method in method_whitelist: method = getattr(self, method) try: try: params = node.getTag('params').getChildren() args = self.parser.get_args(params) except AttributeError: args = [] status, parameters = apply(method, args) message = self.parser.rpc_response(iq_node.getFrom(), iq_node.getID(), status, parameters) conn.send(message) except TypeError: # print sys.exc_info() conn.send(iq_node.buildReply('error')) else: #print 'Method not in whitelist' #print sys.exc_info() conn.send(iq_node.buildReply('error')) except AttributeError: #print sys.exc_info() conn.send(iq_node.buildReply('error')) raise NodeProcessed # # RPC METHODS (job setup and scheduling) # """ Called by Aggregator to establish job """ def run_job(self, sender, aggregator, id, addr, proto, freq, dom, resource): try: job = Job(self, id, addr, proto, freq, dom, resource, self.conn, self.sched) self.jobs[id] = job job.start() return 'success', [int(id)] except: return 'failure', ['Failed to schedule job'] """ Controller calls to notify of parent Aggregator failure """ def aggregator_failure(self, sender): self.aggregator = None self.failed_aggregator = True return 'success', ['Removed parent aggregator'] """ Called when job moved off this Poller """ def remove_job(self, sender, job_id): try: job = self.jobs[int(job_id)] job.end() return 'success', ['Stopped job %s' % job_id] except: return 'failure', ['Failed to stop job %s' % job_id] """ Called by parent Aggregator, sets where add_results will be sent """ def set_aggregator(self, sender, aggregator): self.log.info('Setting aggregator %s' % aggregator) self.aggregator = aggregator if self.failed_aggregator == True: self.failed_aggregator = False for job in self.jobs.values(): job.send_cached_results() return 'success', ['Successfully set aggregator'] # # PRIVATE METHODS # """ Provides parent Aggregator, used by Job instances """ def get_aggregator(self): return self.aggregator """ Setup listener """ def step_on(self): try: self.conn.Process(1) except KeyboardInterrupt: server = 'quae.co.uk' features.unregister(self.conn, server) # Stop all running jobs for job in self.jobs.values(): job.end() self.sched.end() print 'Unregistered from %s' % server return 0 return 1 def go_on(self): while self.step_on(): pass
def get_embed_and_log_exception(exception_type, bot, exception, event=None, message=None, ctx=None, *args, **kwargs): with sentry_sdk.push_scope() as scope: embed = Embed(colour=Colour(0xff0000), timestamp=datetime.utcfromtimestamp(time.time())) # something went wrong and it might have been in on_command_error, make sure we log to the log file first lines = [ "\n===========================================EXCEPTION CAUGHT, DUMPING ALL AVAILABLE INFO===========================================", f"Type: {exception_type}" ] arg_info = "" for arg in list(args): arg_info += extract_info(arg) + "\n" if arg_info == "": arg_info = "No arguments" kwarg_info = "" for name, arg in kwargs.items(): kwarg_info += "{}: {}\n".format(name, extract_info(arg)) if kwarg_info == "": kwarg_info = "No keyword arguments" lines.append("======================Exception======================") lines.append(f"{str(exception)} ({type(exception)})") lines.append("======================ARG INFO======================") lines.append(arg_info) sentry_sdk.add_breadcrumb(category='arg info', message=arg_info, level='info') lines.append("======================KWARG INFO======================") lines.append(kwarg_info) sentry_sdk.add_breadcrumb(category='kwarg info', message=kwarg_info, level='info') lines.append("======================STACKTRACE======================") tb = "".join(traceback.format_tb(exception.__traceback__)) lines.append(tb) if message is None and event is not None and hasattr(event, "message"): message = event.message if message is None and ctx is not None: message = ctx.message if message is not None and hasattr(message, "content"): lines.append("======================ORIGINAL MESSAGE======================") lines.append(message.content) if message.content is None or message.content == "": content = "<no content>" else: content = message.content scope.set_tag('message content', content) embed.add_field(name="Original message", value=trim_message(content, 1000), inline=False) lines.append("======================ORIGINAL MESSAGE (DETAILED)======================") lines.append(extract_info(message)) if event is not None: lines.append("======================EVENT NAME======================") lines.append(event) scope.set_tag('event name', event) embed.add_field(name="Event", value=event) if ctx is not None: lines.append("======================COMMAND INFO======================") lines.append(f"Command: {ctx.command.name}") embed.add_field(name="Command", value=ctx.command.name) scope.set_tag('command', ctx.command.name) channel_name = 'Private Message' if isinstance(ctx.channel, PrivateChannel) else f"{ctx.channel.name} (`{ctx.channel.id}`)" lines.append(f"Channel: {channel_name}") embed.add_field(name="Channel", value=channel_name, inline=False) scope.set_tag('channel', channel_name) sender = f"{str(ctx.author)} (`{ctx.author.id}`)" scope.set_user({"id": ctx.author.id, "username": str(ctx.author)}) lines.append(f"Sender: {sender}") embed.add_field(name="Sender", value=sender, inline=False) lines.append( "===========================================DATA DUMP COMPLETE===========================================") Logging.error("\n".join(lines)) for t in [ConnectionClosed, ClientOSError, ServerDisconnectedError]: if isinstance(exception, t): return # nice embed for info on discord embed.set_author(name=exception_type) embed.add_field(name="Exception", value=f"{str(exception)} (`{type(exception)}`)", inline=False) if len(tb) < 1024: embed.add_field(name="Traceback", value=tb) else: embed.add_field(name="Traceback", value="stacktrace too long, see logs") sentry_sdk.capture_exception(exception) return embed
def compare_time(time1, time2): s_time = time.mktime(time.strptime(time1, '%Y-%m-%d')) e_time = time.mktime(time.strptime(time2, '%Y-%m-%d')) return int(s_time) - int(e_time) def compare_time_test(): result = compare_time('2020-04-17', '2020-04-19') logger.info("result: {}".format(result)) if __name__ == '__main__': logger.info('args: {}'.format(sys.argv)) if len(sys.argv) < 2: logger.error('Usage: python3 check.py start_date[2020-07-01]') sys.exit(1) start_date = sys.argv[1] check_block(start_date) ''' 1. 功能 统计每天的链上区块、交易和operation的总数。 输入一个开始日期,统计该日期之后N天的数据,N由AFTER_DAYS全局变量控制,默认是7,可以根据统计需求任意修改。 如果开始日期和最新区块的间隔小于N天,统计最新区块之前的N天数据。 2. 使用 依赖: python-sdk python3 check_count.py YYYY-MM-DD
class Aggregator: """ Aggregator object Establishes connection and joins MUCs. Registers handlers """ def __init__(self): config = Configuration() self.db = Database() self.results = self.db.get_table('results') self.job_map = {} self.job_pool = [] self.failed_jobs = [] self.evals = {} self.notifier = Notifier() self.sched = MessageScheduler(self.message_handler) conn = Connection('aggregator', 'roflcake') entity_prefix, entity_suffix = conn.get_entity_name() self.entity_name = entity_prefix + entity_suffix self.log = Logging(conn) conn.join_muc('aggregators') self.conn = conn.get_conn() self.roster = self.conn.getRoster() self.conn.RegisterHandler('iq',self.set_handler,'set') self.conn.RegisterHandler('iq',self.get_handler,'get') self.conn.RegisterHandler('iq',self.result_handler,'result') self.conn.RegisterHandler('presence',self.presence_handler) self.temp_messages = [] self.parser = Parser() self.go_on() """ Handler for scheduler """ def message_handler(self, message, retry=False): #print 'Sending message.' if retry == True: self.log.error('Timed out, attempting to resend.') self.conn.send(message) # HANDLERS """ IQ result handler """ def result_handler(self, conn, iq_node): if self.sched.is_managed(int(iq_node.getID())): self.sched.received_response(iq_node) raise NodeProcessed """ Presence handler """ def presence_handler(self, conn, presence_node): if len(self.job_map) > 0: sender = presence_node.getFrom() if presence_node.getAttr('type') == 'unavailable': failed_poller = None for poller in self.job_map: if poller == sender: failed_poller = poller break if failed_poller != None: # Only used if Controller has gone offline self.log.info('Poller %s has gone offline.' % failed_poller) raise NodeProcessed """ IQ get handler """ def get_handler(self, conn, iq_node): if iq_node.getQueryNS() == NS_DISCO_INFO: reply = iq_node.buildReply('result') identity = Node('identity', {'category':'skynet', 'type':'aggregator'}) reply.setQueryPayload([identity]) conn.send(reply) raise NodeProcessed """ IQ set handler. Permits RPC calls in the whitelist, else returns error message """ def set_handler(self, conn, iq_node): sender = iq_node.getFrom() iq_id = iq_node.getAttr('id') if sender == '[email protected]/skynet': query_node = iq_node.getQueryChildren() for node in query_node: try: method = node.getTagData('methodName') method_whitelist = ['run_job', 'add_poller', 'remove_poller', 'remove_job', 'move_job'] if method in method_whitelist: method = getattr(self, method) try: try: params = node.getTag('params').getChildren() args = self.parser.get_args(params) except AttributeError: args = [] status, parameters = apply(method, args) message = self.parser.rpc_response(iq_node.getFrom(), iq_node.getID(), status, parameters) conn.send(message) except TypeError: #print sys.exc_info() conn.send(iq_node.buildReply('error')) else: conn.send(iq_node.buildReply('error')) self.log.error('Method called not in whitelist') except AttributeError: #print sys.exc_info() conn.send(iq_node.buildReply('error')) if len(self.job_map) > 0: if sender in self.job_map: query_node = iq_node.getQueryChildren() for node in query_node: try: method = node.getTagData('methodName') method_whitelist = ['add_result'] if method in method_whitelist: method = getattr(self, method) try: try: params = node.getTag('params').getChildren() args = self.parser.get_args(params) except AttributeError: args = [] status, parameters = apply(method, args) message = self.parser.rpc_response(iq_node.getFrom(), iq_node.getID(), status, parameters) conn.send(message) except TypeError: #print sys.exc_info() conn.send(iq_node.buildReply('error')) else: conn.send(iq_node.buildReply('error')) except AttributeError: #print sys.exc_info() conn.send(iq_node.buildReply('error')) raise NodeProcessed # This stanza is fully processed """ Establish evaluations """ def set_evals(self, job, evaluations): for evaluation in evaluations: if evaluation.string != None: self.evals[job].append((evaluation.comparison, str(evaluation.string))) elif evaluation.float != None: self.evals[job].append((evaluation.comparison, float(evaluation.float))) elif evaluation.int != None: self.evals[job].append((evaluation.comparison, int(evaluation.int))) else: self.log.info('Evaluation contains no comparison value.') """ Callback handler used by scheduler.add_message when poller replies on successful job assignment """ def assign_job(self, sender, query_node): if query_node.getNamespace() == NS_RPC: params = query_node.getTag('methodResponse').getTag('params').getChildren() job_id = self.parser.get_args_no_sender(params)[0] job_id = int(job_id) self.job_map[JID(sender)].append(job_id) self.evals[job_id] = [] evaluations = self.db.get_evaluations(job_id) self.set_evals(job_id, evaluations) else: pass # print 'Receieved iq message with incorrect namespace' # END HANDLERS # RPC METHODS """ Retains details of a job, allocates to Poller with least currently assinged jobs """ def run_job(self, sender, poller, job, addr, proto, freq, dom, resource): # Checks a poller is assigned if len(self.job_map) > 0: # Determines which poller has least assigned jobs # job_comp = None # least_loaded = None # for poller, jobs in self.job_map.items(): # num_jobs = len(jobs) # if job_comp != None: # if len(jobs) < job_comp: # least_loaded = poller # else: # least_loaded = poller # job_comp = num_jobs message = self.parser.rpc_call(poller, 'run_job', [self.entity_name, job, addr, proto, freq, dom, resource]) self.sched.add_message(message, self.assign_job) return 'success', [str(poller), int(job)] else: return 'failure', ['There are no pollers connected'] """ Called when a job is moved to this Aggregator, sets up evals and details """ def move_job(self, sender, poller, job, addr, proto, freq, dom, resource, segment): # Checks a poller is assigned if len(self.job_map) > 0: job_id = int(job) self.job_map[JID(poller)].append(job_id) self.evals[job_id] = [] evaluations = self.db.get_evaluations(job_id) self.set_evals(job_id, evaluations) return 'success', ['Successfully moved job'] else: return 'failure', ['There are no pollers connected'] """ Removes job from the Aggregator, cleans up state """ def remove_job(self, sender, job_id): job_id = int(job_id) parent_poller = None for poller, jobs in self.job_map.items(): for i in range(len(jobs)): if jobs[i] == job_id: jobs.pop(i) parent_poller = poller break try: self.evals.pop(job_id) except KeyError: pass if parent_poller != None: message = self.parser.rpc_call(parent_poller, 'remove_job', [job_id]) self.sched.add_message(message) return 'success', ['Successfully removed job'] else: return 'failure', ['Failed to remove job'] """ Called by child Poller to deliver result """ def add_result(self, sender, id, recorded, val): status = self.insert_result(id, recorded, val) if status != 'failure': return 'success', ['Sucessfully added result'] else: messages = self.temp_messages self.temp_messages = [] return 'failure', messages """ Called by Controller when Poller is assigned """ def add_poller(self, sender, poller): # Subscribe to poller for presence updates poller_jid = JID(poller) self.roster.Subscribe(poller_jid) self.job_map[poller_jid] = [] self.sched.add_message(self.parser.rpc_call(poller_jid, 'set_aggregator', [self.entity_name])) return 'success', ['Successfully added %s' % poller_jid] """ Called by Controller to remove references to Poller """ def remove_poller(self, sender, poller): poller_jid = JID(poller) try: unassigned_jobs = self.job_map.pop(poller_jid) # If controller has also failed # for job in unassigned_jobs: # self.job_pool.append(job) self.roster.Unsubscribe(poller_jid) return 'success', [poller] except KeyError: return 'failure', ['Failed to remove poller %s' % poller] # END RPC METHODS """ Used when inserting results supplied by add_result. Peforms evaluations, casts type, makes notifications and then stores into the database. """ def insert_result(self, id, recorded, val, list_id=None): val_type = type(val).__name__ try: evals = self.evals.get(int(id)) if evals: for comparison, comp_val in evals: if val_type == 'int' or val_type == 'float': eval_statement = str(val) + str(comparison) + str(comp_val) elif val_type == 'str': eval_statement = str('\'' + val + '\'') + str(comparison) + str('\'' + comp_val + '\'') # print 'Eval statement: %s' % eval_statement result = eval(eval_statement) # print 'Eval result: %s' % result if result != True: message = 'Job %s has caused an error! The value %s failed an evaluation.' % (id, comp_val) self.log.error(message) if id not in self.failed_jobs: self.log.info('Sending notifications') self.notifier.send_email(message) self.notifier.send_sms(message) self.failed_jobs.append(id) else: if id in self.failed_jobs: message = 'Job %s is back within normal parameters' % id self.notifier.send_email(message) self.log.info(message) self.failed_jobs.remove(id) except: # traceback.print_exc() self.temp_messages = ['Failed to evaluate returned result'] return 'failure' if val_type == 'int': if list_id != None: self.results.insert().execute(job=id, int=val, recorded=recorded, list=list_id) else: self.results.insert().execute(job=id, int=val, recorded=recorded) elif val_type == 'str': if list_id != None: self.results.insert().execute(job=id, string=val, recorded=recorded, list=list_id) else: self.results.insert().execute(job=id, string=val, recorded=recorded) elif val_type == 'float': if list_item != None: self.results.insert().execute(job=id, float=val, recorded=recorded, list=list_id) else: self.results.insert().execute(job=id, float=val, recorded=recorded) elif val_type == 'list': self.results.insert().execute(job=id, recorded=recorded, list=0) where = and_(self.results.c.recorded == recorded, self.results.c.list == 0) list_id = self.results.select(where).execute().fetchone().id #print "Retrieved list id %s" % list_id for element in val: self.insert_result(id, recorded, element, list_id) else: self.temp_messages = ['Unexpected data type receieved'] return 'failure' """ Setup listener """ def step_on(self): try: self.conn.Process(1) except KeyboardInterrupt: server = 'quae.co.uk' features.unregister(self.conn, server) #print 'Unregistered from %s' % server return 0 return 1 def go_on(self): while self.step_on(): pass
'mortgager': mortgager_id, 'beneficiary': beneficiary_id }) except Exception as e: logger.error( "block num: {}, op_id: {}, db except: '{}'".format( operation["block_num"], op_id, repr(e))) conn.close() if __name__ == '__main__': logger.info('args: {}'.format(sys.argv)) if len(sys.argv) < 3: logger.error( 'Usage: python3 check.py block_number_start, block_number_end') sys.exit(1) start = int(sys.argv[1]) end = int(sys.argv[2]) if start > end or start <= 0 or end <= 0: logger.error( 'block_number_start: {} > block_number_end: {} or start <= 0 or end <= 0' .format(start, end)) sys.exit(1) args = [start, end] t1 = Thread(target=check_block, args=(args, )) t1.start() t2 = Thread(target=data2db) t2.start()
class Controller: """ Controller object. Initializes various data structures used by object. Establishes connection with XMPP server, connects to poller, aggregator and logging Multi-User Chats and registers stanza handlers. """ def __init__(self): self.db = Database() entity_prefix = 'controller' conn = Connection(entity_prefix, static=True) self.entity_name, self.entity_suffix = conn.get_entity_name() # List of nodes known to the controller self.poller_map = {} self.poller_pool = {} self.job_map = {} self.job_pool = [] # Message scheduler self.sched = MessageScheduler(self.message_handler) self.log = Logging(conn) conn.join_muc('pollers') conn.join_muc('aggregators') self.parser = Parser() self.establish_jobs() self.conn = conn.get_conn() self.conn.RegisterHandler('iq',self.result_handler,'result') self.conn.RegisterHandler('iq',self.set_handler,'set') self.conn.RegisterHandler('presence',self.presence_handler) self.go_on() """ Called by the presence handler when an entity connects to the aggregator or poller MUCs. Used to inspect retreive service information from an entity, required in the XEP Jabber-RPC standard. """ def disco_lookup(self, recipient): self.log.info('Performing discovery lookup.') message = Iq('get', queryNS=NS_DISCO_INFO, to=recipient) self.sched.add_message(message, self.disco_handler) """ Method passed and used as handler for messages by MessageScheduler. Sends messages and logs an error if message send is a retry """ # Handler used by message scheduling class def message_handler(self, message, retry=False): # print 'Sending message.' if retry == True: self.log.error('Message timed out, resending.') self.conn.send(message) # # MESSAGE HANDLERS # """ Handler for presence stanzas recieved by the XMPP listener. If """ def presence_handler(self, conn, presence_node): sender = presence_node.getFrom() presence_type = presence_node.getAttr('type') # Ignore self and presence announcements from logging MUC if sender.getResource() != 'controller': if presence_type == 'unavailable': if sender.getNode() == 'aggregators' or sender.getNode() == 'pollers': self.remove_entity(sender.getNode(), sender) elif sender.getNode() == 'aggregators' or sender.getNode() == 'pollers': # Check the service discovery details for a connecting node. self.disco_lookup(sender) raise NodeProcessed """ IQ set handler, runs RPC methods in whitelist """ def set_handler(self, conn, iq_node): query_node = iq_node.getQueryChildren() for node in query_node: try: method = node.getTagData('methodName') method_whitelist = ['get_group', 'get_groups', 'create_group', 'update_group', 'remove_group', 'get_monitor', 'get_monitors', 'create_monitor', 'update_monitor', 'remove_monitor', 'get_monitors_by_gid', 'get_job', 'get_jobs', 'create_job', 'update_job', 'remove_job', 'get_evaluation', 'get_evaluations', 'create_evaluation', 'update_evaluation', 'remove_evaluation', 'get_results', 'get_results_day', 'get_results_week', 'get_results_hour', 'poller_failure', 'get_aggregator'] if method in method_whitelist: method = getattr(self, method) try: try: params = node.getTag('params').getChildren() args = self.parser.get_args(params, iq_node.getFrom()) except AttributeError: args = [] status, parameters = apply(method, args) message = self.parser.rpc_response(iq_node.getFrom(), iq_node.getID(), status, parameters) self.conn.send(message) except TypeError: # print sys.exc_info() conn.send(iq_node.buildReply('error')) else: conn.send(iq_node.buildReply('error')) self.log.error('Method not in whitelist') except AttributeError: traceback.print_exc() conn.send(iq_node.buildReply('error')) raise NodeProcessed def result_handler(self, conn, iq_node): # Check if the reponse is managed by scheduler if self.sched.is_managed(int(iq_node.getAttr('id'))): self.sched.received_response(iq_node) raise NodeProcessed # END MESSAGE HANDLERS # # BEGIN SCHEDULER RESPONSE HANDLERS # def disco_handler(self, sender, query_node): if query_node.getNamespace() == NS_DISCO_INFO: entity_type = query_node.getTagAttr('identity', 'type') if entity_type == 'aggregator' or entity_type == 'poller': adjusted_jid = JID(sender.getResource() + '@quae.co.uk/skynet') category = query_node.getTagAttr('identity', 'category') self.log.info('Registering node %s' % adjusted_jid) self.add_entity(entity_type, category, adjusted_jid) else: self.log.error('Receieved iq message with incorrect namespace') def assign_job(self, sender, query_node): if query_node.getNamespace() == NS_RPC: poller, job_id = self.parser.get_args_no_sender(query_node.getTag('methodResponse').getTag('params').getChildren()) poller_jid = JID(poller) job_id = int(job_id) job = None for i in range(len(self.job_pool)): if self.job_pool[i]['id'] == job_id: job = self.job_pool.pop(i) self.log.info('Removing job %s from the job pool' % job_id) break if job != None: self.job_map[poller_jid].append(job) self.log.info('Job %s successfully assigned to %s' % (job_id, poller_jid)) else: self.log.error('Receieved iq message with incorrect namespace') def poller_removed(self, sender, query_node): if query_node.getNamespace() == NS_RPC: args = self.parser.get_args_no_sender(query_node.getTag('methodResponse').getTag('params').getChildren()) adjusted_jid = JID(args[0]) unassigned_jobs = self.job_map.pop(adjusted_jid) for job in unassigned_jobs: self.log.info('Adding job %s to the job pool' % job['id']) self.job_pool.append(job) parent_poller = None for aggregator, pollers in self.poller_map.items(): for poller, segment in pollers: if poller == adjusted_jid: parent_aggregator = aggregator pollers.remove((poller, segment)) break self.log.info('Removed %s from %s' % (adjusted_jid, parent_aggregator)) self.assign_pooled_jobs() else: self.log.error('Receieved iq message with incorrect namespace') # END SCHEDULER HANDLERS # # BEGIN RPC METHODS # # Requested by an aggregator when an assigned poller has failed/disconnected. def poller_failure(self, sender, previous_poller): pollers = self.poller_map[JID(sender)] poller_jid = JID('[email protected]/' + JID(previous_poller).getNode()) try: pollers.remove(poller_jid) message = 'Removed failed poller' try: self.rebalance_pollers() except: print sys.exc_info() return 'success', [message] except: return 'failure', ['Failed to remove poller'] # Group operations def get_group(self, sender, name): group = self.db.get_group_by_name(name) if group != None: return 'success', [group] return 'failure', ['No such group exists'] def get_groups(self, sender): groups = self.db.get_groups() if groups != None: return 'success', [groups] return 'failure', ['Failed to retreieve groups'] def create_group(self, sender, name, desc): existing = self.db.get_group_by_name(name) if existing == None: self.db.create_group(name, desc) return 'success', ['Sucessfully created group %s' % name] return 'failure', ['failure'] def update_group(self, sender, id, name, desc): group = self.db.get_group_by_id(id) if group != None: self.db.update_group(id, name, desc) return 'success', ['Successfully update group %s' % name] return 'failure', ['Failed to update group'] def remove_group(self, sender, name): self.db.remove_group_by_name(name) if self.db.get_group_by_name(name) == None: return 'success', ['Successfully remove group %s' % name] else: return 'failure', ['Failed to remove group %s' % name] # Monitor operations def get_monitor(self, sender, name): try: monitor = self.db.get_monitor(name) if monitor != False: return 'success', [monitor] except TypeError: return 'failure', ['No such monitor exists'] return 'failure', ['No such monitor exists'] def get_monitors(self, sender, group=None): try: if group != None: monitors = self.db.get_monitors(group) else: monitors = self.db.get_monitors() return 'success', [monitors] except AttributeError: return 'failure', ['Failed to retrieve monitors'] def get_monitors_by_gid(self, sender, group_id): try: if group_id != None: monitors = self.db.get_monitors_by_gid(group_id) return 'success', [monitors] except AttributeError: pass return 'failure', ['Failed to retrieve monitors'] def create_monitor(self, sender, name, description, group): if self.db.create_monitor(name, description, group) == True: return 'success', ['Successfully create monitor %s' % name] return 'failure', ['Failed to create monitor'] def update_monitor(self, sender, name, description, group): group = self.db.get_group_by_id(id) if group != None: self.db.update_group(id, name, desc) return 'success', ['Successfully update monitor %s' % name] return 'failure', ['Failed to update monitor'] def remove_monitor(self, sender, name): self.db.remove_monitor_by_name(name) if self.db.get_monitor_by_name(name) == None: return 'success', ['Successfully removed monitor %s' % name] else: return 'failure', ['Failed to remove monitor %s' % name] # job operations def get_job(self, sender, mon, id): try: job = self.db.get_job(id, mon) return 'success', [job] except TypeError: return 'failure', ['No such job exists'] def get_jobs(self, sender, mon): try: jobs = self.db.get_jobs(mon) return 'success', [jobs] except AttributeError: return 'failure', ['Failed to retreieve jobs'] def create_job(self, sender, mon, address, protocol, frequency, interface, resource): if self.db.get_monitor(mon) != None: if self.db.create_job(address, protocol, frequency, interface, resource, mon) == True: return 'success', ['Successfully created a job for %s' % mon] return 'failure', ['Failed to create job'] def update_job(self, sender, mon, id, address, protocol, frequency, interface, resource): existing = self.db.get_job(id, mon) if existing != None: self.db.update_job(id, address, protocol, frequency, interface, resource) return 'success', ['Successfully updated job'] else: return 'failure', ['failure'] def remove_job(self, sender, mon, id): self.db.remove_job(id) if self.db.get_job(id) == None: return 'success', ['Successfully removed job'] else: return 'failure', ['failure'] # Result read operations def get_results(self, sender, monitor, job, start_datetime, end_datetime): results = self.db.get_results(monitor, job, start_datetime, end_datetime) if results == None: return 'failure', ['No such results exist'] elif results != False: return 'success', results return 'failure', ['Failed to retreive specificied results'] def get_results_day(self, sender, monitor, job, start_datetime): results = self.db.get_results_day(monitor, job, start_datetime) if results == []: return 'failure', ['No such results exist'] elif results != False: job_details = self.db.get_job(job, monitor) return 'success', [job_details, results] return 'failure', ['Failed to retreive specificied results'] def get_results_week(self, sender, monitor, job, start_datetime): pass def get_results_hour(self, sender, monitor, job, start_datetime): results = self.db.get_results_hour(monitor, job, start_datetime) if results == []: return 'failure', ['No such results exist'] elif results != False: job_details = self.db.get_job(job, monitor) return 'success', [job_details, results] return 'failure', ['Failed to retreive specificied results'] # END RPC METHODS # # BEGIN PRIVATE METHODS # """ Called on successful DISCO request. Will register Poller or Aggregator, send jobs or balance jobs. """ def add_entity(self, entity_type, segment, entity): if entity_type == 'aggregator': self.poller_map[entity] = [] # If pollers have been added, but there were no aggregators running self.assign_pooled_pollers() if len(self.poller_map) > 1: self.rebalance_pollers() self.assign_pooled_jobs() elif entity_type == 'poller': self.job_map[entity] = [] self.poller_pool[entity] = segment self.assign_pooled_pollers() if len(self.poller_map) > 1: self.rebalance_pollers() self.assign_pooled_jobs() if len(self.job_map) > 1: self.rebalance_jobs() # Give poller to appropriate aggregator print 'Added %s to %ss' % (entity, entity_type) self.log.info('Node %s was successfully registered with the controller' % entity) """ Removing Poller or Aggregator """ def remove_entity(self, entity_type, entity): try: if entity_type == 'aggregators': adjusted_jid = JID(JID(entity).getResource() + '@quae.co.uk/skynet') unassigned_pollers = self.poller_map.pop(adjusted_jid) for poller, segment in unassigned_pollers: self.poller_pool[poller] = segment self.log.info('Removed %s' % adjusted_jid) if len(self.poller_pool) > 0: # Try and assign pooled pollers if not self.assign_pooled_pollers(): for poller, segment in unassigned_pollers: message = self.parser.rpc_call(poller, 'aggregator_failure', []) self.sched.add_message(message) elif entity_type == 'pollers': adjusted_jid = JID(JID(entity).getResource() + '@quae.co.uk/skynet') if len(self.poller_map) > 0: parent_aggregator = None for aggregator, pollers in self.poller_map.items(): for poller, segment in pollers: if adjusted_jid == poller: parent_aggregator = aggregator break if parent_aggregator != None: remove_call = self.parser.rpc_call(parent_aggregator, 'remove_poller', [str(adjusted_jid)]) self.sched.add_message(remove_call, self.poller_removed) else: try: self.job_map.pop(adjusted_jid) self.poller_pool.pop(adjusted_jid) self.log.info('Poller not assigned, sucessfully removed') except: self.log.error('Failed to remove poller') traceback.print_exc() except ValueError: self.log.error('Failed to remove %s' % entity) """ Assign unassigned pollers """ def assign_pooled_pollers(self): if len(self.poller_map) > 0: while len(self.poller_pool) > 0: unassigned_poller, segment = self.poller_pool.popitem() chosen_aggregator = None poller_comp = None for aggregator, pollers in self.poller_map.items(): # If first loop or number of pollers assigned to aggregator is less than comp, make this agg the comp if (chosen_aggregator == None and poller_comp == None) or len(pollers) < poller_comp: chosen_aggregator = aggregator poller_comp = len(pollers) if chosen_aggregator != None: # Assign Poller to the Aggregtor with least assigned Pollers message = self.parser.rpc_call(chosen_aggregator, 'add_poller', [str(unassigned_poller)]) self.sched.add_message(message) for job in self.job_map[unassigned_poller]: message = self.parser.rpc_call(chosen_aggregator, 'move_job', [str(unassigned_poller), job['id'], job['address'], job['protocol'], job['frequency'], job['interface'], job['resource'], job['segment']]) self.sched.add_message(message) self.poller_map[chosen_aggregator].append((unassigned_poller, segment)) return True else: self.log.info('No aggregators available for poller assignment') return False """ Get Pollers for a given network segment """ def get_segment_pollers(self, segment): segment_pollers = {} for aggregator, pollers in self.poller_map.items(): for poller, poller_segment in pollers: if poller_segment == segment: segment_pollers[poller] = self.job_map[poller] return segment_pollers """ Called to allocate unassigned jobs """ def assign_pooled_jobs(self): if len(self.job_map) > 0 and len(self.poller_map) > 0: for job in self.job_pool: unassigned_job = job least_loaded = None job_comp = None pollers = self.get_segment_pollers(unassigned_job['segment']) for poller, jobs in pollers.items(): # for poller, jobs in self.job_map.items(): if (least_loaded == None and job_comp == None) or len(jobs) < job_comp: least_loaded = poller job_comp = len(jobs) if least_loaded != None: chosen_aggregator = None for aggregator, pollers in self.poller_map.items(): for poller, segment in pollers: if poller == least_loaded: chosen_aggregator = aggregator break if chosen_aggregator != None: self.send_job(unassigned_job, least_loaded, chosen_aggregator) else: self.log.info('No assigned pollers available for job assignment') #print 'Job map %s' % self.job_map #print 'Job pool %s' % self.job_pool """ Rebalance pollers, compares amount assigned to each Aggregator, and moves across to another Poller if there's at least 2 more than another Aggregator """ def rebalance_pollers(self): self.log.info('Attempting to rebalance pollers') poller_comp = None least_pollers = None most_pollers = None # Retrieve aggregators with least and most pollers for aggregator, pollers in self.poller_map.items(): if poller_comp == None: least_pollers = aggregator most_pollers = aggregator elif len(pollers) < poller_comp: least_pollers = aggregator elif len(pollers) > poller_comp: most_pollers = aggregator poller_comp = len(pollers) if least_pollers != None and most_pollers != None: # If the difference between the two pollers is worth balancing if (len(self.poller_map[most_pollers]) - len(self.poller_map[least_pollers])) > 1: poller, segment = self.poller_map[most_pollers].pop() self.poller_map[least_pollers].append((poller, segment)) self.sched.add_message(self.parser.rpc_call(least_pollers, 'add_poller', [str(poller)])) self.sched.add_message(self.parser.rpc_call(most_pollers, 'remove_poller', [str(poller)])) self.rebalance_pollers() else: self.log.info('Pollers balanced') return True """ Similar to above, checks number of assigned jobs through the system, and will level them across all Pollers """ def rebalance_jobs(self): self.log.info('Attempting to rebalance jobs') job_comp = None least_jobs = None most_jobs = None network_segment = 'skynet' pollers = self.get_segment_pollers(network_segment) # for poller, jobs in self.job_map.items(): for poller, jobs in pollers.items(): print poller if job_comp == None: least_jobs = poller most_jobs = poller elif len(jobs) < job_comp: least_jobs = poller elif len(jobs) > job_comp: most_jobs = poller job_comp = len(jobs) if least_jobs != None and most_jobs != None: if (len(self.job_map[most_jobs]) - len(self.job_map[least_jobs])) > 1: job = self.job_map[most_jobs].pop() self.job_map[least_jobs].append(job) least_parent = None most_parent = None for aggregator, pollers in self.poller_map.items(): for poller, node_segment in pollers: if network_segment == node_segment: if poller == least_jobs: least_parent = aggregator if poller == most_jobs: most_parent = aggregator if least_parent != None and most_parent != None: break if least_parent != None and most_parent != None: self.log.info('Moving job %s to %s' % (job['id'], least_jobs)) self.sched.add_message(self.parser.rpc_call(most_parent, 'remove_job', [job['id']])) self.sched.add_message(self.parser.rpc_call(least_parent, 'run_job', [str(least_jobs), job['id'], job['address'], job['protocol'], job['frequency'], job['interface'], job['resource']]), offset=True) self.rebalance_jobs() else: self.log.info('Jobs balanced') return True """ Retrieve jobs on startup """ def establish_jobs(self): monitors = self.db.get_monitors() self.log.info('Retrieving jobs') for monitor in monitors: jobs = self.db.get_jobs(monitor.name) for job in jobs: job = dict(job) segment_name = self.db.get_segment_name(job['segment']) job['segment'] = segment_name # Make the poll freq stored every minute minimum job['frequency'] = job['frequency'] * 60 self.job_pool.append(job) self.log.info('%s jobs added to the pool' % len(self.job_pool)) """ Send job to Aggregator to forward to Poller """ def send_job(self, job, poller, aggregator): message = self.parser.rpc_call(aggregator, 'run_job', [str(poller), job['id'], job['address'], job['protocol'], job['frequency'], job['interface'], job['resource']]) self.log.info('Sending job %s to %s' % (job['id'], aggregator)) self.sched.add_message(message, self.assign_job, offset=True) def step_on(self): try: self.conn.Process(1) except KeyboardInterrupt: return 0 return 1 def go_on(self): while self.step_on(): pass