def doprovisiondispatch(miner_type=None): '''put all miners in provision worker queue''' entries = QueueEntries() miners = PROVISION_DISPATCH.app.allminers() print("{0} miners configured".format(len(miners))) for miner in miners: if miner.is_disabled(): continue try: minerstats, minerinfo, apicall, minerpool = stats(miner) if miner_type is not None and miner_type != '' and minerinfo.miner_type != miner_type: continue mineraccess = PROVISION_DISPATCH.app.antminer.getaccesslevel(miner) if mineraccess == MinerAccessLevel.Restricted: print( Fore.RED + " Log: setting {0} to privileged...".format(miner.name)) PROVISION_DISPATCH.app.antminer.set_privileged(miner) PROVISION_DISPATCH.app.antminer.waitforonline(miner) mineraccess = PROVISION_DISPATCH.app.antminer.getaccesslevel( miner) print(Fore.GREEN + "{0} {1} {2}".format( miner.name, minerinfo.miner_type, mineraccess)) if mineraccess == MinerAccessLevel.Restricted: print(" skipping restricted access") else: entries.add(QueueName.Q_PROVISION, PROVISION_DISPATCH.app.messageencode(miner)) except MinerMonitorException as minerex: print(minerex) except BaseException as ex: print('while provisioning {0} ({1})'.format( miner.ipaddress, miner.key())) print(PROVISION_DISPATCH.app.exceptionmessage(ex)) return entries
def __init__(self): self.minerstotal = 0 self.minersnew = 0 self.shownonminers = True self.hostsup = 0 self.entries = QueueEntries() self.knownminers = None self.hosts_list = None
def test_queue_entries(self): que = QueueEntries() self.assertTrue(que) que.add('test', 'test') que.addbroadcast('qbroad', 'test') que.addalert('msg') self.assertTrue(que.hasentries())
def findminers(hosts_list, knownminers): '''find miners on network''' entries = QueueEntries() minerstotal = 0 minersnew = 0 shownonminers = True hostsup = 0 print('Querying {0} hosts...'.format(len(hosts_list))) for host, status, macaddress in hosts_list: try: if status != 'down': hostsup += 1 if shownonminers: print("{0} {1} {2}".format(host, status, macaddress)) miner = mining.Miner(name=host, ipaddress=host, port=MINERPORT, ftpport='', networkid=macaddress) try: minerstats, minerinfo, apicall, minerpool = antminerhelper.stats(miner) miner.setminerinfo(minerinfo) if minerinfo.miner_type: minerstotal += 1 if not shownonminers: print("{0} {1} {2}".format(host, status, macaddress)) print(Fore.GREEN + ' found {0} with id {1}'.format(minerinfo.miner_type, minerinfo.minerid)) #find by mac address or miner_id, not name found = None #if miner.minerid != "unknown" for known in knownminers: if known.networkid == miner.networkid or (miner.minerid != "unknown" and known.minerid == miner.minerid): found = known if found: print(Fore.YELLOW + ' already know about {0}'.format(found.name)) else: minersnew += 1 entries.add(QueueName.Q_DISCOVERED, DISCOVER.app.messageencode(miner)) print(Fore.GREEN + ' discovered {0}'.format(miner.name)) except antminerhelper.MinerMonitorException as monitorex: try: if monitorex.istimedout: if shownonminers: print(Fore.RED + ' Not a miner') except Exception: DISCOVER.app.logexception(monitorex) except BaseException as baseex: print(Fore.RED + DISCOVER.app.exceptionmessage(baseex)) except KeyboardInterrupt: break print('nmap queried {0} hosts on network'.format(len(hosts_list))) print('{0} hosts are up'.format(hostsup)) print('FCM knows about {0} miners configured'.format(len(knownminers))) print('FCM determined {0} miners this attempt'.format(minerstotal)) print('FCM determined there are {0} new miners on network'.format(minersnew)) return entries
def domonitor(): '''queue workers to run the individual miner monitoring''' entries = QueueEntries() try: miners = MONITOR.app.allminers() print("{0} miners configured".format(len(miners))) for miner in miners: if not miner.is_manually_disabled(): entries.add(QueueName.Q_MONITORMINER, MONITOR.app.messageencode(miner)) print("waiting for next monitor event") except Exception as theex: MONITOR.app.logexception(theex) return entries
def domonitorminer(miner): '''get statistics from miner''' entries = QueueEntries() savedminer = APPMONITOR.app.getminer(miner) if savedminer is None: savedminer = miner try: #individual miner can be monitored even if manually disabled #todo:savedminer and knownminer out of sync. this will be fixed in refactoring redis if not savedminer.should_monitor() and not miner.should_monitor(): print('skipped monitoring {0}'.format(miner.name)) return entries mineroriginalstatus = savedminer.status minerstats, minerinfo, apicall, minerpool = stats(savedminer) #minerlcd = antminerhelper.getminerlcd(miner) if minerstats is None: print('could not monitor {0}({1})'.format(savedminer.name, savedminer.ipaddress)) else: process_stats(entries, savedminer, mineroriginalstatus, minerstats, minerpool, minerinfo, apicall) except pika.exceptions.ConnectionClosed as qex: #could not enqueue a message print(Fore.RED + '{0} Queue Error: {1}'.format( savedminer.name, APPMONITOR.app.exceptionmessage(qex))) APPMONITOR.app.logexception(qex) except MinerMonitorException as monitorex: print(Fore.RED + '{0} Miner Error: {1}'.format( savedminer.name, APPMONITOR.app.exceptionmessage(monitorex))) savedminer.lastmonitor = datetime.datetime.utcnow() #TODO: this should be a rule. publish miner offline event #and let event handler decide how to handle it savedminer.offline_now() print(Fore.RED + APPMONITOR.app.now(), savedminer.name, savedminer.status) entries.add(QueueName.Q_OFFLINE, APPMONITOR.app.messageencode(savedminer)) except BaseException as ex: print(Fore.RED + '{0} Unexpected Error in monitorminer: {1}'.format( savedminer.name, APPMONITOR.app.exceptionmessage(ex))) # we have to consider any exception to be a miner error. sets status to offline #if str(e) == "timed out": #(timeout('timed out',),) APPMONITOR.app.logexception(ex) #TODO: review usage of savedminer and knownminer. should only go with one APPMONITOR.app.putminer(savedminer) APPMONITOR.app.updateknownminer(savedminer) return entries
class DiscoveryResults: def __init__(self): self.minerstotal = 0 self.minersnew = 0 self.shownonminers = True self.hostsup = 0 self.entries = QueueEntries() self.knownminers = None self.hosts_list = None def process_miner(self, miner, status): minerstats, minerinfo, apicall, minerpool = antminerhelper.stats(miner) miner.setminerinfo(minerinfo) if minerinfo.miner_type: self.minerstotal += 1 if not self.shownonminers: print("{0} {1} {2}".format(miner.ipaddress, status, miner.networkid)) print(Fore.GREEN + ' found {0} with id {1}'.format( minerinfo.miner_type, minerinfo.minerid)) #find by mac address or miner_id, not name found = None #if miner.minerid != "unknown" for known in self.knownminers: if known.networkid == miner.networkid or (not miner.is_unknown and known.minerid == miner.minerid): found = known if found: print(Fore.YELLOW + ' already know about {0}'.format(found.name)) else: self.minersnew += 1 self.entries.add(QueueName.Q_DISCOVERED, DISCOVER.app.messageencode(miner)) print(Fore.GREEN + ' discovered {0}'.format(miner.name)) def print(self): print('nmap queried {0} hosts on network'.format(len(self.hosts_list))) print('{0} hosts are up'.format(self.hostsup)) print('FCM knows about {0} miners configured'.format( len(self.knownminers))) print('FCM determined {0} miners this attempt'.format( self.minerstotal)) print('FCM determined there are {0} new miners on network'.format( self.minersnew))
def doprovisiondispatch(miner_type=None): '''put all miners in provision worker queue''' entries = QueueEntries() miners = PROVISION_DISPATCH.app.allminers() print("{0} miners configured".format(len(miners))) for miner in miners: if miner.is_disabled(): continue process_miner(miner, entries, miner_type) return entries
def dooffline(miner: mining.Miner): '''notify user''' entries = QueueEntries() savedminer = OFFLINE.app.getminer(miner) if not savedminer.is_disabled(): if savedminer.is_send_offline_alert(): #update status to offline and alert savedminer.status = mining.MinerStatus.Offline alertmsg = OFFLINE.app.stamp( 'miner {0} is offline! since {1}'.format( savedminer.name, savedminer.laststatuschanged)) OFFLINE.app.putminer(savedminer) entries.addalert(alertmsg) print("Sent offline alert '{0}'".format(alertmsg)) else: #stop annoying the user, disable the miner to stop sending alerts savedminer.status = mining.MinerStatus.Disabled alertmsg = OFFLINE.app.stamp( 'miner {0} is Disabled after many offline alerts. No more alerts will be sent for this miner.' .format(savedminer.name)) OFFLINE.app.putminer(savedminer) entries.addalert(alertmsg) print("Sent disabled alert for {0}".format(savedminer.name)) else: if not savedminer.is_manually_disabled(): print( 'Disabled miner {0} is offline. manually disable miner to get rid of this message' .format(miner.name)) return entries
def rules(miner, minerstats, minerpool): '''this runs the rules''' entries = QueueEntries() if miner is None or minerstats is None: return entries savedminer = RULES.app.getminer(miner) broken = gather_broken_rules(savedminer, minerstats) if broken: #TODO: could raise broken rule event??? for rule in broken: RULES.app.log_mineractivity(make_log_from_rule(rule)) add_entry_for_rule(entries, rule) return entries
def dosave(msg): entries = QueueEntries() if msg.entity == 'miner': miner = saveminer(msg) entries.add(QueueName.Q_MONITORMINER, COMPONENTSAVE.app.messageencode(miner)) entries.add(QueueName.Q_PROVISION, COMPONENTSAVE.app.messageencode(miner)) if msg.entity == 'pool': savepool(msg) return entries
def rules(miner, minerstats, minerpool): '''this runs the rules''' entries = QueueEntries() if miner is None or minerstats is None: return entries savedminer = RULES.app.getminer(miner) cmd_restart = MinerCommand('restart', '') broken = [] for ruleparms in RULES.app.ruleparameters(): rule = MinerStatisticsRule(savedminer, minerstats, ruleparms) if rule.isbroken(): broken += rule.brokenrules if broken: #TODO: could raise broken rule event??? for rule in broken: log = MinerLog() log.createdate = datetime.datetime.utcnow() log.minerid = rule.miner.key() log.minername = rule.miner.name log.action = rule.parameter RULES.app.log_mineractivity(log) if rule.action == 'alert': entries.addalert( RULES.addalert(RULES.app.stamp(rule.parameter))) elif rule.action == 'restart': entries.add( QueueName.Q_RESTART, RULES.app.createmessagecommand(rule.miner, cmd_restart)) entries.addalert( RULES.addalert( RULES.app.stamp('Restarted {0}'.format( rule.miner.name)))) else: RULES.app.logerror('did not process broken rule {0}'.format( rule.parameter)) return entries
def doonline(miner): '''then provision the miner''' entries = QueueEntries() savedminer = ONLINE.app.getminer(miner) if savedminer is None: savedminer = miner #update status savedminer.status = mining.MinerStatus.Online #sending message will also save it #just provision the miner and start to monitor entries.add(QueueName.Q_PROVISION, ONLINE.app.messageencode(savedminer)) #tell them something good happened msg = 'miner {0} is back online'.format(savedminer.name) entries.addalert(msg) print(msg) return entries
def doprovision(miner): '''provision/configure a miner''' entries = QueueEntries() poollist = PROVISION.app.pools.get_all_pools() print("{0} pools configured".format(len(poollist))) print('{0} {1}'.format(miner.name, miner.ipaddress)) mineraccess = '' addpools = None minerinfo = None minerpool = None try: minerstats, minerinfo, apicall, minerpool = antminerhelper.stats(miner) miner.setminerinfo(minerinfo) #find the current pool in known pools knownpool = PROVISION.app.pools.findpool(minerpool) if knownpool is not None: minerpool.poolname = knownpool.name miner.minerpool = minerpool PROVISION.app.updateknownminer(miner) #find pools that need to be added and add them addpools = services.poolstoadd(miner, minerpool, poollist) mineraccess = PROVISION.app.antminer.getaccesslevel(miner) except antminerhelper.MinerMonitorException as ex: if ex.istimedout(): mineraccess = MinerAccessLevel.Waiting if mineraccess == MinerAccessLevel.Restricted or mineraccess == MinerAccessLevel.Waiting: if mineraccess == MinerAccessLevel.Restricted: PROVISION.app.antminer.set_privileged(miner) PROVISION.app.antminer.waitforonline(miner) mineraccess = PROVISION.app.antminer.getaccesslevel(miner) if mineraccess == MinerAccessLevel.Restricted: entries.addalert('could not set {0} to privileged access'.format( miner.name)) #try a few more times then give up else: addpoolstominer(miner, addpools) addminerpools(miner) switchtodefaultpool(miner, poollist, minerpool) enforcedefaultpool(miner, poollist, minerpool) entries.add(QueueName.Q_MONITORMINER, PROVISION.app.messageencode(miner)) return entries
def doonline(miner): '''then provision the miner''' entries = QueueEntries() savedminer = ONLINE.app.getminer(miner) if savedminer is None: savedminer = miner #update status savedminer.online_now() ONLINE.app.putminer(savedminer) ONLINE.app.updateknownminer(savedminer) #just provision the miner and start to monitor entries.add(QueueName.Q_PROVISION, ONLINE.app.messageencode(savedminer)) #tell them something good happened msg = ONLINE.app.stamp('miner {0} is back online'.format(savedminer.name)) entries.addalert(msg) print(msg) return entries
def dodiscovered(miner): '''then provision it''' entries = QueueEntries() entries.add(QueueName.Q_PROVISION, COMPONENTDISCOVERED.app.messageencode(miner)) cachedminer = COMPONENTDISCOVERED.app.getminer(miner) #knownminer should be None if cachedminer is not None: cachedminer.updatefrom(miner) COMPONENTDISCOVERED.app.putminer(cachedminer) knownminer = COMPONENTDISCOVERED.app.getknownminer(miner) if knownminer is None: COMPONENTDISCOVERED.app.addknownminer(miner) else: COMPONENTDISCOVERED.app.updateknownminer(miner) entries.add(QueueName.Q_ALERT, 'discovered miner {0}'.format(miner.name)) print("Discovered {0}".format(miner.name)) return entries
def dosave(msg): entries = QueueEntries() if msg.entity == 'miner': #add or update miner minerid = name = ipaddress = port = None for pair in msg.values: if 'minerid' in pair: minerid = pair['minerid'] if 'name' in pair: name = pair['name'] if 'ipaddress' in pair: ipaddress = pair['ipaddress'] if 'port' in pair: port = pair['port'] miner = Miner(name, '', '', ipaddress, port, '', '') COMPONENTSAVE.app.save_miner(miner) entries.add(QueueName.Q_MONITORMINER, COMPONENTSAVE.app.messageencode(miner)) entries.add(QueueName.Q_PROVISION, COMPONENTSAVE.app.messageencode(miner)) if msg.entity == 'pool': #save the new named pool pool_type = name = url = user = priority = None for pair in msg.values: if 'pool_type' in pair: pool_type = pair['pool_type'] if 'name' in pair: name = pair['name'] if 'url' in pair: url = pair['url'] if 'user' in pair: user = pair['user'] if 'priority' in pair: priority = pair['priority'] pool = Pool(pool_type, name, url, user, priority) COMPONENTSAVE.app.save_pool(pool) return entries
def doprovision(miner): '''provision/configure a miner''' entries = QueueEntries() poollist = PROVISION.app.pools() print("{0} pools configured".format(len(poollist))) print('{0} {1}'.format(miner.name, miner.ipaddress)) mineraccess = '' addpools = None minerinfo = None minerpool = None try: minerinfo = antminerhelper.getminerinfo(miner) miner.minerinfo = minerinfo minerpool = antminerhelper.pools(miner) #find the current pool in known pools knownpool = PROVISION.app.findpool(minerpool) if knownpool is not None: minerpool.poolname = knownpool.name miner.minerpool = minerpool PROVISION.app.updateknownminer(miner) #find pools that need to be added and add them addpools = services.poolstoadd(miner, minerpool, poollist) mineraccess = PROVISION.app.antminer.getaccesslevel(miner) except antminerhelper.MinerMonitorException as ex: if ex.istimedout(): mineraccess = MinerAccessLevel.Waiting if mineraccess == MinerAccessLevel.Restricted or mineraccess == MinerAccessLevel.Waiting: if mineraccess == MinerAccessLevel.Restricted: PROVISION.app.antminer.set_privileged(miner) PROVISION.app.antminer.waitforonline(miner) mineraccess = PROVISION.app.antminer.getaccesslevel(miner) if mineraccess == MinerAccessLevel.Restricted: entries.addalert('could not set {0} to privileged access'.format( miner.name)) #try a few more times then give up else: for pool in addpools or []: print( Fore.YELLOW + " Add", pool.name, "(addpool|{0},{1},{2})".format(pool.url, pool.user + miner.name, "x")) #this command adds the pool to miner and prints the result result = antminerhelper.addpool(miner, pool) if result.startswith("Access denied"): print(Fore.RED + result) else: print(result) namedpools = PROVISION.app.pools() #process the pools found on the miner. This will pick up any pools added manually for pool in miner.pools_available: #check if pools is a named pool... foundnamed = None for namedpool in namedpools: if namedpool.is_same_as(pool): foundnamed = namedpool break if foundnamed: #pool should take on the cononical attributes of the named pool pool.named_pool = foundnamed pool.user = foundnamed.user PROVISION.app.add_pool(MinerPool(miner, pool.priority, pool)) #enforce default pool if miner has one set up if miner.defaultpool: founddefault = next( (p for p in poollist if p.name == miner.defaultpool), None) if founddefault is not None: switchtopool(miner, founddefault) #enforce default pool if it doesnt have one. find highest priority pool if not miner.defaultpool: def sort_by_priority(j): return j.priority filtered = [ x for x in poollist if miner.miner_type.startswith(x.pool_type) ] filtered.sort(key=sort_by_priority) #foundpriority = next((p for p in poollist if p.priority == 0), None) if filtered: switchtopool(miner, filtered[0]) entries.add(QueueName.Q_MONITORMINER, PROVISION.app.messageencode(miner)) return entries
''' A test program that pushes monitoring messages onto the queue A stress tester for monitoring when_monitorminer.py should be running to process the messages ''' from helpers.queuehelper import QueueName, QueueEntries from backend.fcmapp import ApplicationService, ComponentName APP = ApplicationService(ComponentName.fullcycle) MINERS = APP.miners() CNT = 1 while CNT < 2000: ENTRIES = QueueEntries() for miner in MINERS: ENTRIES.add(QueueName.Q_MONITORMINER, APP.messageencode(miner)) APP.enqueue(ENTRIES) print('sent {} messages'.format(len(ENTRIES.entries))) CNT += 1
''' A test program that pushes monitoring messages onto the queue A stress tester for monitoring when_monitorminer.py should be running to process the messages ''' from helpers.queuehelper import QueueName, QueueEntries from backend.fcmapp import ApplicationService, ComponentName APP = ApplicationService(ComponentName.fullcycle) MINERS = APP.miners() cnt = 1 while cnt < 1000: entries = QueueEntries() for miner in MINERS: entries.add(QueueName.Q_MONITORMINER, APP.messageencode(miner)) APP.enqueue(entries) print('sent {} messages'.format(len(entries.entries))) cnt += 1
def domonitorminer(miner): '''get statistics from miner''' entries = QueueEntries() savedminer = APPMONITOR.app.getminer(miner) if savedminer is None: savedminer = miner try: #individual miner can be monitored even if manually disabled #todo:savedminer and knownminer out of sync. this will be fixed in refactoring redis if not savedminer.should_monitor() and not miner.should_monitor(): print('skipped monitoring {0}'.format(miner.name)) return entries mineroriginalstatus = savedminer.status #minerinfo = getminerinfo(savedminer) minerstats, minerinfo, apicall, minerpool = stats(savedminer) #minerlcd = antminerhelper.getminerlcd(miner) if minerstats is None: print('could not monitor {0}'.format(savedminer.name)) else: #what to do if monitored miner type conflicts with saved miner type??? #should probably provision? foundpool = APPMONITOR.app.findpool(minerpool) if foundpool is not None: minerpool.poolname = foundpool.name savedminer.monitored(minerstats, minerpool, minerinfo, apicall.elapsed()) if mineroriginalstatus == '': #first time monitoring since bootup print(Fore.GREEN + APPMONITOR.app.now(), savedminer.name, 'first time monitoring') elif savedminer.status == mining.MinerStatus.Online and (mineroriginalstatus == mining.MinerStatus.Disabled or mineroriginalstatus == mining.MinerStatus.Offline): #changing status from offline to online so raise event entries.add(QueueName.Q_ONLINE, APPMONITOR.app.messageencode(savedminer)) print(Fore.GREEN + APPMONITOR.app.now(), savedminer.name, 'back online!') #TODO: if stats.elapsed < previous.elapsed then raise provision or online events APPMONITOR.app.putminerandstats(savedminer, minerstats, minerpool) #TODO:show name of current pool instead of worker poolname = '?' if minerpool: poolname = '{0} {1}'.format(minerpool.currentpool, minerpool.currentworker) foundpool = APPMONITOR.app.findpool(minerpool) if foundpool is not None: poolname = foundpool.name print('{0} mining at {1}'.format(savedminer.name, poolname)) #most users won't want to mine solo, so provision the miner if not APPMONITOR.app.configuration('mining.allowsolomining'): if not minerpool.currentpool or minerpool.currentpool.startswith(APPMONITOR.app.configuration('mining.solopool')): entries.add(QueueName.Q_PROVISION, APPMONITOR.app.messageencode(savedminer)) print(Fore.CYAN+str(APPMONITOR.app.now()), savedminer.name, savedminer.status, 'h='+str(minerstats.currenthash), str(minerstats.minercount), '{0}/{1}/{2}'.format(str(minerstats.tempboard1), str(minerstats.tempboard2), str(minerstats.tempboard3)), savedminer.uptime(minerstats.elapsed), '{0:d}ms'.format(int(savedminer.monitorresponsetime() * 1000))) msg = APPMONITOR.app.createmessagestats(savedminer, minerstats, minerpool) entries.addbroadcast(QueueName.Q_STATISTICSUPDATED, msg) except pika.exceptions.ConnectionClosed as qex: #could not enqueue a message print(Fore.RED + '{0} Queue Error: {1}'.format(savedminer.name, APPMONITOR.app.exceptionmessage(qex))) APPMONITOR.app.logexception(qex) except MinerMonitorException as monitorex: print(Fore.RED + '{0} Miner Error: {1}'.format(savedminer.name, APPMONITOR.app.exceptionmessage(monitorex))) savedminer.lastmonitor = datetime.datetime.utcnow() #TODO: this should be a rule. publish miner offline event #and let event handler decide how to handle it savedminer.offline_now() print(Fore.RED + APPMONITOR.app.now(), savedminer.name, savedminer.status) entries.add(QueueName.Q_OFFLINE, APPMONITOR.app.messageencode(savedminer)) except BaseException as ex: print(Fore.RED+'{0} Unexpected Error in monitorminer: {1}'.format(savedminer.name, APPMONITOR.app.exceptionmessage(ex))) # we have to consider any exception to be a miner error. sets status to offline #if str(e) == "timed out": #(timeout('timed out',),) APPMONITOR.app.logexception(ex) #TODO: review usage of savedminer and knownminer. should only go with one APPMONITOR.app.putminer(savedminer) APPMONITOR.app.updateknownminer(savedminer) return entries