Ejemplo n.º 1
0
def doprovisiondispatch(miner_type=None):
    '''put all miners in provision worker queue'''
    entries = QueueEntries()
    miners = PROVISION_DISPATCH.app.allminers()
    print("{0} miners configured".format(len(miners)))
    for miner in miners:
        if miner.is_disabled():
            continue
        try:
            minerstats, minerinfo, apicall, minerpool = stats(miner)
            if miner_type is not None and miner_type != '' and minerinfo.miner_type != miner_type:
                continue
            mineraccess = PROVISION_DISPATCH.app.antminer.getaccesslevel(miner)
            if mineraccess == MinerAccessLevel.Restricted:
                print(
                    Fore.RED +
                    "    Log: setting {0} to privileged...".format(miner.name))
                PROVISION_DISPATCH.app.antminer.set_privileged(miner)
                PROVISION_DISPATCH.app.antminer.waitforonline(miner)
                mineraccess = PROVISION_DISPATCH.app.antminer.getaccesslevel(
                    miner)
            print(Fore.GREEN + "{0} {1} {2}".format(
                miner.name, minerinfo.miner_type, mineraccess))
            if mineraccess == MinerAccessLevel.Restricted:
                print("    skipping restricted access")
            else:
                entries.add(QueueName.Q_PROVISION,
                            PROVISION_DISPATCH.app.messageencode(miner))
        except MinerMonitorException as minerex:
            print(minerex)
        except BaseException as ex:
            print('while provisioning {0} ({1})'.format(
                miner.ipaddress, miner.key()))
            print(PROVISION_DISPATCH.app.exceptionmessage(ex))
    return entries
Ejemplo n.º 2
0
 def __init__(self):
     self.minerstotal = 0
     self.minersnew = 0
     self.shownonminers = True
     self.hostsup = 0
     self.entries = QueueEntries()
     self.knownminers = None
     self.hosts_list = None
Ejemplo n.º 3
0
 def test_queue_entries(self):
     que = QueueEntries()
     self.assertTrue(que)
     que.add('test', 'test')
     que.addbroadcast('qbroad', 'test')
     que.addalert('msg')
     self.assertTrue(que.hasentries())
Ejemplo n.º 4
0
def findminers(hosts_list, knownminers):
    '''find miners on network'''
    entries = QueueEntries()
    minerstotal = 0
    minersnew = 0
    shownonminers = True
    hostsup = 0
    print('Querying {0} hosts...'.format(len(hosts_list)))
    for host, status, macaddress in hosts_list:
        try:
            if status != 'down':
                hostsup += 1
                if shownonminers:
                    print("{0} {1} {2}".format(host, status, macaddress))
                miner = mining.Miner(name=host, ipaddress=host, port=MINERPORT, ftpport='', networkid=macaddress)
                try:
                    minerstats, minerinfo, apicall, minerpool = antminerhelper.stats(miner)
                    miner.setminerinfo(minerinfo)
                    if minerinfo.miner_type:
                        minerstotal += 1
                        if not shownonminers:
                            print("{0} {1} {2}".format(host, status, macaddress))
                        print(Fore.GREEN + '   found {0} with id {1}'.format(minerinfo.miner_type, minerinfo.minerid))
                        #find by mac address or miner_id, not name
                        found = None
                        #if miner.minerid != "unknown"
                        for known in knownminers:
                            if known.networkid == miner.networkid or (miner.minerid != "unknown" and known.minerid == miner.minerid):
                                found = known
                        if found:
                            print(Fore.YELLOW + '   already know about {0}'.format(found.name))
                        else:
                            minersnew += 1
                            entries.add(QueueName.Q_DISCOVERED, DISCOVER.app.messageencode(miner))
                            print(Fore.GREEN + '   discovered {0}'.format(miner.name))

                except antminerhelper.MinerMonitorException as monitorex:
                    try:
                        if monitorex.istimedout:
                            if shownonminers:
                                print(Fore.RED + '    Not a miner')
                    except Exception:
                        DISCOVER.app.logexception(monitorex)
                except BaseException as baseex:
                    print(Fore.RED + DISCOVER.app.exceptionmessage(baseex))
        except KeyboardInterrupt:
            break
    print('nmap queried {0} hosts on network'.format(len(hosts_list)))
    print('{0} hosts are up'.format(hostsup))
    print('FCM knows about {0} miners configured'.format(len(knownminers)))
    print('FCM determined {0} miners this attempt'.format(minerstotal))
    print('FCM determined there are {0} new miners on network'.format(minersnew))
    return entries
Ejemplo n.º 5
0
def domonitor():
    '''queue workers to run the individual miner monitoring'''
    entries = QueueEntries()
    try:
        miners = MONITOR.app.allminers()
        print("{0} miners configured".format(len(miners)))

        for miner in miners:
            if not miner.is_manually_disabled():
                entries.add(QueueName.Q_MONITORMINER, MONITOR.app.messageencode(miner))
        print("waiting for next monitor event")
    except Exception as theex:
        MONITOR.app.logexception(theex)
    return entries
Ejemplo n.º 6
0
def domonitorminer(miner):
    '''get statistics from miner'''
    entries = QueueEntries()
    savedminer = APPMONITOR.app.getminer(miner)
    if savedminer is None:
        savedminer = miner
    try:
        #individual miner can be monitored even if manually disabled
        #todo:savedminer and knownminer out of sync. this will be fixed in refactoring redis
        if not savedminer.should_monitor() and not miner.should_monitor():
            print('skipped monitoring {0}'.format(miner.name))
            return entries
        mineroriginalstatus = savedminer.status
        minerstats, minerinfo, apicall, minerpool = stats(savedminer)
        #minerlcd = antminerhelper.getminerlcd(miner)
        if minerstats is None:
            print('could not monitor {0}({1})'.format(savedminer.name,
                                                      savedminer.ipaddress))
        else:
            process_stats(entries, savedminer, mineroriginalstatus, minerstats,
                          minerpool, minerinfo, apicall)

    except pika.exceptions.ConnectionClosed as qex:
        #could not enqueue a message
        print(Fore.RED + '{0} Queue Error: {1}'.format(
            savedminer.name, APPMONITOR.app.exceptionmessage(qex)))
        APPMONITOR.app.logexception(qex)
    except MinerMonitorException as monitorex:
        print(Fore.RED + '{0} Miner Error: {1}'.format(
            savedminer.name, APPMONITOR.app.exceptionmessage(monitorex)))
        savedminer.lastmonitor = datetime.datetime.utcnow()
        #TODO: this should be a rule. publish miner offline event
        #and let event handler decide how to handle it
        savedminer.offline_now()
        print(Fore.RED + APPMONITOR.app.now(), savedminer.name,
              savedminer.status)
        entries.add(QueueName.Q_OFFLINE,
                    APPMONITOR.app.messageencode(savedminer))

    except BaseException as ex:
        print(Fore.RED + '{0} Unexpected Error in monitorminer: {1}'.format(
            savedminer.name, APPMONITOR.app.exceptionmessage(ex)))
        # we have to consider any exception to be a miner error. sets status to offline
        #if str(e) == "timed out": #(timeout('timed out',),)
        APPMONITOR.app.logexception(ex)
    #TODO: review usage of savedminer and knownminer. should only go with one
    APPMONITOR.app.putminer(savedminer)
    APPMONITOR.app.updateknownminer(savedminer)
    return entries
Ejemplo n.º 7
0
class DiscoveryResults:
    def __init__(self):
        self.minerstotal = 0
        self.minersnew = 0
        self.shownonminers = True
        self.hostsup = 0
        self.entries = QueueEntries()
        self.knownminers = None
        self.hosts_list = None

    def process_miner(self, miner, status):
        minerstats, minerinfo, apicall, minerpool = antminerhelper.stats(miner)
        miner.setminerinfo(minerinfo)
        if minerinfo.miner_type:
            self.minerstotal += 1
            if not self.shownonminers:
                print("{0} {1} {2}".format(miner.ipaddress, status,
                                           miner.networkid))
            print(Fore.GREEN + '   found {0} with id {1}'.format(
                minerinfo.miner_type, minerinfo.minerid))
            #find by mac address or miner_id, not name
            found = None
            #if miner.minerid != "unknown"
            for known in self.knownminers:
                if known.networkid == miner.networkid or (not miner.is_unknown
                                                          and known.minerid
                                                          == miner.minerid):
                    found = known
            if found:
                print(Fore.YELLOW +
                      '   already know about {0}'.format(found.name))
            else:
                self.minersnew += 1
                self.entries.add(QueueName.Q_DISCOVERED,
                                 DISCOVER.app.messageencode(miner))
                print(Fore.GREEN + '   discovered {0}'.format(miner.name))

    def print(self):
        print('nmap queried {0} hosts on network'.format(len(self.hosts_list)))
        print('{0} hosts are up'.format(self.hostsup))
        print('FCM knows about {0} miners configured'.format(
            len(self.knownminers)))
        print('FCM determined {0} miners this attempt'.format(
            self.minerstotal))
        print('FCM determined there are {0} new miners on network'.format(
            self.minersnew))
Ejemplo n.º 8
0
def doprovisiondispatch(miner_type=None):
    '''put all miners in provision worker queue'''
    entries = QueueEntries()
    miners = PROVISION_DISPATCH.app.allminers()
    print("{0} miners configured".format(len(miners)))
    for miner in miners:
        if miner.is_disabled():
            continue
        process_miner(miner, entries, miner_type)
    return entries
Ejemplo n.º 9
0
def dooffline(miner: mining.Miner):
    '''notify user'''
    entries = QueueEntries()
    savedminer = OFFLINE.app.getminer(miner)
    if not savedminer.is_disabled():
        if savedminer.is_send_offline_alert():
            #update status to offline and alert
            savedminer.status = mining.MinerStatus.Offline
            alertmsg = OFFLINE.app.stamp(
                'miner {0} is offline! since {1}'.format(
                    savedminer.name, savedminer.laststatuschanged))
            OFFLINE.app.putminer(savedminer)
            entries.addalert(alertmsg)
            print("Sent offline alert '{0}'".format(alertmsg))
        else:
            #stop annoying the user, disable the miner to stop sending alerts
            savedminer.status = mining.MinerStatus.Disabled
            alertmsg = OFFLINE.app.stamp(
                'miner {0} is Disabled after many offline alerts. No more alerts will be sent for this miner.'
                .format(savedminer.name))
            OFFLINE.app.putminer(savedminer)
            entries.addalert(alertmsg)
            print("Sent disabled alert for {0}".format(savedminer.name))
    else:
        if not savedminer.is_manually_disabled():
            print(
                'Disabled miner {0} is offline. manually disable miner to get rid of this message'
                .format(miner.name))

    return entries
Ejemplo n.º 10
0
def rules(miner, minerstats, minerpool):
    '''this runs the rules'''
    entries = QueueEntries()
    if miner is None or minerstats is None:
        return entries
    savedminer = RULES.app.getminer(miner)
    broken = gather_broken_rules(savedminer, minerstats)
    if broken:
        #TODO: could raise broken rule event???
        for rule in broken:
            RULES.app.log_mineractivity(make_log_from_rule(rule))
            add_entry_for_rule(entries, rule)
    return entries
Ejemplo n.º 11
0
def dosave(msg):
    entries = QueueEntries()
    if msg.entity == 'miner':
        miner = saveminer(msg)
        entries.add(QueueName.Q_MONITORMINER,
                    COMPONENTSAVE.app.messageencode(miner))
        entries.add(QueueName.Q_PROVISION,
                    COMPONENTSAVE.app.messageencode(miner))

    if msg.entity == 'pool':
        savepool(msg)
    return entries
Ejemplo n.º 12
0
def rules(miner, minerstats, minerpool):
    '''this runs the rules'''
    entries = QueueEntries()
    if miner is None or minerstats is None:
        return entries
    savedminer = RULES.app.getminer(miner)
    cmd_restart = MinerCommand('restart', '')
    broken = []
    for ruleparms in RULES.app.ruleparameters():
        rule = MinerStatisticsRule(savedminer, minerstats, ruleparms)
        if rule.isbroken():
            broken += rule.brokenrules

    if broken:
        #TODO: could raise broken rule event???
        for rule in broken:
            log = MinerLog()
            log.createdate = datetime.datetime.utcnow()
            log.minerid = rule.miner.key()
            log.minername = rule.miner.name
            log.action = rule.parameter
            RULES.app.log_mineractivity(log)

            if rule.action == 'alert':
                entries.addalert(
                    RULES.addalert(RULES.app.stamp(rule.parameter)))
            elif rule.action == 'restart':
                entries.add(
                    QueueName.Q_RESTART,
                    RULES.app.createmessagecommand(rule.miner, cmd_restart))
                entries.addalert(
                    RULES.addalert(
                        RULES.app.stamp('Restarted {0}'.format(
                            rule.miner.name))))
            else:
                RULES.app.logerror('did not process broken rule {0}'.format(
                    rule.parameter))

    return entries
Ejemplo n.º 13
0
def doonline(miner):
    '''then provision the miner'''
    entries = QueueEntries()
    savedminer = ONLINE.app.getminer(miner)
    if savedminer is None:
        savedminer = miner
    #update status
    savedminer.status = mining.MinerStatus.Online
    #sending message will also save it
    #just provision the miner and start to monitor
    entries.add(QueueName.Q_PROVISION, ONLINE.app.messageencode(savedminer))
    #tell them something good happened
    msg = 'miner {0} is back online'.format(savedminer.name)
    entries.addalert(msg)
    print(msg)
    return entries
Ejemplo n.º 14
0
def doprovision(miner):
    '''provision/configure a miner'''
    entries = QueueEntries()
    poollist = PROVISION.app.pools.get_all_pools()
    print("{0} pools configured".format(len(poollist)))
    print('{0} {1}'.format(miner.name, miner.ipaddress))
    mineraccess = ''
    addpools = None
    minerinfo = None
    minerpool = None
    try:
        minerstats, minerinfo, apicall, minerpool = antminerhelper.stats(miner)
        miner.setminerinfo(minerinfo)
        #find the current pool in known pools
        knownpool = PROVISION.app.pools.findpool(minerpool)
        if knownpool is not None:
            minerpool.poolname = knownpool.name
        miner.minerpool = minerpool
        PROVISION.app.updateknownminer(miner)
        #find pools that need to be added and add them
        addpools = services.poolstoadd(miner, minerpool, poollist)
        mineraccess = PROVISION.app.antminer.getaccesslevel(miner)
    except antminerhelper.MinerMonitorException as ex:
        if ex.istimedout():
            mineraccess = MinerAccessLevel.Waiting
    if mineraccess == MinerAccessLevel.Restricted or mineraccess == MinerAccessLevel.Waiting:
        if mineraccess == MinerAccessLevel.Restricted:
            PROVISION.app.antminer.set_privileged(miner)
        PROVISION.app.antminer.waitforonline(miner)
        mineraccess = PROVISION.app.antminer.getaccesslevel(miner)

    if mineraccess == MinerAccessLevel.Restricted:
        entries.addalert('could not set {0} to privileged access'.format(
            miner.name))
        #try a few more times then give up
    else:
        addpoolstominer(miner, addpools)

        addminerpools(miner)

        switchtodefaultpool(miner, poollist, minerpool)

        enforcedefaultpool(miner, poollist, minerpool)

        entries.add(QueueName.Q_MONITORMINER,
                    PROVISION.app.messageencode(miner))
    return entries
Ejemplo n.º 15
0
def doonline(miner):
    '''then provision the miner'''
    entries = QueueEntries()
    savedminer = ONLINE.app.getminer(miner)
    if savedminer is None:
        savedminer = miner
    #update status
    savedminer.online_now()
    ONLINE.app.putminer(savedminer)
    ONLINE.app.updateknownminer(savedminer)
    #just provision the miner and start to monitor
    entries.add(QueueName.Q_PROVISION, ONLINE.app.messageencode(savedminer))
    #tell them something good happened
    msg = ONLINE.app.stamp('miner {0} is back online'.format(savedminer.name))
    entries.addalert(msg)
    print(msg)
    return entries
Ejemplo n.º 16
0
def dodiscovered(miner):
    '''then provision it'''
    entries = QueueEntries()
    entries.add(QueueName.Q_PROVISION, COMPONENTDISCOVERED.app.messageencode(miner))
    cachedminer = COMPONENTDISCOVERED.app.getminer(miner)
    #knownminer should be None
    if cachedminer is not None:
        cachedminer.updatefrom(miner)
    COMPONENTDISCOVERED.app.putminer(cachedminer)
    knownminer = COMPONENTDISCOVERED.app.getknownminer(miner)
    if knownminer is None:
        COMPONENTDISCOVERED.app.addknownminer(miner)
    else:
        COMPONENTDISCOVERED.app.updateknownminer(miner)
    entries.add(QueueName.Q_ALERT, 'discovered miner {0}'.format(miner.name))
    print("Discovered {0}".format(miner.name))
    return entries
Ejemplo n.º 17
0
def dosave(msg):
    entries = QueueEntries()
    if msg.entity == 'miner':
        #add or update miner
        minerid = name = ipaddress = port = None
        for pair in msg.values:
            if 'minerid' in pair:
                minerid = pair['minerid']
            if 'name' in pair:
                name = pair['name']
            if 'ipaddress' in pair:
                ipaddress = pair['ipaddress']
            if 'port' in pair:
                port = pair['port']
        miner = Miner(name, '', '', ipaddress, port, '', '')
        COMPONENTSAVE.app.save_miner(miner)
        entries.add(QueueName.Q_MONITORMINER,
                    COMPONENTSAVE.app.messageencode(miner))
        entries.add(QueueName.Q_PROVISION,
                    COMPONENTSAVE.app.messageencode(miner))

    if msg.entity == 'pool':
        #save the new named pool
        pool_type = name = url = user = priority = None
        for pair in msg.values:
            if 'pool_type' in pair:
                pool_type = pair['pool_type']
            if 'name' in pair:
                name = pair['name']
            if 'url' in pair:
                url = pair['url']
            if 'user' in pair:
                user = pair['user']
            if 'priority' in pair:
                priority = pair['priority']
        pool = Pool(pool_type, name, url, user, priority)
        COMPONENTSAVE.app.save_pool(pool)
    return entries
Ejemplo n.º 18
0
def doprovision(miner):
    '''provision/configure a miner'''
    entries = QueueEntries()
    poollist = PROVISION.app.pools()
    print("{0} pools configured".format(len(poollist)))
    print('{0} {1}'.format(miner.name, miner.ipaddress))
    mineraccess = ''
    addpools = None
    minerinfo = None
    minerpool = None
    try:
        minerinfo = antminerhelper.getminerinfo(miner)
        miner.minerinfo = minerinfo
        minerpool = antminerhelper.pools(miner)
        #find the current pool in known pools
        knownpool = PROVISION.app.findpool(minerpool)
        if knownpool is not None:
            minerpool.poolname = knownpool.name
        miner.minerpool = minerpool
        PROVISION.app.updateknownminer(miner)
        #find pools that need to be added and add them
        addpools = services.poolstoadd(miner, minerpool, poollist)
        mineraccess = PROVISION.app.antminer.getaccesslevel(miner)
    except antminerhelper.MinerMonitorException as ex:
        if ex.istimedout():
            mineraccess = MinerAccessLevel.Waiting
    if mineraccess == MinerAccessLevel.Restricted or mineraccess == MinerAccessLevel.Waiting:
        if mineraccess == MinerAccessLevel.Restricted:
            PROVISION.app.antminer.set_privileged(miner)
        PROVISION.app.antminer.waitforonline(miner)
        mineraccess = PROVISION.app.antminer.getaccesslevel(miner)

    if mineraccess == MinerAccessLevel.Restricted:
        entries.addalert('could not set {0} to privileged access'.format(
            miner.name))
        #try a few more times then give up
    else:
        for pool in addpools or []:
            print(
                Fore.YELLOW + "     Add", pool.name,
                "(addpool|{0},{1},{2})".format(pool.url,
                                               pool.user + miner.name, "x"))
            #this command adds the pool to miner and prints the result
            result = antminerhelper.addpool(miner, pool)
            if result.startswith("Access denied"):
                print(Fore.RED + result)
            else:
                print(result)

        namedpools = PROVISION.app.pools()
        #process the pools found on the miner. This will pick up any pools added manually
        for pool in miner.pools_available:
            #check if pools is a named pool...
            foundnamed = None
            for namedpool in namedpools:
                if namedpool.is_same_as(pool):
                    foundnamed = namedpool
                    break
            if foundnamed:
                #pool should take on the cononical attributes of the named pool
                pool.named_pool = foundnamed
                pool.user = foundnamed.user
            PROVISION.app.add_pool(MinerPool(miner, pool.priority, pool))

        #enforce default pool if miner has one set up
        if miner.defaultpool:
            founddefault = next(
                (p for p in poollist if p.name == miner.defaultpool), None)
            if founddefault is not None:
                switchtopool(miner, founddefault)

        #enforce default pool if it doesnt have one. find highest priority pool
        if not miner.defaultpool:

            def sort_by_priority(j):
                return j.priority

            filtered = [
                x for x in poollist if miner.miner_type.startswith(x.pool_type)
            ]
            filtered.sort(key=sort_by_priority)
            #foundpriority = next((p for p in poollist if p.priority == 0), None)
            if filtered:
                switchtopool(miner, filtered[0])

        entries.add(QueueName.Q_MONITORMINER,
                    PROVISION.app.messageencode(miner))
    return entries
Ejemplo n.º 19
0
'''
    A test program that pushes monitoring messages onto the queue
    A stress tester for monitoring
    when_monitorminer.py should be running to process the messages
'''

from helpers.queuehelper import QueueName, QueueEntries
from backend.fcmapp import ApplicationService, ComponentName

APP = ApplicationService(ComponentName.fullcycle)
MINERS = APP.miners()

CNT = 1
while CNT < 2000:
    ENTRIES = QueueEntries()
    for miner in MINERS:
        ENTRIES.add(QueueName.Q_MONITORMINER, APP.messageencode(miner))
    APP.enqueue(ENTRIES)
    print('sent {} messages'.format(len(ENTRIES.entries)))
    CNT += 1
Ejemplo n.º 20
0
'''
    A test program that pushes monitoring messages onto the queue
    A stress tester for monitoring
    when_monitorminer.py should be running to process the messages
'''

from helpers.queuehelper import QueueName, QueueEntries
from backend.fcmapp import ApplicationService, ComponentName

APP = ApplicationService(ComponentName.fullcycle)
MINERS = APP.miners()

cnt = 1
while cnt < 1000:
    entries = QueueEntries()
    for miner in MINERS:
        entries.add(QueueName.Q_MONITORMINER, APP.messageencode(miner))
    APP.enqueue(entries)
    print('sent {} messages'.format(len(entries.entries)))
    cnt += 1

Ejemplo n.º 21
0
def domonitorminer(miner):
    '''get statistics from miner'''
    entries = QueueEntries()
    savedminer = APPMONITOR.app.getminer(miner)
    if savedminer is None:
        savedminer = miner
    try:
        #individual miner can be monitored even if manually disabled
        #todo:savedminer and knownminer out of sync. this will be fixed in refactoring redis
        if not savedminer.should_monitor() and not miner.should_monitor():
            print('skipped monitoring {0}'.format(miner.name))
            return entries
        mineroriginalstatus = savedminer.status
        #minerinfo = getminerinfo(savedminer)
        minerstats, minerinfo, apicall, minerpool = stats(savedminer)
        #minerlcd = antminerhelper.getminerlcd(miner)
        if minerstats is None:
            print('could not monitor {0}'.format(savedminer.name))
        else:
            #what to do if monitored miner type conflicts with saved miner type???
            #should probably provision?
            foundpool = APPMONITOR.app.findpool(minerpool)
            if foundpool is not None:
                minerpool.poolname = foundpool.name
            savedminer.monitored(minerstats, minerpool, minerinfo, apicall.elapsed())
            if mineroriginalstatus == '':
                #first time monitoring since bootup
                print(Fore.GREEN + APPMONITOR.app.now(), savedminer.name, 'first time monitoring')
            elif savedminer.status == mining.MinerStatus.Online and (mineroriginalstatus == mining.MinerStatus.Disabled or mineroriginalstatus == mining.MinerStatus.Offline):
                #changing status from offline to online so raise event
                entries.add(QueueName.Q_ONLINE, APPMONITOR.app.messageencode(savedminer))
                print(Fore.GREEN + APPMONITOR.app.now(), savedminer.name, 'back online!')
            #TODO: if stats.elapsed < previous.elapsed then raise provision or online events

            APPMONITOR.app.putminerandstats(savedminer, minerstats, minerpool)
            #TODO:show name of current pool instead of worker
            poolname = '?'
            if minerpool:
                poolname = '{0} {1}'.format(minerpool.currentpool, minerpool.currentworker)
            foundpool = APPMONITOR.app.findpool(minerpool)
            if foundpool is not None:
                poolname = foundpool.name
            print('{0} mining at {1}'.format(savedminer.name, poolname))

            #most users won't want to mine solo, so provision the miner
            if not APPMONITOR.app.configuration('mining.allowsolomining'):
                if not minerpool.currentpool or minerpool.currentpool.startswith(APPMONITOR.app.configuration('mining.solopool')):
                    entries.add(QueueName.Q_PROVISION, APPMONITOR.app.messageencode(savedminer))

            print(Fore.CYAN+str(APPMONITOR.app.now()), savedminer.name, savedminer.status,
                  'h='+str(minerstats.currenthash), str(minerstats.minercount),
                  '{0}/{1}/{2}'.format(str(minerstats.tempboard1),
                                       str(minerstats.tempboard2),
                                       str(minerstats.tempboard3)),
                  savedminer.uptime(minerstats.elapsed),
                  '{0:d}ms'.format(int(savedminer.monitorresponsetime() * 1000)))
            msg = APPMONITOR.app.createmessagestats(savedminer, minerstats, minerpool)
            entries.addbroadcast(QueueName.Q_STATISTICSUPDATED, msg)

    except pika.exceptions.ConnectionClosed as qex:
        #could not enqueue a message
        print(Fore.RED + '{0} Queue Error: {1}'.format(savedminer.name,
                                                       APPMONITOR.app.exceptionmessage(qex)))
        APPMONITOR.app.logexception(qex)
    except MinerMonitorException as monitorex:
        print(Fore.RED + '{0} Miner Error: {1}'.format(savedminer.name,
                                                       APPMONITOR.app.exceptionmessage(monitorex)))
        savedminer.lastmonitor = datetime.datetime.utcnow()
        #TODO: this should be a rule. publish miner offline event
        #and let event handler decide how to handle it
        savedminer.offline_now()
        print(Fore.RED + APPMONITOR.app.now(), savedminer.name, savedminer.status)
        entries.add(QueueName.Q_OFFLINE, APPMONITOR.app.messageencode(savedminer))

    except BaseException as ex:
        print(Fore.RED+'{0} Unexpected Error in monitorminer: {1}'.format(savedminer.name,
                                                                          APPMONITOR.app.exceptionmessage(ex)))
        # we have to consider any exception to be a miner error. sets status to offline
        #if str(e) == "timed out": #(timeout('timed out',),)
        APPMONITOR.app.logexception(ex)
    #TODO: review usage of savedminer and knownminer. should only go with one
    APPMONITOR.app.putminer(savedminer)
    APPMONITOR.app.updateknownminer(savedminer)
    return entries