def doprovisiondispatch(miner_type=None):
'''put all miners in provision worker queue'''
entries = QueueEntries()
miners = PROVISION_DISPATCH.app.allminers()
print("{0} miners configured".format(len(miners)))
for miner in miners:
if miner.is_disabled():
continue
try:
minerstats, minerinfo, apicall, minerpool = stats(miner)
if miner_type is not None and miner_type != '' and minerinfo.miner_type != miner_type:
continue
mineraccess = PROVISION_DISPATCH.app.antminer.getaccesslevel(miner)
if mineraccess == MinerAccessLevel.Restricted:
print(
Fore.RED +
" Log: setting {0} to privileged...".format(miner.name))
PROVISION_DISPATCH.app.antminer.set_privileged(miner)
PROVISION_DISPATCH.app.antminer.waitforonline(miner)
mineraccess = PROVISION_DISPATCH.app.antminer.getaccesslevel(
miner)
print(Fore.GREEN + "{0} {1} {2}".format(
miner.name, minerinfo.miner_type, mineraccess))
if mineraccess == MinerAccessLevel.Restricted:
print(" skipping restricted access")
else:
entries.add(QueueName.Q_PROVISION,
PROVISION_DISPATCH.app.messageencode(miner))
except MinerMonitorException as minerex:
print(minerex)
except BaseException as ex:
print('while provisioning {0} ({1})'.format(
miner.ipaddress, miner.key()))
print(PROVISION_DISPATCH.app.exceptionmessage(ex))
return entries
def test_queue_entries(self):
que = QueueEntries()
self.assertTrue(que)
que.add('test', 'test')
que.addbroadcast('qbroad', 'test')
que.addalert('msg')
self.assertTrue(que.hasentries())
def findminers(hosts_list, knownminers):
'''find miners on network'''
entries = QueueEntries()
minerstotal = 0
minersnew = 0
shownonminers = True
hostsup = 0
print('Querying {0} hosts...'.format(len(hosts_list)))
for host, status, macaddress in hosts_list:
try:
if status != 'down':
hostsup += 1
if shownonminers:
print("{0} {1} {2}".format(host, status, macaddress))
miner = mining.Miner(name=host, ipaddress=host, port=MINERPORT, ftpport='', networkid=macaddress)
try:
minerstats, minerinfo, apicall, minerpool = antminerhelper.stats(miner)
miner.setminerinfo(minerinfo)
if minerinfo.miner_type:
minerstotal += 1
if not shownonminers:
print("{0} {1} {2}".format(host, status, macaddress))
print(Fore.GREEN + ' found {0} with id {1}'.format(minerinfo.miner_type, minerinfo.minerid))
#find by mac address or miner_id, not name
found = None
#if miner.minerid != "unknown"
for known in knownminers:
if known.networkid == miner.networkid or (miner.minerid != "unknown" and known.minerid == miner.minerid):
found = known
if found:
print(Fore.YELLOW + ' already know about {0}'.format(found.name))
else:
minersnew += 1
entries.add(QueueName.Q_DISCOVERED, DISCOVER.app.messageencode(miner))
print(Fore.GREEN + ' discovered {0}'.format(miner.name))
except antminerhelper.MinerMonitorException as monitorex:
try:
if monitorex.istimedout:
if shownonminers:
print(Fore.RED + ' Not a miner')
except Exception:
DISCOVER.app.logexception(monitorex)
except BaseException as baseex:
print(Fore.RED + DISCOVER.app.exceptionmessage(baseex))
except KeyboardInterrupt:
break
print('nmap queried {0} hosts on network'.format(len(hosts_list)))
print('{0} hosts are up'.format(hostsup))
print('FCM knows about {0} miners configured'.format(len(knownminers)))
print('FCM determined {0} miners this attempt'.format(minerstotal))
print('FCM determined there are {0} new miners on network'.format(minersnew))
return entries
def dosave(msg):
entries = QueueEntries()
if msg.entity == 'miner':
miner = saveminer(msg)
entries.add(QueueName.Q_MONITORMINER,
COMPONENTSAVE.app.messageencode(miner))
entries.add(QueueName.Q_PROVISION,
COMPONENTSAVE.app.messageencode(miner))
if msg.entity == 'pool':
savepool(msg)
return entries
def domonitor():
'''queue workers to run the individual miner monitoring'''
entries = QueueEntries()
try:
miners = MONITOR.app.allminers()
print("{0} miners configured".format(len(miners)))
for miner in miners:
if not miner.is_manually_disabled():
entries.add(QueueName.Q_MONITORMINER, MONITOR.app.messageencode(miner))
print("waiting for next monitor event")
except Exception as theex:
MONITOR.app.logexception(theex)
return entries
def domonitorminer(miner):
'''get statistics from miner'''
entries = QueueEntries()
savedminer = APPMONITOR.app.getminer(miner)
if savedminer is None:
savedminer = miner
try:
#individual miner can be monitored even if manually disabled
#todo:savedminer and knownminer out of sync. this will be fixed in refactoring redis
if not savedminer.should_monitor() and not miner.should_monitor():
print('skipped monitoring {0}'.format(miner.name))
return entries
mineroriginalstatus = savedminer.status
minerstats, minerinfo, apicall, minerpool = stats(savedminer)
#minerlcd = antminerhelper.getminerlcd(miner)
if minerstats is None:
print('could not monitor {0}({1})'.format(savedminer.name,
savedminer.ipaddress))
else:
process_stats(entries, savedminer, mineroriginalstatus, minerstats,
minerpool, minerinfo, apicall)
except pika.exceptions.ConnectionClosed as qex:
#could not enqueue a message
print(Fore.RED + '{0} Queue Error: {1}'.format(
savedminer.name, APPMONITOR.app.exceptionmessage(qex)))
APPMONITOR.app.logexception(qex)
except MinerMonitorException as monitorex:
print(Fore.RED + '{0} Miner Error: {1}'.format(
savedminer.name, APPMONITOR.app.exceptionmessage(monitorex)))
savedminer.lastmonitor = datetime.datetime.utcnow()
#TODO: this should be a rule. publish miner offline event
#and let event handler decide how to handle it
savedminer.offline_now()
print(Fore.RED + APPMONITOR.app.now(), savedminer.name,
savedminer.status)
entries.add(QueueName.Q_OFFLINE,
APPMONITOR.app.messageencode(savedminer))
except BaseException as ex:
print(Fore.RED + '{0} Unexpected Error in monitorminer: {1}'.format(
savedminer.name, APPMONITOR.app.exceptionmessage(ex)))
# we have to consider any exception to be a miner error. sets status to offline
#if str(e) == "timed out": #(timeout('timed out',),)
APPMONITOR.app.logexception(ex)
#TODO: review usage of savedminer and knownminer. should only go with one
APPMONITOR.app.putminer(savedminer)
APPMONITOR.app.updateknownminer(savedminer)
return entries
def doonline(miner):
'''then provision the miner'''
entries = QueueEntries()
savedminer = ONLINE.app.getminer(miner)
if savedminer is None:
savedminer = miner
#update status
savedminer.status = mining.MinerStatus.Online
#sending message will also save it
#just provision the miner and start to monitor
entries.add(QueueName.Q_PROVISION, ONLINE.app.messageencode(savedminer))
#tell them something good happened
msg = 'miner {0} is back online'.format(savedminer.name)
entries.addalert(msg)
print(msg)
return entries
def doprovision(miner):
'''provision/configure a miner'''
entries = QueueEntries()
poollist = PROVISION.app.pools.get_all_pools()
print("{0} pools configured".format(len(poollist)))
print('{0} {1}'.format(miner.name, miner.ipaddress))
mineraccess = ''
addpools = None
minerinfo = None
minerpool = None
try:
minerstats, minerinfo, apicall, minerpool = antminerhelper.stats(miner)
miner.setminerinfo(minerinfo)
#find the current pool in known pools
knownpool = PROVISION.app.pools.findpool(minerpool)
if knownpool is not None:
minerpool.poolname = knownpool.name
miner.minerpool = minerpool
PROVISION.app.updateknownminer(miner)
#find pools that need to be added and add them
addpools = services.poolstoadd(miner, minerpool, poollist)
mineraccess = PROVISION.app.antminer.getaccesslevel(miner)
except antminerhelper.MinerMonitorException as ex:
if ex.istimedout():
mineraccess = MinerAccessLevel.Waiting
if mineraccess == MinerAccessLevel.Restricted or mineraccess == MinerAccessLevel.Waiting:
if mineraccess == MinerAccessLevel.Restricted:
PROVISION.app.antminer.set_privileged(miner)
PROVISION.app.antminer.waitforonline(miner)
mineraccess = PROVISION.app.antminer.getaccesslevel(miner)
if mineraccess == MinerAccessLevel.Restricted:
entries.addalert('could not set {0} to privileged access'.format(
miner.name))
#try a few more times then give up
else:
addpoolstominer(miner, addpools)
addminerpools(miner)
switchtodefaultpool(miner, poollist, minerpool)
enforcedefaultpool(miner, poollist, minerpool)
entries.add(QueueName.Q_MONITORMINER,
PROVISION.app.messageencode(miner))
return entries
class DiscoveryResults:
def __init__(self):
self.minerstotal = 0
self.minersnew = 0
self.shownonminers = True
self.hostsup = 0
self.entries = QueueEntries()
self.knownminers = None
self.hosts_list = None
def process_miner(self, miner, status):
minerstats, minerinfo, apicall, minerpool = antminerhelper.stats(miner)
miner.setminerinfo(minerinfo)
if minerinfo.miner_type:
self.minerstotal += 1
if not self.shownonminers:
print("{0} {1} {2}".format(miner.ipaddress, status,
miner.networkid))
print(Fore.GREEN + ' found {0} with id {1}'.format(
minerinfo.miner_type, minerinfo.minerid))
#find by mac address or miner_id, not name
found = None
#if miner.minerid != "unknown"
for known in self.knownminers:
if known.networkid == miner.networkid or (not miner.is_unknown
and known.minerid
== miner.minerid):
found = known
if found:
print(Fore.YELLOW +
' already know about {0}'.format(found.name))
else:
self.minersnew += 1
self.entries.add(QueueName.Q_DISCOVERED,
DISCOVER.app.messageencode(miner))
print(Fore.GREEN + ' discovered {0}'.format(miner.name))
def print(self):
print('nmap queried {0} hosts on network'.format(len(self.hosts_list)))
print('{0} hosts are up'.format(self.hostsup))
print('FCM knows about {0} miners configured'.format(
len(self.knownminers)))
print('FCM determined {0} miners this attempt'.format(
self.minerstotal))
print('FCM determined there are {0} new miners on network'.format(
self.minersnew))
def dodiscovered(miner):
'''then provision it'''
entries = QueueEntries()
entries.add(QueueName.Q_PROVISION, COMPONENTDISCOVERED.app.messageencode(miner))
cachedminer = COMPONENTDISCOVERED.app.getminer(miner)
#knownminer should be None
if cachedminer is not None:
cachedminer.updatefrom(miner)
COMPONENTDISCOVERED.app.putminer(cachedminer)
knownminer = COMPONENTDISCOVERED.app.getknownminer(miner)
if knownminer is None:
COMPONENTDISCOVERED.app.addknownminer(miner)
else:
COMPONENTDISCOVERED.app.updateknownminer(miner)
entries.add(QueueName.Q_ALERT, 'discovered miner {0}'.format(miner.name))
print("Discovered {0}".format(miner.name))
return entries
def doonline(miner):
'''then provision the miner'''
entries = QueueEntries()
savedminer = ONLINE.app.getminer(miner)
if savedminer is None:
savedminer = miner
#update status
savedminer.online_now()
ONLINE.app.putminer(savedminer)
ONLINE.app.updateknownminer(savedminer)
#just provision the miner and start to monitor
entries.add(QueueName.Q_PROVISION, ONLINE.app.messageencode(savedminer))
#tell them something good happened
msg = ONLINE.app.stamp('miner {0} is back online'.format(savedminer.name))
entries.addalert(msg)
print(msg)
return entries
def rules(miner, minerstats, minerpool):
'''this runs the rules'''
entries = QueueEntries()
if miner is None or minerstats is None:
return entries
savedminer = RULES.app.getminer(miner)
cmd_restart = MinerCommand('restart', '')
broken = []
for ruleparms in RULES.app.ruleparameters():
rule = MinerStatisticsRule(savedminer, minerstats, ruleparms)
if rule.isbroken():
broken += rule.brokenrules
if broken:
#TODO: could raise broken rule event???
for rule in broken:
log = MinerLog()
log.createdate = datetime.datetime.utcnow()
log.minerid = rule.miner.key()
log.minername = rule.miner.name
log.action = rule.parameter
RULES.app.log_mineractivity(log)
if rule.action == 'alert':
entries.addalert(
RULES.addalert(RULES.app.stamp(rule.parameter)))
elif rule.action == 'restart':
entries.add(
QueueName.Q_RESTART,
RULES.app.createmessagecommand(rule.miner, cmd_restart))
entries.addalert(
RULES.addalert(
RULES.app.stamp('Restarted {0}'.format(
rule.miner.name))))
else:
RULES.app.logerror('did not process broken rule {0}'.format(
rule.parameter))
return entries
def dosave(msg):
entries = QueueEntries()
if msg.entity == 'miner':
#add or update miner
minerid = name = ipaddress = port = None
for pair in msg.values:
if 'minerid' in pair:
minerid = pair['minerid']
if 'name' in pair:
name = pair['name']
if 'ipaddress' in pair:
ipaddress = pair['ipaddress']
if 'port' in pair:
port = pair['port']
miner = Miner(name, '', '', ipaddress, port, '', '')
COMPONENTSAVE.app.save_miner(miner)
entries.add(QueueName.Q_MONITORMINER,
COMPONENTSAVE.app.messageencode(miner))
entries.add(QueueName.Q_PROVISION,
COMPONENTSAVE.app.messageencode(miner))
if msg.entity == 'pool':
#save the new named pool
pool_type = name = url = user = priority = None
for pair in msg.values:
if 'pool_type' in pair:
pool_type = pair['pool_type']
if 'name' in pair:
name = pair['name']
if 'url' in pair:
url = pair['url']
if 'user' in pair:
user = pair['user']
if 'priority' in pair:
priority = pair['priority']
pool = Pool(pool_type, name, url, user, priority)
COMPONENTSAVE.app.save_pool(pool)
return entries
def doprovision(miner):
'''provision/configure a miner'''
entries = QueueEntries()
poollist = PROVISION.app.pools()
print("{0} pools configured".format(len(poollist)))
print('{0} {1}'.format(miner.name, miner.ipaddress))
mineraccess = ''
addpools = None
minerinfo = None
minerpool = None
try:
minerinfo = antminerhelper.getminerinfo(miner)
miner.minerinfo = minerinfo
minerpool = antminerhelper.pools(miner)
#find the current pool in known pools
knownpool = PROVISION.app.findpool(minerpool)
if knownpool is not None:
minerpool.poolname = knownpool.name
miner.minerpool = minerpool
PROVISION.app.updateknownminer(miner)
#find pools that need to be added and add them
addpools = services.poolstoadd(miner, minerpool, poollist)
mineraccess = PROVISION.app.antminer.getaccesslevel(miner)
except antminerhelper.MinerMonitorException as ex:
if ex.istimedout():
mineraccess = MinerAccessLevel.Waiting
if mineraccess == MinerAccessLevel.Restricted or mineraccess == MinerAccessLevel.Waiting:
if mineraccess == MinerAccessLevel.Restricted:
PROVISION.app.antminer.set_privileged(miner)
PROVISION.app.antminer.waitforonline(miner)
mineraccess = PROVISION.app.antminer.getaccesslevel(miner)
if mineraccess == MinerAccessLevel.Restricted:
entries.addalert('could not set {0} to privileged access'.format(
miner.name))
#try a few more times then give up
else:
for pool in addpools or []:
print(
Fore.YELLOW + " Add", pool.name,
"(addpool|{0},{1},{2})".format(pool.url,
pool.user + miner.name, "x"))
#this command adds the pool to miner and prints the result
result = antminerhelper.addpool(miner, pool)
if result.startswith("Access denied"):
print(Fore.RED + result)
else:
print(result)
namedpools = PROVISION.app.pools()
#process the pools found on the miner. This will pick up any pools added manually
for pool in miner.pools_available:
#check if pools is a named pool...
foundnamed = None
for namedpool in namedpools:
if namedpool.is_same_as(pool):
foundnamed = namedpool
break
if foundnamed:
#pool should take on the cononical attributes of the named pool
pool.named_pool = foundnamed
pool.user = foundnamed.user
PROVISION.app.add_pool(MinerPool(miner, pool.priority, pool))
#enforce default pool if miner has one set up
if miner.defaultpool:
founddefault = next(
(p for p in poollist if p.name == miner.defaultpool), None)
if founddefault is not None:
switchtopool(miner, founddefault)
#enforce default pool if it doesnt have one. find highest priority pool
if not miner.defaultpool:
def sort_by_priority(j):
return j.priority
filtered = [
x for x in poollist if miner.miner_type.startswith(x.pool_type)
]
filtered.sort(key=sort_by_priority)
#foundpriority = next((p for p in poollist if p.priority == 0), None)
if filtered:
switchtopool(miner, filtered[0])
entries.add(QueueName.Q_MONITORMINER,
PROVISION.app.messageencode(miner))
return entries
'''
A test program that pushes monitoring messages onto the queue
A stress tester for monitoring
when_monitorminer.py should be running to process the messages
'''
from helpers.queuehelper import QueueName, QueueEntries
from backend.fcmapp import ApplicationService, ComponentName
APP = ApplicationService(ComponentName.fullcycle)
MINERS = APP.miners()
CNT = 1
while CNT < 2000:
ENTRIES = QueueEntries()
for miner in MINERS:
ENTRIES.add(QueueName.Q_MONITORMINER, APP.messageencode(miner))
APP.enqueue(ENTRIES)
print('sent {} messages'.format(len(ENTRIES.entries)))
CNT += 1
def domonitorminer(miner):
'''get statistics from miner'''
entries = QueueEntries()
savedminer = APPMONITOR.app.getminer(miner)
if savedminer is None:
savedminer = miner
try:
#individual miner can be monitored even if manually disabled
#todo:savedminer and knownminer out of sync. this will be fixed in refactoring redis
if not savedminer.should_monitor() and not miner.should_monitor():
print('skipped monitoring {0}'.format(miner.name))
return entries
mineroriginalstatus = savedminer.status
#minerinfo = getminerinfo(savedminer)
minerstats, minerinfo, apicall, minerpool = stats(savedminer)
#minerlcd = antminerhelper.getminerlcd(miner)
if minerstats is None:
print('could not monitor {0}'.format(savedminer.name))
else:
#what to do if monitored miner type conflicts with saved miner type???
#should probably provision?
foundpool = APPMONITOR.app.findpool(minerpool)
if foundpool is not None:
minerpool.poolname = foundpool.name
savedminer.monitored(minerstats, minerpool, minerinfo, apicall.elapsed())
if mineroriginalstatus == '':
#first time monitoring since bootup
print(Fore.GREEN + APPMONITOR.app.now(), savedminer.name, 'first time monitoring')
elif savedminer.status == mining.MinerStatus.Online and (mineroriginalstatus == mining.MinerStatus.Disabled or mineroriginalstatus == mining.MinerStatus.Offline):
#changing status from offline to online so raise event
entries.add(QueueName.Q_ONLINE, APPMONITOR.app.messageencode(savedminer))
print(Fore.GREEN + APPMONITOR.app.now(), savedminer.name, 'back online!')
#TODO: if stats.elapsed < previous.elapsed then raise provision or online events
APPMONITOR.app.putminerandstats(savedminer, minerstats, minerpool)
#TODO:show name of current pool instead of worker
poolname = '?'
if minerpool:
poolname = '{0} {1}'.format(minerpool.currentpool, minerpool.currentworker)
foundpool = APPMONITOR.app.findpool(minerpool)
if foundpool is not None:
poolname = foundpool.name
print('{0} mining at {1}'.format(savedminer.name, poolname))
#most users won't want to mine solo, so provision the miner
if not APPMONITOR.app.configuration('mining.allowsolomining'):
if not minerpool.currentpool or minerpool.currentpool.startswith(APPMONITOR.app.configuration('mining.solopool')):
entries.add(QueueName.Q_PROVISION, APPMONITOR.app.messageencode(savedminer))
print(Fore.CYAN+str(APPMONITOR.app.now()), savedminer.name, savedminer.status,
'h='+str(minerstats.currenthash), str(minerstats.minercount),
'{0}/{1}/{2}'.format(str(minerstats.tempboard1),
str(minerstats.tempboard2),
str(minerstats.tempboard3)),
savedminer.uptime(minerstats.elapsed),
'{0:d}ms'.format(int(savedminer.monitorresponsetime() * 1000)))
msg = APPMONITOR.app.createmessagestats(savedminer, minerstats, minerpool)
entries.addbroadcast(QueueName.Q_STATISTICSUPDATED, msg)
except pika.exceptions.ConnectionClosed as qex:
#could not enqueue a message
print(Fore.RED + '{0} Queue Error: {1}'.format(savedminer.name,
APPMONITOR.app.exceptionmessage(qex)))
APPMONITOR.app.logexception(qex)
except MinerMonitorException as monitorex:
print(Fore.RED + '{0} Miner Error: {1}'.format(savedminer.name,
APPMONITOR.app.exceptionmessage(monitorex)))
savedminer.lastmonitor = datetime.datetime.utcnow()
#TODO: this should be a rule. publish miner offline event
#and let event handler decide how to handle it
savedminer.offline_now()
print(Fore.RED + APPMONITOR.app.now(), savedminer.name, savedminer.status)
entries.add(QueueName.Q_OFFLINE, APPMONITOR.app.messageencode(savedminer))
except BaseException as ex:
print(Fore.RED+'{0} Unexpected Error in monitorminer: {1}'.format(savedminer.name,
APPMONITOR.app.exceptionmessage(ex)))
# we have to consider any exception to be a miner error. sets status to offline
#if str(e) == "timed out": #(timeout('timed out',),)
APPMONITOR.app.logexception(ex)
#TODO: review usage of savedminer and knownminer. should only go with one
APPMONITOR.app.putminer(savedminer)
APPMONITOR.app.updateknownminer(savedminer)
return entries
'''
A test program that pushes monitoring messages onto the queue
A stress tester for monitoring
when_monitorminer.py should be running to process the messages
'''
from helpers.queuehelper import QueueName, QueueEntries
from backend.fcmapp import ApplicationService, ComponentName
APP = ApplicationService(ComponentName.fullcycle)
MINERS = APP.miners()
cnt = 1
while cnt < 1000:
entries = QueueEntries()
for miner in MINERS:
entries.add(QueueName.Q_MONITORMINER, APP.messageencode(miner))
APP.enqueue(entries)
print('sent {} messages'.format(len(entries.entries)))
cnt += 1