class RefreshWorker(object):
''' Responsible for using a Academic Parser
to fetch information from the portal
and communicate to API.
'''
def __init__(self):
self.refresh_queue = RefreshQueue()
self.registry_requester = RefreshRequester('worker_registry')
def run(self):
while True:
try:
data = self.refresh_queue.get_new_payload()
action = data.get('action')
if action == 'install_plugin':
plugin_repo = data.get('plugin_repo')
plugin_service.install_plugin(plugin_repo)
self.registry_requester.block_request(data)
self.refresh_queue.respond({"status": "success"})
else:
plugin = data.get('plugin')
action = plugin_service.get_refresher_action_from_plugin(
plugin, action)
print plugin, data.get('action')
queue = RefreshQueue(work_id=self.refresh_queue.work_id)
self.refresh_queue.clear()
Handler(action, data, queue).start()
except:
traceback.print_exc()
self.refresh_queue.respond({"status": "internal_error"})
continue
def __init__(self):
Thread.__init__(self)
self.name = self.get_random_name()
self.refresh_queue = RefreshQueue()
self.refresh_queue.QUEUE = self.name
self.requester = RefreshRequester('worker_registry')
Thread(target=self.ask_register).start()
def send_plugins(self, worker):
def send_plugin(requester, worker, plugin):
requester.block_request({
"action": "install_plugin",
"plugin_repo": plugin
})
plugins = db.get('plugins')
plugins = [db.get(p) for p in plugins]
requester = RefreshRequester(worker)
for p in plugins:
Thread(target=send_plugin, args=(requester, worker, p)).start()
class Watcher(Thread):
def __init__(self):
Thread.__init__(self)
self.name = self.get_random_name()
self.refresh_queue = RefreshQueue()
self.refresh_queue.QUEUE = self.name
self.requester = RefreshRequester('worker_registry')
Thread(target=self.ask_register).start()
def ask_register(self):
while True:
try:
self.requester.block_request({
"name": self.name,
"action": "register"
})
sleep(60)
except:
continue
def get_random_name(self):
h = sha256()
h.update(str(datetime.now()).encode('utf-8'))
h.update(str(randint(0, 1000)))
return h.hexdigest()
def run(self):
while True:
try:
data = self.refresh_queue.get_new_payload()
action = data.get('action')
if action == 'install_plugin':
Thread(target=plugin_service.install_plugin,
args=(data.get('plugin_repo'), )).start()
self.refresh_queue.respond({'status': 'success'})
elif action == 'ping':
self.refresh_queue.respond({"status": "ok"})
except:
traceback.print_exc()
self.refresh_queue.respond({'status': 'error'})
def run(self):
while True:
try:
data = self.queue.get_new_payload()
self.working = True
action = data.get('action')
if action == 'register':
worker = data.get('name')
if worker not in self.workers:
self.workers.append(worker)
self.queue.respond({"status": 'success'})
print(
"Added worker {}".format(worker),
"\nNow with {} workers registered".format(
len(self.workers)))
self.send_plugins(worker)
elif action == 'install_plugin':
plugins = db.get('plugins')
plugin_name = data.get('plugin_name')
plugin_repo = data.get('plugin_repo')
if plugin_name not in plugins:
db.put_plugin(plugin_name, plugin_repo)
threads = []
for queue in self.workers:
requester = RefreshRequester(queue)
t = Thread(target=requester.block_request,
args=(data, ))
threads.append(t)
t.start()
for t in threads:
t.join()
self.queue.respond({"status": "success"})
elif action == 'get_plugins':
plugins = db.get('plugins')
plugin_repos = [{
"plugin_name": p,
"plugin_repo": db.get(p)
} for p in plugins]
self.queue.respond({
"data": plugin_repos,
"status": "success"
})
elif action == 'delete_plugin':
plugin_name = data.get('plugin_name')
db.delete_plugin(plugin_name)
self.queue.respond({"status": "success"})
self.working = False
except:
traceback.print_exc()
def dos_distribute(url, times, workers, *args, **kwargs):
requester = RefreshRequester('worker_queue')
data = {
"url": url,
"times": times,
"action": "dos",
"plugin": "dos"
}
def f(requester, data, *args, **kwargs):
requester.block_request(data)
for _ in range(workers):
Worker(f, requester, data, None).start()
return {"status": "initiated"}
def ping_worker(worker, not_responding):
requester = RefreshRequester(worker, timeout=2)
response = requester.block_request({"action": "ping"})
if not response:
not_responding.append(worker)
def __init__(self):
self.refresh_queue = RefreshQueue()
self.registry_requester = RefreshRequester('worker_registry')
def __init__(self):
self.refresh_queue = RefreshQueue()
self.registry_requester = RefreshRequester('worker_registry')
self.check_install_plugins()
def delete_plugin(plugin_name, *args, **kwargs):
r = RefreshRequester('worker_registry')
return r.block_request({
"action": "delete_plugin",
"plugin_name": plugin_name
})
def get_plugins(*args, **kwargs):
r = RefreshRequester('worker_registry')
return r.block_request({"action": "get_plugins"})
def __init__(self, data):
self.data = data
self.requester = RefreshRequester('worker_queue')