def __init__(self):
threading.Thread.__init__ (self)
Component.__init__(self)
self.stats = Stats.getInstance()
# Some thread related stuff
self.daemon = True
self.kill_switch = False
# The socket framework
self.context = zmq.Context()
self.frontend = self.context.socket(zmq.PULL)
self.frontend.bind('tcp://*:{port}'.format(port=self.frontend_port))
self.frontend.setsockopt(zmq.LINGER, 0)
self.frontend.set_hwm(0)
self.backend = self.context.socket(zmq.PUSH)
self.backend.bind('tcp://*:{port}'.format(port=self.backend_port))
self.backend.setsockopt(zmq.LINGER, 0)
self.backend.set_hwm(0)
# The poller is used to poll for incomming messages for both
# the frontend (internet) and the backend (scheduling)
self.poll = zmq.Poller()
self.poll.register(self.frontend, zmq.POLLIN)
# Connected socket locally to frontend to send tasks, this socket
# provides a lock to be able to be thread-safe
self.frontend_push = self.context.socket(zmq.PUSH)
self.frontend_push.connect('tcp://localhost:{port}'.format(port=self.frontend_port))
self.frontend_push.setsockopt(zmq.LINGER, 0)
self.frontend_push.set_hwm(0)
# Our lock used to protect the frontend_push socket
self.lock = threading.Lock()
def app_main(self):
"""
Launch a concurrent application
"""
# Generate rest API
self.generate_api()
# Now run our API listener
self.log.debug("Hosting application on port %d" % (self.port))
# Get a ref to our stats helper
self.stats = Stats.getInstance()
# Create cryto helper used for network communciation
self.crypto_helper = CryptoHelper(self.salt_size, self.num_iterations,
self.aes_padding)
# Make sure the URL proxy knows us
global global_hook
global_hook = GlobalHook({'node': self})
#api should only be there for the master node and used for node registration and heartbeats. Each node will have a socket
#while slave nodes will have a local server too. There servers are no web servers because they are too expensive!
#refactor server thingy tomorrow and add client which will be connected with the server through a normal socket!
#The master server will act as only that, a controller and will distribute work using a better performing mechanism: UDP?
#Use asycn calls for heartbeat for example
#Create the server the same way the guys from PP do! (See ppserver) Try using a multithreaded pool to handle connections instead of threads!
self.api_thread = api_thread(self.log, self.urls, self.port,
self.use_gzip)
self.api_thread.daemon = True
self.api_thread.start()
self.heartbeat_threshold = self.heartbeat_timer
self.current_time = 0
self.last_time = 0
self.last_delta_time = 0
self.stats_dump_threshold = self.stats_dump_timer
# Bool flag used to control the main loop
self.kill_received = False
# Give us some time until its up
time.sleep(0.5)
return APP_RET_CODE_SUCCESS
def app_main(self):
"""
Launch a concurrent application
"""
# Generate rest API
self.generate_api()
# Now run our API listener
self.log.debug("Hosting application on port %d" % (self.port))
# Get a ref to our stats helper
self.stats = Stats.getInstance()
# Create cryto helper used for network communciation
self.crypto_helper = CryptoHelper(self.salt_size, self.num_iterations, self.aes_padding)
# Make sure the URL proxy knows us
global global_hook
global_hook = GlobalHook({'node':self})
#api should only be there for the master node and used for node registration and heartbeats. Each node will have a socket
#while slave nodes will have a local server too. There servers are no web servers because they are too expensive!
#refactor server thingy tomorrow and add client which will be connected with the server through a normal socket!
#The master server will act as only that, a controller and will distribute work using a better performing mechanism: UDP?
#Use asycn calls for heartbeat for example
#Create the server the same way the guys from PP do! (See ppserver) Try using a multithreaded pool to handle connections instead of threads!
self.api_thread = api_thread(self.log, self.urls, self.port, self.use_gzip)
self.api_thread.daemon = True
self.api_thread.start()
self.heartbeat_threshold = self.heartbeat_timer
self.current_time = 0
self.last_time = 0
self.last_delta_time = 0
self.stats_dump_threshold = self.stats_dump_timer
# Bool flag used to control the main loop
self.kill_received = False
# Give us some time until its up
time.sleep(0.5)
return APP_RET_CODE_SUCCESS
def run(self):
self._pid = os.getpid()
#self.context = zmq.Context()
#self.work_receiver = context.socket(zmq.PULL)
#self.work_receiver.connect("tcp://127.0.0.1:%d" % WORKER_PORT)
# Create sockets to communicate with master, this way we optimize our resources
self.socket=TCPSocketZMQ("{}_{}".format(self._id, self.identity), self.host, self.port)
self.socket.connect()
self.stats = Stats.getInstance()
self.log("Running")
while True:
try:
#next_task = receive_from_zmq_zipped(self.work_receiver)
next_task = self.task_queue.get()
if next_task is None:
# A None task is used to shut us down
self.task_queue.task_done()
break
result = None
try:
#start = time.time()
result = next_task()
self.task_queue.task_done()
#ellapsed = time.time() - start
error = None
#self.stats.add_avg("task-time",ellapsed)
#self.log("Finished [%s:%s]" % (next_task.name, next_task.task_id))
except Exception as err:
result = None
error = err
finally:
#self.result_queue.put(Bunch({'task':next_task,'result':result,'error':error}))
#print("sending back")
next_task.clean_up()
self.socket.send_to('task_finished', next_task, result, error)
except KeyboardInterrupt:
self.log("Keyboard interrupt received, exiting!")
break
self.socket.close()
#self.work_receiver.close()
#self.context.term()
self.log("Exiting")
def __init__(self):
Component.__init__(self)
self.stats = Stats.getInstance()
# Map that maps tasks and slaves to be able to resend the tasks if the slave was deleted from the system
self.task_map = {}
def __init__(self, ProxyObject, log):
self._obj=ProxyObject
self.log = log
self.stats = Stats.getInstance()
def __init__(self, ProxyObject, log, ok_cb, ko_cb):
self._obj=ProxyObject
self.log = log
self.ok_cb = ok_cb
self.ko_cb = ko_cb
self.stats = Stats.getInstance()
def __init__(self, socket, identity, log):
self.socket=socket
self.identity=identity
self.log = log
self.stats = Stats.getInstance()
def __init__(self, ProxyObject, log, ok_cb, ko_cb):
self._obj = ProxyObject
self.log = log
self.ok_cb = ok_cb
self.ko_cb = ko_cb
self.stats = Stats.getInstance()
def __init__(self, socket, identity, log):
self.socket = socket
self.identity = identity
self.log = log
self.stats = Stats.getInstance()
def __init__(self, ProxyObject, log):
self._obj = ProxyObject
self.log = log
self.stats = Stats.getInstance()