def render_POST_advanced(self, request, response): print("Received Task Request!") # unpack request wrapper = messages_pb2.WrapperMessage() wrapper.ParseFromString(request.payload) # print request (do nothing right now) print(" Framework Name: " + wrapper.run_task.task.framework.name) print(" Framework ID: " + wrapper.run_task.task.framework.framework_id) print(" Task Name: " + wrapper.run_task.task.name) print(" Task ID: " + wrapper.run_task.task.task_id) print(" Selected Agent: " + wrapper.run_task.task.agent_id) for i in range(len(wrapper.run_task.task.resources)): resource = wrapper.run_task.task.resources[i] print(" Resource: (" + resource.name + ") type: " + str(resource.type) + " amt: " + str(resource.scalar).strip()) # TODO: Forward the request onto the particular device through a ping/pong db.add_task(wrapper.run_task) # construct response wrapper = messages_pb2.WrapperMessage() wrapper.pong.agent_id = "1234" response.payload = wrapper.SerializeToString() response.code = defines.Codes.CHANGED.number response.content_type = defines.Content_types[ "application/octet-stream"] return self, response
def render_POST_advanced(self, request, response): wrapper = messages_pb2.WrapperMessage() wrapper.ParseFromString(request.payload) framework_id = wrapper.request.framework_id #first, clear any agents that have dropped off db.clear_stale_agents() #construct resource offer print("\nGot resource offer request! Framework \"" + framework_id + "\"\n") #currently just giving the framework everything we got wrapper = messages_pb2.WrapperMessage() wrapper.offermsg.framework_id = framework_id for agent in db.get_all_agents(): offer = wrapper.offermsg.offers.add() offer.id = db.get_offer_id() offer.framework_id = framework_id offer.agent_id = agent.id offer.resources.extend(agent.resources) offer.attributes.extend(agent.attributes) response.payload = wrapper.SerializeToString() response.code = defines.Codes.CHANGED.number response.content_type = defines.Content_types[ "application/octet-stream"] return self, response
def render_POST_advanced(self, request, response): #unpack request wrapper = messages_pb2.WrapperMessage() wrapper.ParseFromString(request.payload) agent_id = wrapper.ping.agent.id agent_name = wrapper.ping.agent.name if not agent_id: return self print("Ping! Agent ID:(" + str(agent_id) + ") Name:(" + str(agent_name) + ")") #refresh the agent timing db.refresh_agent(agent_id, wrapper.ping.agent) #update the state of any tasks it may have sent db.refresh_tasks(wrapper.ping.tasks) task_to_run = db.get_next_unissued_task_by_agent(agent_id) # construct response wrapper = messages_pb2.WrapperMessage() wrapper.pong.agent_id = str(agent_id) if task_to_run: print("Got a task to schedule!!!") wrapper.pong.run_task.task.CopyFrom(task_to_run) response.payload = wrapper.SerializeToString() response.code = defines.Codes.CONTENT.number # response.code = defines.Codes.CHANGED.number response.content_type = defines.Content_types[ "application/octet-stream"] return self, response
def submitDummyTask(offers): print("Searching for a good offer...") slave_to_use = None resources_to_use = {} for i in reversed(range(len(offers))): offer = offers[i] if offer.slave_id: resources_to_use = {} for resource in offer.resources: if resource.name == "cpus" and resource.scalar.value >= 1: resources_to_use["cpus"] = 1 if resource.name == "mem" and resource.scalar.value > 100000000: resources_to_use["mem"] = 100000000 if len(resources_to_use) == 2: slave_to_use = offer.slave_id break if not slave_to_use: print("No available agents...") return print("Submitting task to agent " + slave_to_use + "...") # construct message wrapper = messages_pb2.WrapperMessage() wrapper.run_task.task.framework.name = framework_name wrapper.run_task.task.framework.framework_id = framework_id wrapper.run_task.task.name = "test task" wrapper.run_task.task.task_id = str(uuid.uuid1()) wrapper.run_task.task.slave_id = slave_to_use for resource in resources_to_use: r = wrapper.run_task.task.resources.add() r.name = resource r.type = messages_pb2.Value.SCALAR r.scalar.value = resources_to_use[resource] # wrapper.run_task.task.resources.extend(resources_to_use) wrapper.run_task.task.container.type = messages_pb2.ContainerInfo.Type.DOCKER wrapper.run_task.task.container.docker.image = "hello-world" wrapper.run_task.task.container.docker.network = messages_pb2.ContainerInfo.DockerInfo.Network.HOST port_mapping = wrapper.run_task.task.container.docker.port_mappings.add() port_mapping.host_port = 3000 port_mapping.container_port = 3000 runtask_payload = wrapper.SerializeToString() ct = {'content_type': defines.Content_types["application/octet-stream"]} response = client.post('task', runtask_payload, timeout=2, **ct) if response: wrapper = messages_pb2.WrapperMessage() wrapper.ParseFromString(response.payload) print("Task Running!") #TODO: Generate confirmation protobuf message else: print("Failed to submit task...") client.stop() sys.exit(1)
def main(host, port): # pragma: no cover global client try: tmp = socket.gethostbyname(host) host = tmp except socket.gaierror: pass client = HelperClient(server=(host, int(port))) # construct message wrapper = messages_pb2.WrapperMessage() constructPing(wrapper) register_payload = wrapper.SerializeToString() print("Registering with master...") print("My Agent ID is " + agent_id) print("My Agent name is " + agent_name) # loop ping/pong try: while True: time.sleep(ping_rate / 1000) wrapper = messages_pb2.WrapperMessage() constructPing(wrapper) print("") print("Ping!") ct = {'content_type': defines.Content_types["application/octet-stream"]} response = client.post('ping', wrapper.SerializeToString(), timeout=2, **ct) if response: print("Pong!") wrapper = messages_pb2.WrapperMessage() wrapper.ParseFromString(response.payload) if wrapper.pong.run_task.task.name: if wrapper.pong.run_task.task.container.type == messages_pb2.ContainerInfo.Type.DOCKER: print("Received Docker Task!!") print("Storing task") tasks[wrapper.pong.run_task.task.task_id] = wrapper.pong.run_task.task print("Launching task") #for now just grab the container info. Let ping check the state on the next run containerInfo = dockerhelper.runImageFromRunTask(wrapper.pong.run_task) else: print("Agent cannot run this type of task") except KeyboardInterrupt: print("Client Shutdown") # TODO: Deregister client.stop()
def GetPing(wrapper): lock.acquire() agent_id = wrapper.ping.agent.id agent_name = wrapper.ping.agent.name if not agent_id: return self print("Ping! Agent ID:(" + str(agent_id) + ") Name:(" + str(agent_name) + ")") #refresh the agent timing db.refresh_agent(agent_id, wrapper.ping.agent) #update the state of any tasks it may have sent db.refresh_tasks(agent_id, wrapper.ping.tasks) task_to_run = db.get_next_unissued_task_by_agent(agent_id) task_to_kill = db.get_next_unissued_kill_by_agent(agent_id) # construct response wrapper = messages_pb2.WrapperMessage() wrapper.type = messages_pb2.WrapperMessage.Type.PONG wrapper.pong.agent_id = str(agent_id) if task_to_run: print("Got a task to schedule!!!") wrapper.pong.run_task.task.CopyFrom(task_to_run) if task_to_kill: print("Got a task to kill!") wrapper.pong.kill_task.CopyFrom(task_to_kill) lock.release() return wrapper.SerializeToString()
def post_ping(): wrapper = messages_pb2.WrapperMessage() try: wrapper.ParseFromString(flask.request.get_data()) except: print("Error parsing protobuf.") return "Error parsing protobuf." return GetPing(wrapper)
def getOffer(): # get offers print("Requesting resource offers...") wrapper = messages_pb2.WrapperMessage() wrapper.request.framework_id = framework_id request_payload = wrapper.SerializeToString() ct = {'content_type': defines.Content_types["application/octet-stream"]} response = client.post('request', request_payload, timeout=2, **ct) if response: wrapper = messages_pb2.WrapperMessage() wrapper.ParseFromString(response.payload) offers = wrapper.offermsg.offers print("Got offers!") return offers else: print("Couldn't receive resource offer... ?") client.stop() sys.exit(1)
def getOffer(host, port): # get offers print("Requesting resource offers...") wrapper = messages_pb2.WrapperMessage() wrapper.type = messages_pb2.WrapperMessage.Type.RESOURCE_REQUEST wrapper.request.framework_id = framework_id response = requests.post("http://" + host + ":" + str(port) + '/request', data=wrapper.SerializeToString(), timeout=2, headers={'Content-Type': 'application/protobuf'}) if response: wrapper = messages_pb2.WrapperMessage() wrapper.ParseFromString(response.content) offers = wrapper.offermsg.offers print("Got offers!") return offers else: print("Couldn't receive resource offer... ?") sys.exit(1)
def render_POST_advanced(self, request, response): wrapper = messages_pb2.WrapperMessage() try: wrapper.ParseFromString(request.payload) except: print("Error parsing protobuf - not responding.") response.code = defines.Codes.BAD_REQUEST.number return self, response response.payload = GetResourceOffer(wrapper) response.code = defines.Codes.CHANGED.number response.content_type = defines.Content_types[ "application/octet-stream"] return self, response
async def entry(websocket, path): try: while True: msg = await websocket.recv() wrapper = messages_pb2.WrapperMessage() wrapper.ParseFromString(msg) if wrapper.HasField('register_slave'): await register_slave(websocket, wrapper) else: print("Unsupported message...") except websockets.exceptions.ConnectionClosedError: pass finally: await unregister_slave(websocket)
def KillTask(wrapper): lock.acquire() # print request (do nothing right now) print(" Framework Name: " + wrapper.kill_task.framework.name) print(" Framework ID: " + wrapper.kill_task.framework.framework_id) print(" Task Name: " + wrapper.kill_task.name) print(" Task ID: " + wrapper.kill_task.task_id) print(" Selected Agent: " + wrapper.kill_task.agent_id) db.add_kill_task(wrapper.kill_task) # construct response wrapper = messages_pb2.WrapperMessage() wrapper.type = messages_pb2.WrapperMessage.Type.PONG wrapper.pong.agent_id = wrapper.kill_task.agent_id lock.release() return wrapper.SerializeToString()
async def register_slave(websocket, wrapper): print("Registering slave...") # build agent info resources = [] for resource in wrapper.register_slave.slave.resources: if resource.type == messages_pb2.Value.SCALAR: resources.append( [resource.name, resource.type, resource.scalar.value]) elif resource.type == messages_pb2.Value.SET: resources.append([resource.name, resource.type, resource.set.item]) elif resource.type == messages_pb2.Value.RANGE: resources.append( [resource.name, resource.type, resource.ranges.range]) elif resource.type == messages_pb2.Value.TEXT: resources.append( [resource.name, resource.type, resource.text.value]) # build agent info attributes = [] for resource in wrapper.register_slave.slave.attributes: if resource.type == messages_pb2.Value.SCALAR: attributes.append( [resource.name, resource.type, resource.scalar.value]) elif resource.type == messages_pb2.Value.SET: attributes.append( [resource.name, resource.type, resource.set.item]) elif resource.type == messages_pb2.Value.RANGE: attributes.append( [resource.name, resource.type, resource.ranges.range]) elif resource.type == messages_pb2.Value.TEXT: attributes.append( [resource.name, resource.type, resource.text.value]) # add agent to db agent_id = db.add_agent(resources, attributes, "webs") sockets[websocket] = agent_id print(db.get_all()) # send response message response = messages_pb2.WrapperMessage() response.slave_registered.slave_id = str(agent_id) await websocket.send(response.SerializeToString())
def render_POST_advanced(self, request, response): #unpack request print("Recevied CoAP ping request: " + str(request)) wrapper = messages_pb2.WrapperMessage() try: wrapper.ParseFromString(request.payload) except: print("Error parsing protobuf - not responding.") response.code = defines.Codes.BAD_REQUEST.number return self, response response.payload = GetPing(wrapper) response.code = defines.Codes.CONTENT.number # response.code = defines.Codes.CHANGED.number response.content_type = defines.Content_types[ "application/octet-stream"] return self, response
def render_POST_advanced(self, request, response): print("Received Kill Task!") # unpack request wrapper = messages_pb2.WrapperMessage() try: wrapper.ParseFromString(request.payload) except: print("Error parsing protobuf - not responding.") response.code = defines.Codes.BAD_REQUEST.number return self, response # construct response wrapper = KillTask(wrapper) wrapper.type = messages_pb2.WrapperMessage.Type.PONG wrapper.pong.agent_id = wrapper.kill_task.agent_id response.payload = wrapper.SerializeToString() response.code = defines.Codes.CHANGED.number response.content_type = defines.Content_types[ "application/octet-stream"] return self, response
def GetResourceOffer(wrapper): lock.acquire() framework_id = wrapper.request.framework_id #first, clear any agents that have dropped off db.clear_stale_agents() #construct resource offer subtracting the resources of any tasks in the task queue print("\nGot resource offer request! Framework \"" + framework_id + "\"\n") #currently just giving the framework everything we got wrapper = messages_pb2.WrapperMessage() wrapper.type = messages_pb2.WrapperMessage.Type.RESOURCE_OFFER wrapper.offermsg.framework_id = framework_id for agent in db.get_all_agents(): offer = messages_pb2.Offer() offer.id = db.get_offer_id() offer.framework_id = framework_id offer.agent_id = agent.id offer.attributes.extend(agent.attributes) offer.resources.extend(agent.resources) #What are free resources? # The amount of resources that are free # plus the amount of resources currently being used <- how do we collect this one? for now ignore # minus the amount of resources dedicated running or pending tasks # minus the amount of resources that are currently in valid offers #get pending tasks for agent to subract pending resources pending_tasks = db.get_all_pending_tasks_by_agent(agent.id) #currently only do subtraction for scalars for resource in offer.resources: for task in pending_tasks: for tresource in task.resources: if resource.name == tresource.name: if resource.scalar.value and tresource.scalar.value: resource.scalar.value -= tresource.scalar.value pending_offers = db.get_offers_by_agent_id(agent.id) for resource in offer.resources: for off in pending_offers: if time.time() < off.expiration_time: for tresource in off.resources: if resource.name == tresource.name: if resource.scalar.value and tresource.scalar.value: resource.scalar.value -= tresource.scalar.value removal_indexes = [] for i, r in enumerate(offer.resources): if r.type == messages_pb2.Value.Type.SCALAR and r.scalar.value is not None and r.scalar.value <= 0.001: removal_indexes.append(i) #removal_list for index in sorted(removal_indexes, reverse=True): del offer.resources[index] #set the offer time offer.offer_time = time.time() #set the offer expiration offer.expiration_time = time.time() + offer_timeout #add the offer to the db db.add_offer(offer.id, agent.id, offer) if len(offer.resources) > 0: wrapper.offermsg.offers.append(offer) lock.release() return wrapper.SerializeToString()
def main(host, port, configPath, verbose): # pragma: no cover global agent_name global ping_rate try: tmp = socket.gethostbyname(host) host = tmp except socket.gaierror: pass db.load() dockerhelper.load() #get the devices configuration config = dict() if configPath: config = parseConfig(configPath) print(config) print("My Agent ID is " + agent_id) if 'name' in config: agent_name = config['name'] if 'pingRate' in config: ping_rate = config['pingRate'] print("My Agent name is " + agent_name) wasmhelper.establishWasmRuntime(config) # loop ping/pong try: while True: wrapper = messages_pb2.WrapperMessage() constructPing(wrapper, config, verbose) print("") print("Ping!") try: response = requests.post( "http://" + host + ":" + str(port) + '/ping', data=wrapper.SerializeToString(), timeout=2, headers={'Content-Type': 'application/protobuf'}) if response and response.content: print("Pong!") wrapper = messages_pb2.WrapperMessage() wrapper.ParseFromString(response.content) if wrapper.pong.kill_task.task_id: print("Received kill task request!") dockerhelper.killContainer( wrapper.pong.kill_task.task_id) wasmhelper.killTask(wrapper.pong.kill_task.task_id) if wrapper.pong.run_task.task.name: if wrapper.pong.run_task.task.container.type == messages_pb2.ContainerInfo.Type.DOCKER: print("Received Docker Task!!") print("Storing task") db.set_task(wrapper.pong.run_task.task.task_id, wrapper.pong.run_task.task) print("Launching task") #for now just grab the container info. Let ping check the state on the next run containerInfo = dockerhelper.runImageFromRunTask( wrapper.pong.run_task, config.get('devices', [])) elif wrapper.pong.run_task.task.container.type == messages_pb2.ContainerInfo.Type.WASM: print("Received WebAssembly Task!!") if not wasmhelper.enabled(): print("Agent cannot run this type of task") else: print("Storing task") db.set_task(wrapper.pong.run_task.task.task_id, wrapper.pong.run_task.task) print("Launching task") wasmhelper.runModuleFromRunTask( wrapper.pong.run_task) else: print("Agent cannot run this type of task") except requests.exceptions.ConnectionError: print("ERROR: Connection Error") except requests.exceptions.ReadTimeout: print("WARNING: Ping Response timeout") time.sleep(ping_rate / 1000) except KeyboardInterrupt: wasmhelper.stopWasmRuntime() print("Shutting down...")
def submitRunTask(host, port, name, agent_to_use, resources_to_use, offer_id, dockerimg, port_mappings, env_variables, domain_to_store=''): print("Submitting task to agent " + agent_to_use + "...") # construct message wrapper = messages_pb2.WrapperMessage() wrapper.type = messages_pb2.WrapperMessage.Type.RUN_TASK wrapper.run_task.offer_id = offer_id wrapper.run_task.task.framework.name = framework_name wrapper.run_task.task.framework.framework_id = framework_id wrapper.run_task.task.name = name task_id = str(uuid.uuid1()) wrapper.run_task.task.task_id = task_id wrapper.run_task.task.agent_id = agent_to_use for resource in resources_to_use: r = wrapper.run_task.task.resources.add() r.name = resource if r.name == 'picam': r.type = messages_pb2.Value.DEVICE else: r.type = messages_pb2.Value.SCALAR r.scalar.value = resources_to_use[resource] # wrapper.run_task.task.resources.extend(resources_to_use) wrapper.run_task.task.container.type = messages_pb2.ContainerInfo.Type.DOCKER wrapper.run_task.task.container.docker.image = dockerimg wrapper.run_task.task.container.docker.network = messages_pb2.ContainerInfo.DockerInfo.Network.HOST for host_port, container_port in port_mappings.items(): port_mapping = wrapper.run_task.task.container.docker.port_mappings.add( ) port_mapping.host_port = host_port port_mapping.container_port = container_port wrapper.run_task.task.container.docker.environment_variables.extend( env_variables) response = requests.post("http://" + host + ":" + str(port) + '/task', data=wrapper.SerializeToString(), timeout=2, headers={'Content-Type': 'application/protobuf'}) if response: wrapper = messages_pb2.WrapperMessage() wrapper.ParseFromString(response.content) if wrapper.type == messages_pb2.WrapperMessage.Type.ERROR: print("Task issuance failed", file=sys.stderr) print(wrapper.error.error, file=sys.stderr) else: print("Task Running!") tasks.append({ 'name': name, 'task_id': task_id, 'agent_id': agent_to_use, 'domain': domain_to_store }) dumpTasks() #TODO: Generate confirmation protobuf message else: print("Failed to submit task.", file=sys.stderr) sys.exit(1)
def RunTask(wrapper): lock.acquire() # print request (do nothing right now) print(" Framework Name: " + wrapper.run_task.task.framework.name) print(" Framework ID: " + wrapper.run_task.task.framework.framework_id) print(" Task Name: " + wrapper.run_task.task.name) print(" Task ID: " + wrapper.run_task.task.task_id) print(" Selected Agent: " + wrapper.run_task.task.agent_id) for i in range(len(wrapper.run_task.task.resources)): resource = wrapper.run_task.task.resources[i] print(" Resource: (" + resource.name + ") type: " + str(resource.type) + " amt: " + str(resource.scalar).strip()) # Verify that the RunTaskRequest is on an unexpired offer of the right resources and try: offer = db.get_offer_by_offer_id(wrapper.run_task.offer_id) except ValueError: print("Not a valid offer ID") wrapper = messages_pb2.WrapperMessage() wrapper.type = messages_pb2.WrapperMessage.Type.ERROR wrapper.error.error = "Not a valid offer ID" lock.release() return wrapper.SerializeToString() #unexpired if time.time() > offer.expiration_time: print("Offer expired") wrapper = messages_pb2.WrapperMessage() wrapper.type = messages_pb2.WrapperMessage.Type.ERROR wrapper.error.error = "Offer Expired" lock.release() return wrapper.SerializeToString() #check for valid resources - just scalars for now for tresource in wrapper.run_task.task.resources: r = { x.name: (x.type, x.scalar, x.ranges, x.set, x.text, x.device) for x in offer.resources } if tresource.name not in r: print("Resource not in offer") wrapper = messages_pb2.WrapperMessage() wrapper.type = messages_pb2.WrapperMessage.Type.ERROR wrapper.error.error = "Resource not in offer" lock.release() return wrapper.SerializeToString() elif tresource.type != r[tresource.name][0]: print("Resource type doesn't match offer") wrapper = messages_pb2.WrapperMessage() wrapper.type = messages_pb2.WrapperMessage.Type.ERROR wrapper.error.error = "Resource type doesn't match offer" lock.release() return wrapper.SerializeToString() elif tresource.scalar.value and r[ tresource.name][1].value and tresource.scalar.value > r[ tresource.name][1].value: print("Resource value exceeds offer") wrapper = messages_pb2.WrapperMessage() wrapper.type = messages_pb2.WrapperMessage.Type.ERROR wrapper.error.error = "Resource value exceeds offer" lock.release() return wrapper.SerializeToString() # TODO: Forward the request onto the particular device through a ping/pong db.add_task(wrapper.run_task) # construct response wrapper = messages_pb2.WrapperMessage() wrapper.type = messages_pb2.WrapperMessage.Type.PONG wrapper.pong.agent_id = wrapper.run_task.task.agent_id lock.release() return wrapper.SerializeToString()