def __init__(self, inboxer=Inboxer(), docker_agent=None): self.inboxer = inboxer self.inboxer.on("received", self.__handle_inbox_received_event) self.docker_agent = docker_agent self.id_to_cpu = {} self.id_to_metadata = {} if self.docker_agent is None: self.docker_agent = DockerAgent() # Setup pending queue self.pending = {} self.pending_thread = Thread(target=self.__process_pending) self.pending_thread.daemon = True self.pending_thread.start()
class Orchestrator(object): def __init__(self, inboxer=Inboxer(), docker_agent=None): self.inboxer = inboxer self.inboxer.on("received", self.__handle_inbox_received_event) self.docker_agent = docker_agent self.id_to_cpu = {} self.id_to_metadata = {} if self.docker_agent is None: self.docker_agent = DockerAgent() # Setup pending queue self.pending = {} self.pending_thread = Thread(target=self.__process_pending) self.pending_thread.daemon = True self.pending_thread.start() # Clean up the id_to_cpu dict def __clean_container(self, container_id): if container_id in self.id_to_metadata: meta = self.id_to_metadata[container_id] Event.create(topic=meta['topic'], container=meta['container'], timestamp=meta['timestamp'], duration=(int(time() * 1000) - meta['timestamp'])) del self.id_to_metadata[container_id] if container_id in self.id_to_cpu: del self.id_to_cpu[container_id] # Save stat to database def __handle_stat(self, stat, id, topic, container_name): try: read_dt = parser.parse(stat['read']) timestamp = int((time.mktime(read_dt.timetuple()) + (read_dt.microsecond / 1000000.0)) * 1000) memory_usage = float(stat['memory_stats']['usage']) / float( stat['memory_stats']['limit']) Metric.create(topic=topic, container=container_name, timestamp=timestamp, name='memory', value=memory_usage) # Calculate CPU usage. The docker API returns the number of cycles consumed # by the container and the number of cycles consumed by the system. We need # to take the difference over time and divide them to retrieve the usage # percentage. total_usage = float(stat['cpu_stats']['cpu_usage']['total_usage']) system_usage = float(stat['cpu_stats']['system_cpu_usage']) if id in self.id_to_cpu: usage_diff = total_usage - self.id_to_cpu[id]['total'] system_diff = system_usage - self.id_to_cpu[id]['system'] if usage_diff >= 0: usage_pct = usage_diff / system_diff else: usage_pct = 0.0 Metric.create(topic=topic, container=container_name, timestamp=timestamp, name='cpu', value=usage_pct) self.id_to_cpu[id] = {'total': total_usage, 'system': system_usage} except: # We don't want to kill the stat thread, and we don't really mind # if some statistics aren't saved properly pass # Get listing of registered containers filtered by topic def __get_registered_containers(self, topic): return resolve_query( Registration.select().where(Registration.topic == topic)) # Clear the pending queue of a specific topic def __clear_pending(self, topic, container_name): if topic is not None and topic in self.pending: if container_name is not None and container_name in self.pending[ topic]: del self.pending[topic][container_name] # Iterate through the self.pending queue and act on anything queued # that has passed its timer def __process_pending(self): while (True): for_process = [] for topic in self.pending: for container in self.pending[topic]: if self.__get_current_time_in_seconds( ) > self.pending[topic][container]: # It's go time! Pop into array so we can run after the loop # Otherwise modiying the dictionary during runtime is bad (item is # removed from pending immediately upon start of processing) for_process.append({ "topic": topic, "container": container }) # Iterate the array and run the things inside for process in for_process: self.__process(process["topic"], process["container"]) del for_process[:] try: sleep(TIMER_INTERVAL) except KeyboardInterrupt: self.pending_thread.stop() def __process(self, topic, container_name): self.__clear_pending( topic, container_name) # Clear the topic so this isn't hit multiple times # Promote all files in the inbox, delineated by topic, to a directory for # a container to mount; returns a list of created inboxes container_inboxes = self.inboxer.promote_to_container_inbox( topic, str(uuid4())) if container_inboxes is None or len(container_inboxes) == 0: print "There are no container inboxes available for the event run; did something weird happen?" return None for container_inbox in container_inboxes: # container_inbox is the path inboxer promoted to be mounted in the docker container self.docker_agent.pull(container_name) container_id = self.docker_agent.start_container( host_inbox=container_inbox, image_name=container_name, on_terminate=self.__clean_container, topic=topic) self.id_to_metadata[container_id] = { 'topic': topic, 'container': container_name, 'timestamp': int(time() * 1000) } self.docker_agent.stats(container_id, topic, container_name, self.__handle_stat) # Get the current time, in seconds, since epoch def __get_current_time_in_seconds(self): return timegm(gmtime()) # This event handler is executed every time a file is put into the master inbox # The data argument includes a property specify the topic that was appended to def __handle_inbox_received_event(self, data): topic = data["topic"] count = self.inboxer.get_inbox_count(topic) # Get a list of all files in the inbox delineated by the topic in the event inbox_list = self.inboxer.get_inbox_list(topic) # If there are no items in the inbox then there is literally nothing to do if count == 0: return None container_infos = self.__get_registered_containers(data["topic"]) # We have no container infos! :( if container_infos is None: print "Container for topic not found; doing nothing..." return None for container_info in container_infos: container = container_info["container"] # If the count has reached the threshold then it's time to execute! threshold = container_info["threshold"] if container_info[ "threshold"] is not None else 1 if count >= threshold: self.__process(topic, container) else: # We haven't hit the threshold so we need to start counting down # Set the current time if topic not in self.pending: self.pending[topic] = {} self.pending[topic][container] = ( self.__get_current_time_in_seconds() + container_info["timeout"])
configuration.bootstrap_app_config(app) # Setup authorization plugin token_secret = app.config.get('token_secret', 'password') auth_plugin = TokenAuthorizationPlugin(token_secret) app.install(auth_plugin) count = AtomicLong(0) inbox = Inboxer(atomic_counter=count, master_inbox_path=app.config.get('inbox.master', DEFAULT_MASTER_INBOX), container_inboxes_path=app.config.get( 'inbox.containers', DEFAULT_CONTAINER_INBOXES)) docker_agent = DockerAgent(registry_host=app.config.get("registry.host", None), username=app.config.get("registry.username", None), password=app.config.get("registry.password", None), docker_timeout=app.config.get("docker.timeout", 2)) orchestrator = Orchestrator(inboxer=inbox, docker_agent=docker_agent) # Accepts a multipart form # Field 'topic' -> The topic for the attached file(s) # Field * -> Any field name that is a UploadFile object @app.post('/event') def post_event(): topic = request.forms.get("topic") successfully_queued = [] # Bottle default way of accessing files doesn't seem to like multiple files # which use the same form name (which is standard practice). So let's # access them manually. Muwhahahahaha.
class Orchestrator(object): def __init__(self, inboxer=Inboxer(), docker_agent=None): self.inboxer = inboxer self.inboxer.on("received", self.__handle_inbox_received_event) self.docker_agent = docker_agent self.id_to_cpu = {} self.id_to_metadata = {} if self.docker_agent is None: self.docker_agent = DockerAgent() # Setup pending queue self.pending = {} self.pending_thread = Thread(target=self.__process_pending) self.pending_thread.daemon = True self.pending_thread.start() # Clean up the id_to_cpu dict def __clean_container(self, container_id): if container_id in self.id_to_metadata: meta = self.id_to_metadata[container_id] Event.create( topic=meta["topic"], container=meta["container"], timestamp=meta["timestamp"], duration=(int(time() * 1000) - meta["timestamp"]), ) del self.id_to_metadata[container_id] if container_id in self.id_to_cpu: del self.id_to_cpu[container_id] # Save stat to database def __handle_stat(self, stat, id, topic, container_name): try: read_dt = parser.parse(stat["read"]) timestamp = int((time.mktime(read_dt.timetuple()) + (read_dt.microsecond / 1000000.0)) * 1000) memory_usage = float(stat["memory_stats"]["usage"]) / float(stat["memory_stats"]["limit"]) Metric.create(topic=topic, container=container_name, timestamp=timestamp, name="memory", value=memory_usage) # Calculate CPU usage. The docker API returns the number of cycles consumed # by the container and the number of cycles consumed by the system. We need # to take the difference over time and divide them to retrieve the usage # percentage. total_usage = float(stat["cpu_stats"]["cpu_usage"]["total_usage"]) system_usage = float(stat["cpu_stats"]["system_cpu_usage"]) if id in self.id_to_cpu: usage_diff = total_usage - self.id_to_cpu[id]["total"] system_diff = system_usage - self.id_to_cpu[id]["system"] if usage_diff >= 0: usage_pct = usage_diff / system_diff else: usage_pct = 0.0 Metric.create(topic=topic, container=container_name, timestamp=timestamp, name="cpu", value=usage_pct) self.id_to_cpu[id] = {"total": total_usage, "system": system_usage} except: # We don't want to kill the stat thread, and we don't really mind # if some statistics aren't saved properly pass # Get listing of registered containers filtered by topic def __get_registered_containers(self, topic): return resolve_query(Registration.select().where(Registration.topic == topic)) # Clear the pending queue of a specific topic def __clear_pending(self, topic, container_name): if topic is not None and topic in self.pending: if container_name is not None and container_name in self.pending[topic]: del self.pending[topic][container_name] # Iterate through the self.pending queue and act on anything queued # that has passed its timer def __process_pending(self): while True: for_process = [] for topic in self.pending: for container in self.pending[topic]: if self.__get_current_time_in_seconds() > self.pending[topic][container]: # It's go time! Pop into array so we can run after the loop # Otherwise modiying the dictionary during runtime is bad (item is # removed from pending immediately upon start of processing) for_process.append({"topic": topic, "container": container}) # Iterate the array and run the things inside for process in for_process: self.__process(process["topic"], process["container"]) del for_process[:] try: sleep(TIMER_INTERVAL) except KeyboardInterrupt: self.pending_thread.stop() def __process(self, topic, container_name): self.__clear_pending(topic, container_name) # Clear the topic so this isn't hit multiple times # Promote all files in the inbox, delineated by topic, to a directory for # a container to mount; returns a list of created inboxes container_inboxes = self.inboxer.promote_to_container_inbox(topic, str(uuid4())) if container_inboxes is None or len(container_inboxes) == 0: print "There are no container inboxes available for the event run; did something weird happen?" return None for container_inbox in container_inboxes: # container_inbox is the path inboxer promoted to be mounted in the docker container self.docker_agent.pull(container_name) container_id = self.docker_agent.start_container( host_inbox=container_inbox, image_name=container_name, on_terminate=self.__clean_container, topic=topic ) self.id_to_metadata[container_id] = { "topic": topic, "container": container_name, "timestamp": int(time() * 1000), } self.docker_agent.stats(container_id, topic, container_name, self.__handle_stat) # Get the current time, in seconds, since epoch def __get_current_time_in_seconds(self): return timegm(gmtime()) # This event handler is executed every time a file is put into the master inbox # The data argument includes a property specify the topic that was appended to def __handle_inbox_received_event(self, data): topic = data["topic"] count = self.inboxer.get_inbox_count(topic) # Get a list of all files in the inbox delineated by the topic in the event inbox_list = self.inboxer.get_inbox_list(topic) # If there are no items in the inbox then there is literally nothing to do if count == 0: return None container_infos = self.__get_registered_containers(data["topic"]) # We have no container infos! :( if container_infos is None: print "Container for topic not found; doing nothing..." return None for container_info in container_infos: container = container_info["container"] # If the count has reached the threshold then it's time to execute! threshold = container_info["threshold"] if container_info["threshold"] is not None else 1 if count >= threshold: self.__process(topic, container) else: # We haven't hit the threshold so we need to start counting down # Set the current time if topic not in self.pending: self.pending[topic] = {} self.pending[topic][container] = self.__get_current_time_in_seconds() + container_info["timeout"]