def get_process_docs(self): for proc in self.input.get_process_docs(): process_id = get_process_id(proc) if process_id not in self.input_proc_guids: self.input_proc_guids.add(get_process_id(proc)) yield proc if self.traverse_tree: for tree_proc in self.traverse_up_down(proc): yield tree_proc
def output_process_doc(self, doc_content): proc_guid = get_process_id(doc_content) pathname = os.path.join(self.pathname, 'procs', get_process_path(proc_guid)) if os.path.exists(pathname): log.warning('process %s already existed, writing twice' % proc_guid) self.format_date_fields(doc_content) open(os.path.join(self.pathname, 'procs', get_process_path(proc_guid)), 'w').write(json_encode(doc_content)) self.written_docs['proc'] += 1
def output_process_doc(self, doc): for munger in self.mungers: doc = munger.munge_document('proc', doc) sys.stdout.write('%-70s\r' % ("Uploading process %s..." % get_process_id(doc))) sys.stdout.flush() self.output.output_process_doc(doc)
def traverse_down(self, guid): total = [] for proc in self.input.get_process_docs('parent_unique_id:%s' % (guid,)): process_id = get_process_id(proc) if process_id not in self.input_proc_guids: self.input_proc_guids.add(process_id) total.append(proc) total.extend(self.traverse_down(process_id)) return total
def traverse_up_down(self, proc): # TODO: infinite recursion prevention parent_process_id = get_parent_process_id(proc) process_id = get_process_id(proc) total = [] # get parents if parent_process_id: total.extend(self.traverse_up(parent_process_id)) total.extend(self.traverse_down(process_id)) for proc in total: yield proc
def traverse_up(self, guid): # TODO: this prompts a larger issue of - how do we handle process segments? total = [] for proc in self.input.get_process_docs('unique_id:%s' % (guid,)): process_id = get_process_id(proc) if process_id not in self.input_proc_guids: self.input_proc_guids.add(process_id) total.append(proc) parent_process_id = get_parent_process_id(proc) if parent_process_id and parent_process_id not in self.input_proc_guids: total.extend(self.traverse_up(parent_process_id)) return total
def output_process_doc(self, doc_content): # first, update the sensor_id in the process document to match the target settings if doc_content['sensor_id'] not in self.sensor_id_map: log.warning("Got process document %s without associated sensor data" % get_process_id(doc_content)) else: sensor_id = self.sensor_id_map[doc_content['sensor_id']] doc_content = deepcopy(doc_content) update_sensor_id_refs(doc_content, sensor_id) # fix up the last_update field last_update = doc_content.get("last_update", None) or self.now doc_content["last_update"] = {"set": last_update} doc_content.pop("last_server_update", None) self.output_doc("proc", doc_content)