def get_process_docs(self):
for proc in self.input.get_process_docs():
process_id = get_process_id(proc)
if process_id not in self.input_proc_guids:
self.input_proc_guids.add(get_process_id(proc))
yield proc
if self.traverse_tree:
for tree_proc in self.traverse_up_down(proc):
yield tree_proc
def output_process_doc(self, doc_content):
proc_guid = get_process_id(doc_content)
pathname = os.path.join(self.pathname, 'procs', get_process_path(proc_guid))
if os.path.exists(pathname):
log.warning('process %s already existed, writing twice' % proc_guid)
self.format_date_fields(doc_content)
open(os.path.join(self.pathname, 'procs', get_process_path(proc_guid)), 'w').write(json_encode(doc_content))
self.written_docs['proc'] += 1
def output_process_doc(self, doc):
for munger in self.mungers:
doc = munger.munge_document('proc', doc)
sys.stdout.write('%-70s\r' % ("Uploading process %s..." % get_process_id(doc)))
sys.stdout.flush()
self.output.output_process_doc(doc)
def traverse_down(self, guid):
total = []
for proc in self.input.get_process_docs('parent_unique_id:%s' % (guid,)):
process_id = get_process_id(proc)
if process_id not in self.input_proc_guids:
self.input_proc_guids.add(process_id)
total.append(proc)
total.extend(self.traverse_down(process_id))
return total
def traverse_up_down(self, proc):
# TODO: infinite recursion prevention
parent_process_id = get_parent_process_id(proc)
process_id = get_process_id(proc)
total = []
# get parents
if parent_process_id:
total.extend(self.traverse_up(parent_process_id))
total.extend(self.traverse_down(process_id))
for proc in total:
yield proc
def traverse_up(self, guid):
# TODO: this prompts a larger issue of - how do we handle process segments?
total = []
for proc in self.input.get_process_docs('unique_id:%s' % (guid,)):
process_id = get_process_id(proc)
if process_id not in self.input_proc_guids:
self.input_proc_guids.add(process_id)
total.append(proc)
parent_process_id = get_parent_process_id(proc)
if parent_process_id and parent_process_id not in self.input_proc_guids:
total.extend(self.traverse_up(parent_process_id))
return total
def output_process_doc(self, doc_content):
# first, update the sensor_id in the process document to match the target settings
if doc_content['sensor_id'] not in self.sensor_id_map:
log.warning("Got process document %s without associated sensor data" % get_process_id(doc_content))
else:
sensor_id = self.sensor_id_map[doc_content['sensor_id']]
doc_content = deepcopy(doc_content)
update_sensor_id_refs(doc_content, sensor_id)
# fix up the last_update field
last_update = doc_content.get("last_update", None) or self.now
doc_content["last_update"] = {"set": last_update}
doc_content.pop("last_server_update", None)
self.output_doc("proc", doc_content)
