def __init__(self, key, value, user_id): self.key = key self.value = value self.user_id = user_id self.callback_count = 0 self.servicesModels = {"tw": TwitterModel(), "gh": GithubModel()} self.cross_model = CrossModel() setattr(self.cross_model, key, value) super(MainWorker, self).__init__()
class MainWorker(multiprocessing.Process, ExchangeRpcWorker): key = None value = None # services = {"bing": "BingCrawler", "github": "GithubCrawler", "twitter": "TwitterCrawler"} model = None def __init__(self, key, value, user_id): self.key = key self.value = value self.user_id = user_id self.callback_count = 0 self.servicesModels = {"tw": TwitterModel(), "gh": GithubModel()} self.cross_model = CrossModel() setattr(self.cross_model, key, value) super(MainWorker, self).__init__() def run(self): self.set_models_from_db() self.connect() self.declare_queues() count = 0 while True: count += 1 if count > 10: print " [+] Security case. Over 10 loops" break self.cross_model.changed = False map(ServiceModel.reset_changed, self.servicesModels.itervalues()) self.callback_count = 0 self.serialize_body_msg() self.publish() self.wait_responses() # self.set_results_in_instance() inside wait_response > on_emit_callback() > print " [*] All services finished and results were mixed. Writing results in db..." print " [x] Do we have new values to research about? %s " % (self.are_changes(),) if self.are_changes(): print " [*] == LOOP ==\n\n" else: print " [*] == DONE ==\n\n" break self.set_results_in_db() return # Mixes existing values with coming ones. def set_results_in_instance(self, body): callback_result = self.unserialize_body_msg(body) result_services_models = callback_result.get("services", None) result_cross_model = callback_result.get("cross", None) self.cross_model.mix_results(result_cross_model) for key, value in self.servicesModels.iteritems(): service = result_services_models.get(key, None) if service: value.mix_results(service) self.cross_model.populate_name() def set_results_in_db(self): maindb = MainDB() maindb.set_user_status(user_id=self.user_id, status=200) maindb.set_services_models(self.user_id, self.servicesModels) maindb.set_user_model(self.user_id, self.cross_model) maindb.close() def check_stop_condition(self): self.callback_count += 1 if self.callback_count >= len(servicesDict): self.stop_consuming() def serialize_body_msg(self): json_cross = pickle.dumps(self.cross_model) json_services = pickle.dumps(self.servicesModels) return json.dumps( { "msg": "self.msg", "exchange_str": self.callback_queue.method.queue, "cross": json_cross, "services": json_services, } ) def unserialize_body_msg(self, body): json_obj = json.loads(body) cross = pickle.loads(json_obj["cross"]) services = pickle.loads(json_obj["services"]) return {"cross": cross, "reply_queue": json_obj["reply_queue"], "services": services, "msg": json_obj["msg"]} def log_publish(self): print ' [*] Asking services to get info from "%s"' % (self.cross_model.email,) print " [*] Waiting reply @%s" % (self.callback_queue.method.queue,) def log_callback(self, body): callback_result = self.unserialize_body_msg(body) reply_queue = callback_result.get("reply_queue") print " [+] %s replied. Mixing results..." % (reply_queue,) def set_models_from_db(self): self.cross_model.set_user_values(self.user_id) for key, service in self.servicesModels.iteritems(): service.set_user_values(self.user_id) def are_changes(self): return self.cross_model.changed or ( True in list(map(ServiceModel.is_changed, self.servicesModels.itervalues())) )