def __init__(self, key, value, user_id):
self.key = key
self.value = value
self.user_id = user_id
self.callback_count = 0
self.servicesModels = {"tw": TwitterModel(), "gh": GithubModel()}
self.cross_model = CrossModel()
setattr(self.cross_model, key, value)
super(MainWorker, self).__init__()
class MainWorker(multiprocessing.Process, ExchangeRpcWorker):
key = None
value = None
# services = {"bing": "BingCrawler", "github": "GithubCrawler", "twitter": "TwitterCrawler"}
model = None
def __init__(self, key, value, user_id):
self.key = key
self.value = value
self.user_id = user_id
self.callback_count = 0
self.servicesModels = {"tw": TwitterModel(), "gh": GithubModel()}
self.cross_model = CrossModel()
setattr(self.cross_model, key, value)
super(MainWorker, self).__init__()
def run(self):
self.set_models_from_db()
self.connect()
self.declare_queues()
count = 0
while True:
count += 1
if count > 10:
print " [+] Security case. Over 10 loops"
break
self.cross_model.changed = False
map(ServiceModel.reset_changed, self.servicesModels.itervalues())
self.callback_count = 0
self.serialize_body_msg()
self.publish()
self.wait_responses()
# self.set_results_in_instance() inside wait_response > on_emit_callback() >
print " [*] All services finished and results were mixed. Writing results in db..."
print " [x] Do we have new values to research about? %s " % (self.are_changes(),)
if self.are_changes():
print " [*] == LOOP ==\n\n"
else:
print " [*] == DONE ==\n\n"
break
self.set_results_in_db()
return
# Mixes existing values with coming ones.
def set_results_in_instance(self, body):
callback_result = self.unserialize_body_msg(body)
result_services_models = callback_result.get("services", None)
result_cross_model = callback_result.get("cross", None)
self.cross_model.mix_results(result_cross_model)
for key, value in self.servicesModels.iteritems():
service = result_services_models.get(key, None)
if service:
value.mix_results(service)
self.cross_model.populate_name()
def set_results_in_db(self):
maindb = MainDB()
maindb.set_user_status(user_id=self.user_id, status=200)
maindb.set_services_models(self.user_id, self.servicesModels)
maindb.set_user_model(self.user_id, self.cross_model)
maindb.close()
def check_stop_condition(self):
self.callback_count += 1
if self.callback_count >= len(servicesDict):
self.stop_consuming()
def serialize_body_msg(self):
json_cross = pickle.dumps(self.cross_model)
json_services = pickle.dumps(self.servicesModels)
return json.dumps(
{
"msg": "self.msg",
"exchange_str": self.callback_queue.method.queue,
"cross": json_cross,
"services": json_services,
}
)
def unserialize_body_msg(self, body):
json_obj = json.loads(body)
cross = pickle.loads(json_obj["cross"])
services = pickle.loads(json_obj["services"])
return {"cross": cross, "reply_queue": json_obj["reply_queue"], "services": services, "msg": json_obj["msg"]}
def log_publish(self):
print ' [*] Asking services to get info from "%s"' % (self.cross_model.email,)
print " [*] Waiting reply @%s" % (self.callback_queue.method.queue,)
def log_callback(self, body):
callback_result = self.unserialize_body_msg(body)
reply_queue = callback_result.get("reply_queue")
print " [+] %s replied. Mixing results..." % (reply_queue,)
def set_models_from_db(self):
self.cross_model.set_user_values(self.user_id)
for key, service in self.servicesModels.iteritems():
service.set_user_values(self.user_id)
def are_changes(self):
return self.cross_model.changed or (
True in list(map(ServiceModel.is_changed, self.servicesModels.itervalues()))
)
