def loop_Validator(self):
with open("./pids/validator.pid", "w") as fobj:
fobj.write(str(os.getpid()))
workers = Master()
redis_connector = RedisWrapper()
worker = ValidatorService(redis_connector, workers, validate_file)
worker.start_loop()
def loop_Preprocessor(self):
with open("./pids/preprocessing.pid", "w") as fobj:
fobj.write(str(os.getpid()))
workers = Master()
redis_connector = RedisWrapper()
worker = PeprocessService(redis_connector, workers, preprocesse_file)
worker.start_loop()
def main():
app = Application()
app.redis = RedisWrapper()
http_server = tornado.httpserver.HTTPServer()
http_server.listen(options.port)
loop_event = tornado.ioloop.IOLoop.instance()
loop_event.start()
def loop_SessionGenerator(self):
with open("./pids/sessions.pid", "w") as fobj:
fobj.write(str(os.getpid()))
workers = Master()
redis_connector = RedisWrapper()
worker = SessionGeneratorService(redis_connector, workers,
SessionGeneratorWorker().generate)
worker.start_loop()
def loop_Apriori(self):
with open("./pids/apriory.pid", "w") as fobj:
fobj.write(str(os.getpid()))
workers = Master()
redis_connector = RedisWrapper()
worker = AprioriService(redis_connector, workers, None)
worker.start_loop()
print("VALIDATOR RESULT {0}".format(result))
if not result:
self.redis.set_value(task.split("/")[-1], 'file is not valid')
self.redis.set_value(task, 'valid_file')
for channel in REDIS_CHANNELS:
print("VALIDATOR public {}".format(channel))
print(channel)
status = self.redis.publish_message("./validate/" + task.split("/")[-1], channel)
print("VALIDATOR public status {}".format(status))
print('complete {0}'.format(task))
with open("./validate/" + task.split("/")[-1], "w") as fobj:
pass
copyfile(task, "./validate/" + task.split("/")[-1])
def start_loop(self):
# self.app._shutting_down = False
self.ioloop.add_callback(self.loop)
self.ioloop.start()
def stop_loop(self, shutdown=False):
self.ioloop.stop()
if __name__ == '__main__':
with open("./pids/validator.pid", "w") as fobj:
fobj.write(str(os.getpid()))
workers = Master()
redis_connctor = RedisWrapper()
worker = Validator(redis_connctor, workers, validate_file)
worker.start_loop()
print(data)
task = data.get('data').decode('utf8')
result = list(self.workers.do([task], self.method))[0]
if not result:
self.redis.set_value(task, 'preprocesse_file problem')
print("preprocesse_file problem {} ".format(task))
self.redis.set_value(result, 'preprocessfile_file')
for channel in REDIS_CHANNELS:
print("Preprocessing public {}".format(channel))
print(channel)
self.redis.publish_message(result, channel)
print('complete {0}'.format(result))
def start_loop(self):
# self.app._shutting_down = False
self.ioloop.add_callback(self.loop)
self.ioloop.start()
def stop_loop(self, shutdown=False):
self.ioloop.stop()
if __name__ == '__main__':
with open("./pids/preprocessing.pid", "w") as fobj:
fobj.write(str(os.getpid()))
workers = Master()
redis_connetor = RedisWrapper()
worker = PeprocessService(redis_connetor, workers, preprocesse_file)
worker.start_loop()