def run(self):
self.setStatus(201)
if DEBUG:
if hasattr(self, "task_queue"):
print "NEW TASK QUEUE:"
print self.task_queue
task_path = ".".join([TASKS_ROOT, self.task_path])
p, f = task_path.rsplit(".", 1)
# start a websocket for the task
self.task_channel = UnveillanceTaskChannel("annex_channel", "localhost", API_PORT + 1, use_ssl=False)
try:
module = import_module(p)
func = getattr(module, f)
# TODO: for cellery:
# args = [(self,), ({'queue' :self.queue})]
args = [self]
if DEBUG:
print args
# p = Process(target=func.apply_async, args=args)
p = Process(target=func, args=args)
self.communicate()
sleep(1)
p.start()
except Exception as e:
printAsLog(e)
self.fail()
def initElasticsearch(self):
if DEBUG: print "INITING ELASTICSEARCH"
from vars import ELASTICSEARCH_MAPPINGS
index = { "mappings": ELASTICSEARCH_MAPPINGS }
try:
res = self.sendELSRequest(method="delete")
if DEBUG:
print "DELETED OLD MAPPING:"
print res
except Exception as e:
if DEBUG: print e
printAsLog(e, as_error=True)
try:
res = self.sendELSRequest(data=index, method="put")
if DEBUG:
print "INITIALIZED NEW MAPPING:"
print res
if not res['acknowledged']: return False
except Exception as e:
printAsLog(e, as_error=True)
return False
def stopServer(self):
printAsLog("stopping NLP server")
stopDaemon(self.pid_file, extra_pids_port=getConfig('nlp_server.port'))
try:
del self.nlp_server
except Exception as e:
print "error stopping NLP server\n%s" % e
def tokenize(self, texts):
if type(texts) is not list: texts = [texts]
tokenized = []
for text in texts:
printAsLog("Attempting to tokenize:\n%s..." % text[:135])
try:
parse = self.nlp_server.parse(text)
print type(parse)
tokenized.append(json.loads(parse))
except Exception as e:
if DEBUG: print e
continue
if len(tokenized) > 0: return tokenized
return None
def stopElasticsearch(self):
printAsLog("stopping elasticsearch")
self.stopCronJobs()
p = Popen(['lsof', '-t', '-i:9200'], stdout=PIPE, close_fds=True)
data = p.stdout.readline()
while data:
p_ = Popen(['kill', data.strip()])
p_.wait()
data = p.stdout.readline()
p.stdout.close()
stopDaemon(self.els_pid_file)
with open(self.els_status_file, 'wb+') as f: f.write("False")
def register_upload_attempt(_id):
from Utils.funcs import printAsLog
from lib.Worker.Models.uv_document import UnveillanceDocument
try:
doc = UnveillanceDocument(_id=_id)
if doc.getFileMetadata('upload_attempts') == None:
upload_attempts = 1
else:
upload_attempts += 1
doc.set_file_metadata('upload_attempts', upload_attempts)
except Exception as e:
printAsLog(e, as_error=True)
return False
return True
def run_task(task_id):
from Utils.funcs import printAsLog
from lib.Worker.Models.uv_task import UnveillanceTask
try:
task = UnveillanceTask(_id=task_id)
except Exception as e:
printAsLog("no task id. quitting", as_error=True)
return False
import requests
from conf import HOST, API_PORT
try:
r = requests.post("http://%s:%d/task/" % (HOST, API_PORT),
data={ '_id' : task._id })
except Exception as e:
printAsLog(e, as_error=True)
return False
return True
def startWorker(self):
printAsLog("STARTING CELERY WORKER!")
from lib.Worker.vars import TASKS_ROOT, buildCeleryTaskList, ALL_WORKERS
self.celery_tasks = buildCeleryTaskList()
sys.argv.extend(['worker', '-l', 'info', '-Q', ",".join([ALL_WORKERS, UUID])])
self.celery_app = Celery(TASKS_ROOT,
broker='amqp://guest@localhost//', include=self.celery_tasks)
startDaemon(self.worker_log_file, self.worker_pid_file)
logging.getLogger().setLevel(logging.DEBUG)
self.task_channel = sockjs.tornado.SockJSRouter(TaskChannel, '/annex_channel')
tc = tornado.web.Application(
[(r'/info', TaskChannel.InfoHandler)] + self.task_channel.urls)
tc.listen(TASK_CHANNEL_PORT, no_keep_alive=True)
if DEBUG:
print "TaskChannel started on port %d" % TASK_CHANNEL_PORT
tornado.ioloop.IOLoop.instance().start()
def sync_file(file_name, with_metadata=None):
import requests
from Utils.funcs import printAsLog
from conf import HOST, API_PORT, DEBUG
if with_metadata is not None:
if DEBUG:
print "FIRST METADATA:"
print with_metadata
if type(with_metadata) is list and len(with_metadata) > 0:
with_metadata = dict(tuple([d.replace("--","") for d in m.split("=")]) for m in with_metadata if m[:2] == "--")
if DEBUG:
print "TRANSFORMED METADATA:"
print with_metadata
try:
r = requests.post("http://%s:%d/sync/%s" % (HOST, API_PORT, file_name), data=with_metadata)
return (r.status_code == 200)
except Exception as e:
printAsLog(e, as_error=True)
return False
def stopWorker(self):
printAsLog("WORKER EXHAUSTED. FINISHING!")
stopDaemon(self.worker_pid_file)
