def enterWorkLoop(functions):
from MongoStampedAPI import globalMongoStampedAPI
api = globalMongoStampedAPI()
logs.info("starting worker for %s" % functions.keys())
worker = StampedWorker(getHosts())
def wrapper(worker, job):
try:
k = job.task
logs.begin(saveLog=api._logsDB.saveLog, saveStat=api._statsDB.addStat, nodeName=api.node_name)
logs.async_request(k)
v = functions[k]
data = pickle.loads(job.data)
logs.info("%s: %s: %s" % (k, v, data))
v(k, data)
except Exception as e:
logs.error(str(e))
finally:
try:
logs.save()
except Exception:
print "Unable to save logs"
import traceback
traceback.print_exc()
logs.warning(traceback.format_exc())
return ""
for k, v in functions.items():
worker.register_task(k, wrapper)
worker.work(poll_timeout=1)
def apiTasks():
from MongoStampedAPI import globalMongoStampedAPI
api = globalMongoStampedAPI()
m = {}
def customizeStampAsyncHelper(key, data):
api.customizeStampAsync()
m[api.taskName(api.customizeStampAsync)] = customizeStampAsyncHelper
return m
def __init__(self):
self._db = S3ImageDB()
self._collages = {
'default' : DefaultImageCollage(),
'place' : PlaceImageCollage(),
'music' : MusicImageCollage(),
'book' : BookImageCollage(),
'film' : FilmImageCollage(),
'app' : AppImageCollage(),
}
self._categories = self._collages.keys()
self.api = globalMongoStampedAPI()
def enrichTasks():
from MongoStampedAPI import globalMongoStampedAPI
api = globalMongoStampedAPI()
m = {}
def mergeEntityAsyncHelper(key, data):
api.mergeEntityAsync(data["entityDict"])
m[api.taskName(api.mergeEntityAsync)] = mergeEntityAsyncHelper
def mergeEntityIdAsyncHelper(key, data):
api.mergeEntityIdAsync(data["entity_id"])
m[api.taskName(api.mergeEntityIdAsync)] = mergeEntityIdAsyncHelper
return m
def _api():
from MongoStampedAPI import globalMongoStampedAPI
return globalMongoStampedAPI()
def enrichHandler():
from MongoStampedAPI import globalMongoStampedAPI
api = globalMongoStampedAPI()
return loggingHandler(attributeHandler(api))