def __init__(self,proto="tcp",host="*",port="5555"):
self.address = "%s://%s:%s" % (proto,host,port)
self.up = False
self.socket = None
#self.resource_map = {}
self.mapper = ObjectMapper()
self.return_data = True
self.router = Router(self)
path = tempfile.mkdtemp()
w0 = open("%s/part-r-00000" % path, 'w')
w0.write("\t".join(weights) + "\n")
w0.close()
#
# Create schema for weights, place under weight-0 dir
#
w0_schema = {
"fields": w0_fields,
"version": 0,
"sortKeys": [],
"sortKeyOrders": []
}
w0_schema_file = open("%s/.pig_schema" % path, 'w')
ObjectMapper().writeValue(w0_schema_file, w0_schema)
w0_schema_file.close()
#
# Copy initial weights to fs
#
copyFromLocal = "copyFromLocal %s %s/%s" % (path, data_dir, "weight-0")
Pig.fs(copyFromLocal)
#
# Iterate until converged
#
features = "%s/%s" % (data_dir, features)
script = Pig.compileFromFile(pig_script)
weight_queue = Queue.Queue(25) # for moving average
avg_weight = [0.0 for i in xrange(int(num_features))]
class Server(Resource):
# the server itself is a resource because
# it could have it's own methods
def __init__(self,proto="tcp",host="*",port="5555"):
self.address = "%s://%s:%s" % (proto,host,port)
self.up = False
self.socket = None
#self.resource_map = {}
self.mapper = ObjectMapper()
self.return_data = True
self.router = Router(self)
#self.gson = Gson()
def handle_request(self,request):
# you may want to add server methods someday
return self.not_found(request)
def shutdown(self):
self.stop()
#
# Resource-specific methods
#
def get_resource_map(self):
# TODO: traverse resources and build map
pass
def add_resource(self,name,resource):
self.router.add(name,resource)
def get_resource(self,name):
return self.router.get(name)
def start(self):
print "Starting Lightsocket..."
context = ZMQ.context(4) # number of IO threads to use
self.socket = context.socket(ZMQ.REP)
self.socket.bind(self.address)
self.up = True
self.receive_requests()
def receive_requests(self):
print "Ready to receive requests..."
while self.up:
raw = self.socket.recv(0)
# max req/sec with one python client: 10174 req/sec
# python client json serialization drops 10174 to 8153
# jython server json read drops it from 8153 to 6302
request = Request(self.mapper.readValue(raw.tostring(),HashMap))
# router drops is from 6302 to 4600 (this is the only thing to optimize)
resp = self.router.get(request)
# re-serialization drops it from 4600 to 4282 (you don't have to serialize)
self.send_response(resp)
#self.socket.send(str(dict()), 0)
# evidently pickle with zlib is not faster than json
#decomp = zlib.decompress(raw)
#data = cPickle.loads(decomp)
#request = Request(data)
def send_response(self,resp):
self.socket.send(resp.to_json(self.return_data), 0)
#self.socket.send("OK", 0)
def stop(self):
self.up = False
for resource in self.resource_map.values():
resource.shutdown()
