def __init__(self,proto="tcp",host="*",port="5555"): self.address = "%s://%s:%s" % (proto,host,port) self.up = False self.socket = None #self.resource_map = {} self.mapper = ObjectMapper() self.return_data = True self.router = Router(self)
path = tempfile.mkdtemp() w0 = open("%s/part-r-00000" % path, 'w') w0.write("\t".join(weights) + "\n") w0.close() # # Create schema for weights, place under weight-0 dir # w0_schema = { "fields": w0_fields, "version": 0, "sortKeys": [], "sortKeyOrders": [] } w0_schema_file = open("%s/.pig_schema" % path, 'w') ObjectMapper().writeValue(w0_schema_file, w0_schema) w0_schema_file.close() # # Copy initial weights to fs # copyFromLocal = "copyFromLocal %s %s/%s" % (path, data_dir, "weight-0") Pig.fs(copyFromLocal) # # Iterate until converged # features = "%s/%s" % (data_dir, features) script = Pig.compileFromFile(pig_script) weight_queue = Queue.Queue(25) # for moving average avg_weight = [0.0 for i in xrange(int(num_features))]
class Server(Resource): # the server itself is a resource because # it could have it's own methods def __init__(self,proto="tcp",host="*",port="5555"): self.address = "%s://%s:%s" % (proto,host,port) self.up = False self.socket = None #self.resource_map = {} self.mapper = ObjectMapper() self.return_data = True self.router = Router(self) #self.gson = Gson() def handle_request(self,request): # you may want to add server methods someday return self.not_found(request) def shutdown(self): self.stop() # # Resource-specific methods # def get_resource_map(self): # TODO: traverse resources and build map pass def add_resource(self,name,resource): self.router.add(name,resource) def get_resource(self,name): return self.router.get(name) def start(self): print "Starting Lightsocket..." context = ZMQ.context(4) # number of IO threads to use self.socket = context.socket(ZMQ.REP) self.socket.bind(self.address) self.up = True self.receive_requests() def receive_requests(self): print "Ready to receive requests..." while self.up: raw = self.socket.recv(0) # max req/sec with one python client: 10174 req/sec # python client json serialization drops 10174 to 8153 # jython server json read drops it from 8153 to 6302 request = Request(self.mapper.readValue(raw.tostring(),HashMap)) # router drops is from 6302 to 4600 (this is the only thing to optimize) resp = self.router.get(request) # re-serialization drops it from 4600 to 4282 (you don't have to serialize) self.send_response(resp) #self.socket.send(str(dict()), 0) # evidently pickle with zlib is not faster than json #decomp = zlib.decompress(raw) #data = cPickle.loads(decomp) #request = Request(data) def send_response(self,resp): self.socket.send(resp.to_json(self.return_data), 0) #self.socket.send("OK", 0) def stop(self): self.up = False for resource in self.resource_map.values(): resource.shutdown()