Beispiel #1
0
 def __init__(self,proto="tcp",host="*",port="5555"):
     self.address = "%s://%s:%s" % (proto,host,port)
     self.up = False
     self.socket = None
     #self.resource_map = {}
     self.mapper = ObjectMapper()
     self.return_data = True     
     self.router = Router(self)
Beispiel #2
0
path = tempfile.mkdtemp()
w0 = open("%s/part-r-00000" % path, 'w')
w0.write("\t".join(weights) + "\n")
w0.close()

#
# Create schema for weights, place under weight-0 dir
#
w0_schema = {
    "fields": w0_fields,
    "version": 0,
    "sortKeys": [],
    "sortKeyOrders": []
}
w0_schema_file = open("%s/.pig_schema" % path, 'w')
ObjectMapper().writeValue(w0_schema_file, w0_schema)
w0_schema_file.close()

#
# Copy initial weights to fs
#
copyFromLocal = "copyFromLocal %s %s/%s" % (path, data_dir, "weight-0")
Pig.fs(copyFromLocal)

#
# Iterate until converged
#
features = "%s/%s" % (data_dir, features)
script = Pig.compileFromFile(pig_script)
weight_queue = Queue.Queue(25)  # for moving average
avg_weight = [0.0 for i in xrange(int(num_features))]
Beispiel #3
0
class Server(Resource):

    # the server itself is a resource because 
    # it could have it's own methods
    
    def __init__(self,proto="tcp",host="*",port="5555"):
        self.address = "%s://%s:%s" % (proto,host,port)
        self.up = False
        self.socket = None
        #self.resource_map = {}
        self.mapper = ObjectMapper()
        self.return_data = True     
        self.router = Router(self)
    
        #self.gson = Gson()

    def handle_request(self,request):
        # you may want to add server methods someday
        return self.not_found(request)

    def shutdown(self):
        self.stop()

    #
    # Resource-specific methods
    #

    def get_resource_map(self):
        # TODO: traverse resources and build map
        pass

    def add_resource(self,name,resource):
        self.router.add(name,resource)

    def get_resource(self,name):
        return self.router.get(name)
        
    def start(self):
        print "Starting Lightsocket..."
        context = ZMQ.context(4)  # number of IO threads to use
        self.socket = context.socket(ZMQ.REP)
        self.socket.bind(self.address)
        self.up = True
        self.receive_requests()

    def receive_requests(self):
        print "Ready to receive requests..."
        while self.up:
            raw = self.socket.recv(0)
            # max req/sec with one python client: 10174 req/sec
            # python client json serialization drops 10174 to 8153
            # jython server json read drops it from 8153 to 6302
            request = Request(self.mapper.readValue(raw.tostring(),HashMap))
            # router drops is from 6302 to 4600 (this is the only thing to optimize)
            resp = self.router.get(request)
            # re-serialization drops it from 4600 to 4282 (you don't have to serialize)
            self.send_response(resp)
            #self.socket.send(str(dict()), 0)        

            # evidently pickle with zlib is not faster than json
            #decomp = zlib.decompress(raw)
            #data = cPickle.loads(decomp)
            #request = Request(data)

    def send_response(self,resp):
        self.socket.send(resp.to_json(self.return_data), 0)
        #self.socket.send("OK", 0)
                    
    def stop(self):
        self.up = False
        for resource in self.resource_map.values():
            resource.shutdown()