Пример #1
0
    def GetConfig(self):
        # get configuration from the host
        sconf = None
        if self.configfile:
            try:
                f = open(self.configfile)
                sconf = f.read()
                f.close()
            except:
                pass
        elif self.host:
            sconf = client.config(self.host)

        if not sconf:
            print "*** no configuration"
            return None

        self.config = simplejson.loads(sconf)

        self.var = self.config.get("var")
        self.route = self.config.get("route")
        self.tasks = self.config.get("tasks")

        dbunch = self.config.get("bunch")
        hostlist = self.config.get("hosts")

        if dbunch:
            dinfo = dbunch.get(self.name)
            if dinfo:
                self.range = int(dinfo.get("range"))

        if self.route:
            for key, routes in self.route.iteritems():
                dest = routes.get(self.name)
                if dest:
                    if not self.target:
                        self.target = {}
                    self.target[key] = dest

        if (self.var == None or self.route == None or hostlist == None
                or self.tasks == None or self.range == None
                or self.target == None):
            return None

        self.hosts = {}
        for h in hostlist:
            self.hosts[h["id"]] = h["host"] + ":" + h["port"]

        self.flog.write(str(self.config))
        self.flog.write("\n")
        self.flog.write("Got configuration size %d\n" %
                        (len(str(self.config))))
        self.flog.flush()

        return self.config
Пример #2
0
    def GetConfig(self):
        # get configuration from the host
        sconf = None
        if self.configfile:
            try:
                f = open(self.configfile)
                sconf = f.read()
                f.close()
            except:
                pass
        elif self.host:
            sconf = client.config(self.host)

        if not sconf:
            print "*** no configuration"
            return None

        self.config = simplejson.loads(sconf)

        self.var = self.config.get("var")
        self.route = self.config.get("route")
        self.tasks = self.config.get("tasks")

        dbunch = self.config.get("bunch")
        hostlist = self.config.get("hosts")

        if dbunch:
            dinfo = dbunch.get(self.name)
            if dinfo:
                self.range = int(dinfo.get("range"))

        if self.route:
            for key, routes in self.route.iteritems():
                dest = routes.get(self.name)
                if dest:
                    if not self.target:
                        self.target = {}
                    self.target[key] = dest

        if (self.var   == None  or  self.route  == None  or
            hostlist   == None  or  self.tasks  == None  or
            self.range == None  or  self.target == None):
            return None

        self.hosts = {}
        for h in hostlist:
            self.hosts[h["id"]] = h["host"] + ":" + h["port"]

        self.flog.write(str(self.config))
        self.flog.write("\n")
        self.flog.write("Got configuration size %d\n" % (len(str(self.config))))
        self.flog.flush()

        return self.config
Пример #3
0
    server.running = True
    
    # no locking required...
    server.superstep_count = 0

    handler = BaseHTTPServer.BaseHTTPRequestHandler
    handler.port = port
    handler.id = id
    handler.pid = os.getpid()
    handler.workdir = workdir
    handler.master = master

    if master != None:
        SYS_STATS = True
        # get configuration from master
        sconf = client.config(master)
    
        dconf = json.loads(sconf)
        handler.config = dconf
    
        logging.debug("Config size: %d" % (len(sconf)))
        logging.debug(str(dconf))
            
        # send done to master
        client.done(master, id)

        # NOTE: Time conflict. The master might already send 'step' request,
        #       before server_forever() is started, so 'step' might be lost.
        #       Delay done until the server is up and running.
        # Check out Asynchronous Mixins example for SocketServer
        # Comment: the constructor might already activate the server,
Пример #4
0
    server.running = True
    
    # no locking required...
    server.superstep_count = 0

    handler = BaseHTTPServer.BaseHTTPRequestHandler
    handler.port = port
    handler.id = id
    handler.pid = os.getpid()
    handler.workdir = workdir
    handler.master = master

    if master != None:
        # SYS_STATS = True
        # get configuration from master
        sconf = client.config(master)
    
        dconf = json.loads(sconf)
        handler.config = dconf
    
        logging.debug("Config size: %d" % (len(sconf)))
        logging.debug(str(dconf))
            
        # send done to master
        client.done(master, id)

        # NOTE: Time conflict. The master might already send 'step' request,
        #       before server_forever() is started, so 'step' might be lost.
        #       Delay done until the server is up and running.
        # Check out Asynchronous Mixins example for SocketServer
        # Comment: the constructor might already activate the server,
Пример #5
0
    def GetConfig(self):
        # get configuration from the host
        sconf = None
        if self.configfile:
            try:
                f = open(self.configfile)
                sconf = f.read()
                f.close()
            except:
                pass
        elif self.host:
            sconf = client.config(self.host)

        if not sconf:
            print "*** no configuration"
            return None

        self.config = json.loads(sconf)

        self.var = self.config.get("var")
        self.route = self.config.get("route")
        self.tasks = self.config.get("tasks")

        dbunch = self.config.get("bunch")
        hostlist = self.config.get("hosts")

        if dbunch:
            dinfo = dbunch.get(self.name)
            if dinfo:
                self.range = int(dinfo.get("range"))
                seg_bits = self.var.get('seg_bits')
                if seg_bits is not None:
                    seg_bits = int(seg_bits)
                    if seg_bits <= 0:
                        print 'Error! for node-segmented BFS, the number of segment bits must be positive!'
                        sys.exit(1)
                    if (1<<seg_bits) % self.range != 0:
                        print 'Error! for node-segmented BFS, the segment size must be a multiple of the range!'
                        sys.exit(1)

        if self.route:
            for key, routes in self.route.iteritems():
                dest = routes.get(self.name)
                if dest:
                    if not self.target:
                        self.target = {}
                    self.target[key] = dest

        if (self.var   == None  or  self.route  == None  or
            hostlist   == None  or  self.tasks  == None  or
            self.range == None  or  self.target == None):
            return None

        self.hosts = {}
        for h in hostlist:
            self.hosts[h["id"]] = h["host"] + ":" + h["port"]

        self.log.debug("Configuration: %s" % str(self.config))
        self.log.debug("config size %d" % (len(str(self.config))))

        return self.config
Пример #6
0
    def GetConfig(self):
        # get configuration from the host
        sconf = None
        if self.configfile:
            try:
                f = open(self.configfile)
                sconf = f.read()
                f.close()
            except:
                pass
        elif self.host:
            sconf = client.config(self.host)

        if not sconf:
            print "*** no configuration"
            return None

        self.config = json.loads(sconf)

        self.var = self.config.get("var")
        self.route = self.config.get("route")
        self.tasks = self.config.get("tasks")

        dbunch = self.config.get("bunch")
        hostlist = self.config.get("hosts")

        if dbunch:
            dinfo = dbunch.get(self.name)
            if dinfo:
                self.range = int(dinfo.get("range"))
                seg_bits = self.var.get('seg_bits')
                if seg_bits is not None:
                    seg_bits = int(seg_bits)
                    if seg_bits <= 0:
                        print 'Error! for node-segmented BFS, the number of segment bits must be positive!'
                        sys.exit(1)
                    if (1 << seg_bits) % self.range != 0:
                        print 'Error! for node-segmented BFS, the segment size must be a multiple of the range!'
                        sys.exit(1)

        if self.route:
            for key, routes in self.route.iteritems():
                dest = routes.get(self.name)
                if dest:
                    if not self.target:
                        self.target = {}
                    self.target[key] = dest

        if (self.var == None or self.route == None or hostlist == None
                or self.tasks == None or self.range == None
                or self.target == None):
            return None

        self.hosts = {}
        for h in hostlist:
            self.hosts[h["id"]] = h["host"] + ":" + h["port"]

        self.log.debug("Configuration: %s" % str(self.config))
        self.log.debug("config size %d" % (len(str(self.config))))

        return self.config
Пример #7
0
    def do_GET(self):
        parsed_path = urlparse.urlparse(self.path)
        message_parts = [
                'CLIENT VALUES:',
                'client_address=%s (%s)' % (self.client_address,
                                            self.address_string()),
                'command=%s' % self.command,
                'path=%s' % self.path,
                'real path=%s' % parsed_path.path,
                'query=%s' % parsed_path.query,
                'request_version=%s' % self.request_version,
                '',
                'SERVER VALUES:',
                'server_type=%s' % "host server",
                'server_version=%s' % self.server_version,
                'sys_version=%s' % self.sys_version,
                'protocol_version=%s' % self.protocol_version,
                '',
                'HEADERS RECEIVED:',
                ]
        for name, value in sorted(self.headers.items()):
            message_parts.append('%s=%s' % (name, value.rstrip()))
        message_parts.append('')
        message = '\r\n'.join(message_parts)
        subpath = self.path.split("/")

        if self.path == "/prepare":
            # move qin to qact
            qinname  = "snapw.%d/qin" % (self.pid)
            qactname = "snapw.%d/qact" % (self.pid)

            # rename an existing qact
            qactnewname = "none"
            if os.path.exists(qactname):
                t = time.time()
                s = time.strftime("%Y%m%d-%H%M%S", time.localtime(t))
                mus = "%06d" % (t*1000000 - int(t)*1000000)
                qactnewname = "%s-%s-%s" % (qactname, s, mus)
                os.rename(qactname, qactnewname)

            # get the number of active tasks, rename existing qin
            numtasks = 0
            if os.path.exists(qinname):
                os.rename(qinname, qactname)
                active = os.listdir(qactname)
                numtasks = len(active)

            # create new qin
            config.mkdir_p(qinname)
    
            line = "preparing next step: %s, %s, %s\n" % (
                        qinname, qactname, qactnewname)
            self.flog.write(line)
            self.flog.flush()

	    # initially just get the whole config all over again -- Extremely inefficient
            sconf = client.config(master)
            dconf = simplejson.loads(sconf)
            handler.config = dconf
            flog.write("Got configuration size %d\n" % (len(sconf)))
            flog.write(str(dconf))
            flog.write("\n")
            flog.flush()

            # send ready to master
            client.ready(self.master, self.id, numtasks)

            self.send_response(200)
            self.send_header('Content-Length', 0)
            self.end_headers()
            return

        elif self.path == "/quit":
            self.send_response(200)
            self.send_header('Content-Length', 0)
            self.end_headers()

            # set the flag to terminate the server
            self.server.running = False
            self.server.self_dummy()
            return

        elif self.path == "/dummy":
            print "dummy request"

            self.send_response(200)
            self.send_header('Content-Length', 0)
            self.end_headers()
            return

        elif self.path == "/step":
            line = "execute next step\n"
            self.flog.write(line)
            self.flog.flush()

            self.send_response(200)
            self.send_header('Content-Length', 0)
            self.end_headers()

            # TODO, implement null action,
            #   skip execution if there are no tasks to execute,
            #   qact does not exist

            # get the tasks to execute

            qactname = "snapw.%d/qact" % (self.pid)
            active = []
            if os.path.exists(qactname):
                active = os.listdir(qactname)

            line = "active tasks %s\n" % (str(active))
            self.flog.write(line)
            self.flog.flush()

            self.qactname = qactname
            self.active = active
            # start a thread to execute the work tasks
            t = threading.Thread(target=Execute, args=(self, ))
            t.start()
            return

        elif self.path == "/config":
    
            line = "get configuration\n"
            self.flog.write(line)
            self.flog.flush()

            body = simplejson.dumps(self.config)
            self.send_response(200)
            self.send_header('Content-Length', len(body))
            self.end_headers()
            self.wfile.write(body)

            return

        elif self.path == "/quit":
    
            line = "terminate execution\n"
            self.flog.write(line)
            self.flog.flush()

            self.send_response(200)
            self.send_header('Content-Length', 0)
            self.end_headers()
            sys.exit(0)

        self.send_response(200)
        #self.send_header('Last-Modified', self.date_time_string(time.time()))
        self.end_headers()
        self.wfile.write(message)
        return