Example #1
0
class Running(object):
    def __init__(self,
                 scheduler,
                 name,
                 user='',
                 master=os.getenv('MESOS_MASTER'),
                 implicit_acknowledge=1,
                 *args,
                 **kwargs):
        scheduler = SchedulerProxy(scheduler)
        framework = FrameworkInfo(name=name, user=user, *args, **kwargs)
        self.driver = MesosSchedulerDriver(scheduler, encode(framework),
                                           master, implicit_acknowledge)

        def shutdown(signal, frame):
            self.driver.stop()

        signal.signal(signal.SIGINT, shutdown)
        signal.signal(signal.SIGTERM, shutdown)
        atexit.register(self.driver.stop)

    def run(self):
        return self.driver.run()

    def start(self):
        status = self.driver.start()
        assert status == mesos_pb2.DRIVER_RUNNING
        return status

    def stop(self):
        logging.info("Stopping Mesos driver")
        self.driver.stop()
        logging.info("Joining Mesos driver")
        result = self.driver.join()
        logging.info("Joined Mesos driver")
        if result != mesos_pb2.DRIVER_STOPPED:
            raise RuntimeError("Mesos driver failed with %i", result)

    def join(self):
        return self.driver.join()

    def __enter__(self):
        self.start()
        return self

    def __exit__(self, type, value, traceback):
        self.stop()
Example #2
0
class Running(object):

    def __init__(self, scheduler, name, user='', master=os.getenv('MESOS_MASTER'),
                 implicit_acknowledge=1, *args, **kwargs):
        scheduler = SchedulerProxy(scheduler)
        framework = FrameworkInfo(name=name, user=user, *args, **kwargs)
        self.driver = MesosSchedulerDriver(scheduler, encode(framework),
                                           master, implicit_acknowledge)

        def shutdown(signal, frame):
            self.driver.stop()
        signal.signal(signal.SIGINT, shutdown)
        signal.signal(signal.SIGTERM, shutdown)
        atexit.register(self.driver.stop)

    def run(self):
        return self.driver.run()

    def start(self):
        status = self.driver.start()
        assert status == mesos_pb2.DRIVER_RUNNING
        return status

    def stop(self):
        logging.info("Stopping Mesos driver")
        self.driver.stop()
        logging.info("Joining Mesos driver")
        result = self.driver.join()
        logging.info("Joined Mesos driver")
        if result != mesos_pb2.DRIVER_STOPPED:
            raise RuntimeError("Mesos driver failed with %i", result)

    def join(self):
        return self.driver.join()

    def __enter__(self):
        self.start()
        return self

    def __exit__(self, type, value, traceback):
        self.stop()
Example #3
0
class Running(object):

    def __init__(self, scheduler, name, user='', master=os.getenv('MESOS_MASTER'),
                 implicit_acknowledge=1, *args, **kwargs):
        scheduler = SchedulerProxy(scheduler)
        framework = FrameworkInfo(name=name, user=user, *args, **kwargs)
        self.driver = MesosSchedulerDriver(scheduler, encode(framework),
                                           master, implicit_acknowledge)

        def shutdown(signal, frame):
            self.stop()

        signal.signal(signal.SIGINT, shutdown)
        signal.signal(signal.SIGTERM, shutdown)
        atexit.register(self.stop)

    def run(self):
        return self.driver.run()

    def start(self):
        status = self.driver.start()
        assert status == mesos_pb2.DRIVER_RUNNING
        return status

    def stop(self):
        return self.driver.stop()

    def join(self):
        return self.driver.join()

    def __enter__(self):
        self.start()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.stop()
        self.join()
        if exc_type:
            raise exc_type, exc_value, traceback
Example #4
0
class Running(object):

    def __init__(self, scheduler, name, user='', master=os.getenv('MESOS_MASTER'),
                 implicit_acknowledge=1, *args, **kwargs):
        framework = FrameworkInfo(name=name, user=user, *args, **kwargs)
        scheduler = SchedulerProxy(scheduler)
        self.driver = MesosSchedulerDriver(scheduler, encode(framework),
                                           master, implicit_acknowledge)

        def shutdown(signal, frame):
            self.stop()

        signal.signal(signal.SIGINT, shutdown)
        signal.signal(signal.SIGTERM, shutdown)
        atexit.register(self.stop)

    def run(self):
        return self.driver.run()

    def start(self):
        status = self.driver.start()
        assert status == mesos_pb2.DRIVER_RUNNING
        return status

    def stop(self):
        return self.driver.stop()

    def join(self):
        return self.driver.join()

    def __enter__(self):
        self.start()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.stop()
        self.join()
        if exc_type:
            raise exc_type, exc_value, traceback
                         "using offer %s.",
                         task.task_id.value,
                         offer.id.value)
            driver.launchTasks(offer.id, [task])

    def statusUpdate(self, driver, update):
        '''
        when a task is started, over,
        killed or lost (slave crash, ....), this method
        will be triggered with a status message.
        '''
        logging.info("Task %s is in state %s" %
                     (update.task_id.value,
                      mesos_pb2.TaskState.Name(update.state)))

if __name__ == '__main__':
    framework = mesos_pb2.FrameworkInfo()
    framework.user = ""  # Have Mesos fill in the current user.
    framework.name = "hello-world"
    driver = MesosSchedulerDriver(
        HelloWorldScheduler(),
        framework,
        "zk://localhost:2181/mesos"  # assumes running on the master
    )
    driver.start()
    logging.info("Listening for Ctrl-C")
    signal.signal(signal.SIGINT, shutdown)
    while True:
        time.sleep(5)
    sys.exit(0)
Example #6
0
class MesosScheduler(Scheduler):
    def __init__(self, manager, master, options):
        Scheduler.__init__(self,manager)
        self.master = master
        self.cpus = options.cpus
        self.mem = parse_mem(options.mem)
        self.gpus = options.gpus
        self.task_per_node = options.parallel or multiprocessing.cpu_count()
        self.options = options
        self.group = options.group
        self.last_finish_time = 0
        self.executor = None
        self.driver = None
        self.lock = threading.RLock()
        self.task_waiting = []
        self.task_launched = {}
        self.slaveTasks = {}
        self.starting = False

    def start_driver(self):
        name = 'OpenCluster'
        if self.options.name :
            name = "%s-%s" % (name,self.options.name)
        else:
            name = "%s-%s" % (name,datetime.datetime.now().strftime("%Y%m%d%H%M%S%f"))

        if len(name) > 256:
            name = name[:256] + '...'

        framework = mesos_pb2.FrameworkInfo()
        framework.user = getuser()
        if framework.user == 'root':
            raise Exception("OpenCluster is not allowed to run as 'root'")
        framework.name = name
        framework.hostname = socket.gethostname()

        self.driver = MesosSchedulerDriver(self, framework, self.master)
        self.driver.start()
        logger.debug("Mesos Scheudler driver started")

        self.shuttingdown = False
        self.last_finish_time = time.time()
        self.stopped = False
        #
        # def check():
        #     while self.started:
        #         now = time.time()
        #         if not self.task_waiting and now - self.last_finish_time > MAX_IDLE_TIME:
        #             logger.info("stop mesos scheduler after %d seconds idle", now - self.last_finish_time)
        #             self.shutdown()
        #             break
        #         time.sleep(1)
        #
        #         if len(self.task_success()) + len(self.task_failed) == self.taskNum:
        #             self.shutdown()
        # spawn(check)

    @safe
    def registered(self, driver, frameworkId, masterInfo):
        self.started = True
        logger.debug("connect to master %s:%s(%s), registered as %s",
            int2ip(masterInfo.ip), masterInfo.port, masterInfo.id,
            frameworkId.value)
        self.executor = self.getExecutorInfo(str(frameworkId.value))

    @safe
    def reregistered(self, driver, masterInfo):
        logger.warning("re-connect to mesos master %s:%s(%s)",
            int2ip(masterInfo.ip), masterInfo.port, masterInfo.id)

    @safe
    def disconnected(self, driver):
        logger.debug("framework is disconnected")

    @safe
    def getExecutorInfo(self, framework_id):
        execInfo = mesos_pb2.ExecutorInfo()
        execInfo.executor_id.value = "multiframework"
        execInfo.command.value = '%s %s' % (
            sys.executable, # /usr/bin/python.exe or .../python
            os.path.abspath(os.path.join(os.path.dirname(__file__), 'simpleexecutor.py'))
        )
        v = execInfo.command.environment.variables.add()
        v.name = 'UID'
        v.value = str(os.getuid())
        v = execInfo.command.environment.variables.add()
        v.name = 'GID'
        v.value = str(os.getgid())

        if hasattr(execInfo, 'framework_id'):
            execInfo.framework_id.value = str(framework_id)

        Script = os.path.realpath(sys.argv[0])
        if hasattr(execInfo, 'name'):
            execInfo.name = Script

        execInfo.data = marshal.dumps((Script, os.getcwd(), sys.path, dict(os.environ), self.task_per_node, env.environ))

        return execInfo
    @safe
    def clearCache(self):
        self.task_launched.clear()
        self.slaveTasks.clear()

    @safe
    def submitTasks(self, tasks):
        if not tasks:
            return
        self.completionEvents.join() #Blocks until all items in the events queue have been gotten and processed.
        self.clearCache()
        self.task_waiting.extend(tasks)
        self.taskNum = self.taskNum + len(tasks)
        logger.debug("Got job with %d tasks",  len(tasks))

        if not self.started and not self.starting:
            self.starting = True
            self.start_driver()
        while not self.started:
            self.lock.release()
            time.sleep(0.01)
            self.lock.acquire()

        self.requestMoreResources()
        self.manager.statusUpdate()

    def requestMoreResources(self):
        if self.started:
            self.driver.reviveOffers()

    @safe
    def resourceOffers(self, driver, offers):

        rf = mesos_pb2.Filters()
        if not self.task_waiting:
            rf.refuse_seconds = 5
            for o in offers:
                driver.launchTasks(o.id, [], rf)
            return

        random.shuffle(offers)
        self.last_offer_time = time.time()
        for offer in offers:
            if self.shuttingdown:
                print "Shutting down: declining offer on [%s]" % offer.hostname
                driver.declineOffer(offer.id)
                continue

            attrs = self.getAttributes(offer)
            if self.options.group and attrs.get('group', 'None') not in self.options.group:
                driver.launchTasks(offer.id, [], rf)
                continue

            cpus, mem, gpus = self.getResources(offer)
            logger.debug("got resource offer %s: cpus:%s, mem:%s, gpus:%s at %s", offer.id.value, cpus, mem, gpus, offer.hostname)
            logger.debug("attributes,gpus:%s",attrs.get('gpus', None))
            sid = offer.slave_id.value
            tasks = []
            while (len(self.task_waiting)>0 and cpus >= self.cpus and mem >= self.mem and (self.gpus==0 or attrs.get('gpus', None) is not None)):

                logger.debug("Accepting resource on slave %s (%s)", offer.slave_id.value, offer.hostname)
                t = self.task_waiting.pop()
                t.state = mesos_pb2.TASK_STARTING
                t.state_time = time.time()

                task = self.create_task(offer, t, cpus)
                tasks.append(task)

                self.task_launched[t.id] = t
                self.slaveTasks.setdefault(sid, set()).add(t.id)

                cpus -= self.cpus
                mem -= self.mem
                # gpus -= self.gpus

            operation = mesos_pb2.Offer.Operation()
            operation.type = mesos_pb2.Offer.Operation.LAUNCH
            operation.launch.task_infos.extend(tasks)
            driver.acceptOffers([offer.id], [operation])

    @safe
    def offerRescinded(self, driver, offer_id):
        logger.debug("rescinded offer: %s", offer_id)
        if self.task_waiting:
            self.requestMoreResources()

    def getResources(self, offer):
        cpus, mem, gpus = 0, 0, 0
        for r in offer.resources:
            if r.name == 'gpus':
                gpus = float(r.scalar.value)
            elif r.name == 'cpus':
                cpus = float(r.scalar.value)
            elif r.name == 'mem':
                mem = float(r.scalar.value)
        return cpus, mem, gpus

    def getResource(self, res, name):
        for r in res:
            if r.name == name:
                return r.scalar.value
        return 0

    def getAttribute(self, attrs, name):
        for r in attrs:
            if r.name == name:
                return r.scalar.value

    def getAttributes(self, offer):
        attrs = {}
        for a in offer.attributes:
            attrs[a.name] = a.scalar.value
        return attrs

    def create_task(self, offer, t, cpus):
        task = mesos_pb2.TaskInfo()

        task.task_id.value = t.id
        task.slave_id.value = offer.slave_id.value
        task.name = "task(%s/%d)" % (t.id, self.taskNum)
        task.executor.MergeFrom(self.executor)

        task.data = compress(cPickle.dumps((t, t.tried), -1))

        cpu = task.resources.add()
        cpu.name = "cpus"
        cpu.type = 0 # mesos_pb2.Value.SCALAR
        cpu.scalar.value = min(self.cpus, cpus)

        mem = task.resources.add()
        mem.name = "mem"
        mem.type = 0 # mesos_pb2.Value.SCALAR
        mem.scalar.value = self.mem
        #
        # gpu = task.resources.add()
        # gpu.name = "gpus"
        # gpu.type = 0 # mesos_pb2.Value.SCALAR
        # gpu.scalar.value = self.gpus

        return task

    @safe
    def statusUpdate(self, driver, update):
        logger.debug("Task %s in state [%s]" % (update.task_id.value, mesos_pb2.TaskState.Name(update.state)))
        tid = str(update.task_id.value)

        if tid not in self.task_launched:
            # check failed after launched
            for t in self.task_waiting:
                if t.id == tid:
                    self.task_launched[tid] = t
                    self.task_waiting.remove(t)
                    break
            else:
                logger.debug("Task %s is finished, ignore it", tid)
                return

        t = self.task_launched[tid]
        t.state = update.state
        t.state_time = time.time()
        self.last_finish_time = t.state_time

        if update.state == mesos_pb2.TASK_RUNNING:
            self.started = True
            # to do task timeout handler
        elif update.state == mesos_pb2.TASK_LOST:
            self.task_launched.pop(tid)

            if t.tried < self.options.retry:
                t.tried += 1
                logger.warning("task %s lost, retry %s", t.id, update.state, t.tried)
                self.task_waiting.append(t) # try again
            else:
                self.taskEnded(t, OtherFailure("task lost,exception:" + str(update.data)), "task lost")

        elif update.state in (mesos_pb2.TASK_FINISHED, mesos_pb2.TASK_FAILED, mesos_pb2.TASK_ERROR, mesos_pb2.TASK_KILLED):
            self.task_launched.pop(tid)

            slave = None
            for s in self.slaveTasks:
                if tid in self.slaveTasks[s]:
                    slave = s
                    self.slaveTasks[s].remove(tid)
                    break

            if update.state == mesos_pb2.TASK_FINISHED :
                self.taskEnded(t, Success(), update.data)

            if update.state == mesos_pb2.TASK_ERROR :
                logger.error(update.message)
                self.taskEnded(t, OtherFailure(update.message), update.message)
                driver.abort()
                self.shutdown()

            if update.state == mesos_pb2.TASK_FAILED or update.state == mesos_pb2.TASK_KILLED or update.state == mesos_pb2.TASK_LOST:
                if t.tried < self.options.retry:
                    t.tried += 1
                    logger.warning("task %s failed with %s, retry %s", t.id, update.state, t.tried)
                    self.task_waiting.append(t) # try again
                else:
                    self.taskEnded(t, OtherFailure("exception:" + str(update.data)), None)
                    logger.error("task %s failed on %s", t.id, slave)

        if not self.task_waiting:
            self.requestMoreResources() # request more offers again

    @safe
    def check(self, driver):
        now = time.time()
        for tid, t in self.task_launched.items():
            if t.state == mesos_pb2.TASK_STARTING and t.state_time + 30 < now:
                logger.warning("task %s lauched failed, assign again", tid)
                if not self.task_waiting:
                    self.requestMoreResources()
                t.tried += 1
                t.state = -1
                self.task_launched.pop(tid)
                self.task_waiting.append(t)
            # TODO: check run time

    @safe
    def shutdown(self):
        if not self.started:
            return

        wait_started = datetime.datetime.now()
        while (len(self.task_launched) > 0) and \
          (SHUTDOWN_TIMEOUT > (datetime.datetime.now() - wait_started).seconds):
            time.sleep(1)

        logger.debug("total:%d, task finished: %d,task failed: %d", self.taskNum, self.finished_count, self.fail_count)

        self.shuttingdown = True
        # self.driver.join()
        self.driver.stop(False)

        #self.driver = None
        logger.debug("scheduler stop!!!")
        self.stopped = True
        self.started = False

    @safe
    def error(self, driver, code):
        logger.warning("Mesos error message: %s", code)

    def defaultParallelism(self):
        return 16

    def frameworkMessage(self, driver, executor, slave, data):
        logger.warning("[slave %s] %s", slave.value, data)

    def executorLost(self, driver, executorId, slaveId, status):
        logger.warning("executor at %s %s lost: %s", slaveId.value, executorId.value, status)
        self.slaveTasks.pop(slaveId.value, None)

    def slaveLost(self, driver, slaveId):
        logger.warning("slave %s lost", slaveId.value)
        self.slaveTasks.pop(slaveId.value, None)

    def killTask(self, job_id, task_id, tried):
        tid = mesos_pb2.TaskID()
        tid.value = "%s:%s:%s" % (job_id, task_id, tried)
        self.driver.killTask(tid)
Example #7
0
def start_factory_mesos():
    global pyroLoopCondition
    parser = OptionParser(
        usage="Usage: python factorymesos.py [options] <command>")
    parser.allow_interspersed_args = False
    parser.add_option("-s",
                      "--master",
                      type="string",
                      default="",
                      help="url of master (mesos://172.31.252.180:5050)")
    parser.add_option("-f",
                      "--factory",
                      type="string",
                      default="",
                      help="host:port of master (172.31.252.180:6666)")
    parser.add_option(
        "-w",
        "--warehouse_addr",
        type="string",
        default="",
        help=
        "kafka-172.31.252.182:9092|mysql-172.31.254.25:3306,db,username,password"
    )
    parser.add_option("-p",
                      "--task_per_node",
                      type="int",
                      default=0,
                      help="max number of tasks on one node (default: 0)")
    parser.add_option("-I",
                      "--image",
                      type="string",
                      help="image name for Docker")
    parser.add_option("-V",
                      "--volumes",
                      type="string",
                      help="volumes to mount into Docker")
    parser.add_option("-r",
                      "--retry",
                      type="int",
                      default=0,
                      help="retry times when failed (default: 0)")
    parser.add_option(
        "-e",
        "--config",
        type="string",
        default="/work/opencluster/config.ini",
        help=
        "absolute path of configuration file(default:/work/opencluster/config.ini)"
    )

    parser.add_option("-g",
                      "--group",
                      type="string",
                      default='',
                      help="which group to run (default: ''")
    parser.add_option(
        "-q",
        "--quiet",
        action="store_true",
        help="be quiet",
    )
    parser.add_option(
        "-v",
        "--verbose",
        action="store_true",
        help="show more useful log",
    )

    (options, command) = parser.parse_args()

    if not options:
        parser.print_help()
        sys.exit(2)

    if options.config:
        Conf.setConfigFile(options.config)

    options.master = options.master or Conf.getMesosMaster()
    options.warehouse_addr = options.warehouse_addr or Conf.getWareHouseAddr()

    servers = options.factory or Conf.getFactoryServers()
    servs = servers.split(",")
    server = servs[0].split(":")

    options.logLevel = (options.quiet and logging.ERROR
                        or options.verbose and logging.DEBUG or logging.INFO)
    setLogger(Conf.getFactoryServiceName(), "MESOS", options.logLevel)

    implicitAcknowledgements = 1
    if os.getenv("MESOS_EXPLICIT_ACKNOWLEDGEMENTS"):
        implicitAcknowledgements = 0
    sched = FactoryMesos(options, command, implicitAcknowledgements)

    driver = MesosSchedulerDriver(sched, sched.framework, options.master,
                                  implicitAcknowledgements)
    driver.start()
    logger.debug("Mesos Scheudler driver started")

    warehouse_addrs = options.warehouse_addr.split(",")

    def fetchTasksFromMySQL():
        global pyroLoopCondition
        mysqlIpAndPort = warehouse_addrs[0].split(":")
        last_data_time = time.time()

        while pyroLoopCondition:
            db = MySQLdb.connect(host=mysqlIpAndPort[0],
                                 port=int(mysqlIpAndPort[1]),
                                 db=warehouse_addrs[1],
                                 user=warehouse_addrs[2],
                                 passwd=warehouse_addrs[3])
            try:
                cur = db.cursor()
                curUpt = db.cursor()
                dataResults = cur.execute(
                    "select task_id,task_desc,task_start_time,status from t_task where status=0 order by priority asc limit 200"
                )
                results = cur.fetchmany(dataResults)
                for r in results:
                    sched.append_task(cPickle.loads(r[1]))
                    curUpt.execute(
                        "update t_task set task_start_time=now(),status=1 where task_id='"
                        + r[0] + "'")
                if len(results) > 0:
                    db.commit()
                    last_data_time = time.time()
                    driver.reviveOffers()

                if sched.tasks_total_len() > MAX_WAITING_TASK:
                    time.sleep(2)
                if time.time() - last_data_time > MAX_EMPTY_TASK_PERIOD:
                    time.sleep(10)

                if cur:
                    cur.close()
                if curUpt:
                    curUpt.close()
            finally:
                db.close()

    def fetchTasksFromKafka(priority):
        global pyroLoopCondition

        consumer = KafkaConsumer('OpenCluster%s' % priority,
                                 bootstrap_servers=[options.warehouse_addr],
                                 group_id="cnlab",
                                 auto_commit_enable=True,
                                 auto_commit_interval_ms=30 * 1000,
                                 auto_offset_reset='smallest')

        last_data_time = time.time()
        while pyroLoopCondition:
            for message in consumer.fetch_messages():
                logger.error("%s:%s:%s: key=%s " %
                             (message.topic, message.partition, message.offset,
                              message.key))
                sched.append_task(cPickle.loads(message.value))
                consumer.task_done(message)
                last_data_time = time.time()
            if sched.tasks_len(priority) > MAX_WAITING_TASK:
                time.sleep(2)
            if time.time() - last_data_time > MAX_EMPTY_TASK_PERIOD:
                time.sleep(10)

    if len(warehouse_addrs) > 2:
        spawn(fetchTasksFromMySQL)
    else:
        for i in range(1, sched.priority_size + 1):
            spawn(fetchTasksFromKafka, i)

    def handler(signm, frame):
        logger.warning("got signal %d, exit now", signm)
        sched.stop(3)

    signal.signal(signal.SIGTERM, handler)
    signal.signal(signal.SIGABRT, handler)

    try:
        while not sched.stopped:
            time.sleep(0.5)
            sched.check(driver)

            now = time.time()
            if now > sched.last_offer_time + 60 + random.randint(0, 5):
                logger.warning("too long to get offer, reviving...")
                sched.last_offer_time = now
                driver.reviveOffers()

    except KeyboardInterrupt:
        logger.warning(
            'stopped by KeyboardInterrupt. The Program is exiting gracefully! Please wait...'
        )
        sched.stop(4)

    #terminate pyrothread
    pyroLoopCondition = False

    time.sleep(5)
    driver.stop(False)
    sys.exit(sched.status)
Example #8
0
def run(api_url, mesos_master, user, config_dir, state_file, stats=None):
    scheduler = ChangesScheduler(config_dir, state_file, api=ChangesAPI(api_url), stats=stats)

    executor = mesos_pb2.ExecutorInfo()
    executor.executor_id.value = "default"
    executor.command.value = os.path.abspath("./executor.py")
    executor.name = "Changes Executor"
    executor.source = "changes"

    framework = mesos_pb2.FrameworkInfo()
    framework.user = user
    framework.name = "Changes Scheduler"
    framework.principal = "changes"
    # Give the scheduler 30s to restart before mesos cancels the tasks.
    framework.failover_timeout = 30

    if scheduler.framework_id:
        framework.id.value = scheduler.framework_id
        executor.framework_id.value = scheduler.framework_id

    driver = MesosSchedulerDriver(
        scheduler,
        framework,
        mesos_master)

    stopped = threading.Event()

    def handle_interrupt(signal, frame):
        stopped.set()
        logging.info("Received interrupt, shutting down")
        logging.warning("Not saving state. Will wait for running tasks to finish.")
        scheduler.shuttingDown.set()
        while scheduler.activeTasks > 0:
            logging.info("Waiting for %d tasks to finish running", scheduler.activeTasks)
            sleep(5)
        driver.stop()

    def handle_sigterm(signal, frame):
        stopped.set()
        logging.info("Received sigterm, shutting down")
        scheduler.shuttingDown.set()
        if scheduler.state_file:
            try:
                scheduler.save_state()
                logging.info("Successfully saved state to %s.", state_file)
            except Exception:
                logging.exception("Failed to save state")
                driver.stop()
                return
            # With `failover` set to true, we do not tell Mesos to stop the existing tasks
            # started by this framework. Instead, the tasks will run for
            # `fail_timeout` more seconds set above or we start a scheduler with
            # the same framework id.
            driver.stop(True)
        else:
            logging.warning("State file location not set. Not saving state. Existing builds will be cancelled.")
            driver.stop()

    signal.signal(signal.SIGINT, handle_interrupt)
    signal.signal(signal.SIGTERM, handle_sigterm)

    driver.start()
    logging.info("Driver started")
    while not stopped.is_set():
        stopped.wait(3)
    status = 0 if driver.join() == mesos_pb2.DRIVER_STOPPED else 1

    # Ensure that the driver process terminates.
    if status == 1:
        driver.stop()

    sys.exit(status)
Example #9
0
class MesosScheduler(Scheduler):
    def __init__(self, manager, master, options):
        Scheduler.__init__(self, manager)
        self.master = master
        self.cpus = options.cpus
        self.mem = parse_mem(options.mem)
        self.gpus = options.gpus
        self.task_per_node = options.parallel or multiprocessing.cpu_count()
        self.options = options
        self.group = options.group
        self.last_finish_time = 0
        self.executor = None
        self.driver = None
        self.lock = threading.RLock()
        self.task_waiting = []
        self.task_launched = {}
        self.slaveTasks = {}
        self.starting = False

    def start_driver(self):
        name = 'OpenCluster'
        if self.options.name:
            name = "%s-%s" % (name, self.options.name)
        else:
            name = "%s-%s" % (
                name, datetime.datetime.now().strftime("%Y%m%d%H%M%S%f"))

        if len(name) > 256:
            name = name[:256] + '...'

        framework = mesos_pb2.FrameworkInfo()
        framework.user = getuser()
        if framework.user == 'root':
            raise Exception("OpenCluster is not allowed to run as 'root'")
        framework.name = name
        framework.hostname = socket.gethostname()

        self.driver = MesosSchedulerDriver(self, framework, self.master)
        self.driver.start()
        logger.debug("Mesos Scheudler driver started")

        self.shuttingdown = False
        self.last_finish_time = time.time()
        self.stopped = False
        #
        # def check():
        #     while self.started:
        #         now = time.time()
        #         if not self.task_waiting and now - self.last_finish_time > MAX_IDLE_TIME:
        #             logger.info("stop mesos scheduler after %d seconds idle", now - self.last_finish_time)
        #             self.shutdown()
        #             break
        #         time.sleep(1)
        #
        #         if len(self.task_success()) + len(self.task_failed) == self.taskNum:
        #             self.shutdown()
        # spawn(check)

    @safe
    def registered(self, driver, frameworkId, masterInfo):
        self.started = True
        logger.debug("connect to master %s:%s(%s), registered as %s",
                     int2ip(masterInfo.ip), masterInfo.port, masterInfo.id,
                     frameworkId.value)
        self.executor = self.getExecutorInfo(str(frameworkId.value))

    @safe
    def reregistered(self, driver, masterInfo):
        logger.warning("re-connect to mesos master %s:%s(%s)",
                       int2ip(masterInfo.ip), masterInfo.port, masterInfo.id)

    @safe
    def disconnected(self, driver):
        logger.debug("framework is disconnected")

    @safe
    def getExecutorInfo(self, framework_id):
        execInfo = mesos_pb2.ExecutorInfo()
        execInfo.executor_id.value = "multiframework"
        execInfo.command.value = '%s %s' % (
            sys.executable,  # /usr/bin/python.exe or .../python
            os.path.abspath(
                os.path.join(os.path.dirname(__file__), 'simpleexecutor.py')))
        v = execInfo.command.environment.variables.add()
        v.name = 'UID'
        v.value = str(os.getuid())
        v = execInfo.command.environment.variables.add()
        v.name = 'GID'
        v.value = str(os.getgid())

        if hasattr(execInfo, 'framework_id'):
            execInfo.framework_id.value = str(framework_id)

        Script = os.path.realpath(sys.argv[0])
        if hasattr(execInfo, 'name'):
            execInfo.name = Script

        execInfo.data = marshal.dumps(
            (Script, os.getcwd(), sys.path, dict(os.environ),
             self.task_per_node, env.environ))

        return execInfo

    @safe
    def clearCache(self):
        self.task_launched.clear()
        self.slaveTasks.clear()

    @safe
    def submitTasks(self, tasks):
        if not tasks:
            return
        self.completionEvents.join(
        )  #Blocks until all items in the events queue have been gotten and processed.
        self.clearCache()
        self.task_waiting.extend(tasks)
        self.taskNum = self.taskNum + len(tasks)
        logger.debug("Got job with %d tasks", len(tasks))

        if not self.started and not self.starting:
            self.starting = True
            self.start_driver()
        while not self.started:
            self.lock.release()
            time.sleep(0.01)
            self.lock.acquire()

        self.requestMoreResources()
        self.manager.statusUpdate()

    def requestMoreResources(self):
        if self.started:
            self.driver.reviveOffers()

    @safe
    def resourceOffers(self, driver, offers):

        rf = mesos_pb2.Filters()
        if not self.task_waiting:
            rf.refuse_seconds = 5
            for o in offers:
                driver.launchTasks(o.id, [], rf)
            return

        random.shuffle(offers)
        self.last_offer_time = time.time()
        for offer in offers:
            if self.shuttingdown:
                print "Shutting down: declining offer on [%s]" % offer.hostname
                driver.declineOffer(offer.id)
                continue

            attrs = self.getAttributes(offer)
            if self.options.group and attrs.get(
                    'group', 'None') not in self.options.group:
                driver.launchTasks(offer.id, [], rf)
                continue

            cpus, mem, gpus = self.getResources(offer)
            logger.debug(
                "got resource offer %s: cpus:%s, mem:%s, gpus:%s at %s",
                offer.id.value, cpus, mem, gpus, offer.hostname)
            logger.debug("attributes,gpus:%s", attrs.get('gpus', None))
            sid = offer.slave_id.value
            tasks = []
            while (len(self.task_waiting) > 0 and cpus >= self.cpus
                   and mem >= self.mem and
                   (self.gpus == 0 or attrs.get('gpus', None) is not None)):

                logger.debug("Accepting resource on slave %s (%s)",
                             offer.slave_id.value, offer.hostname)
                t = self.task_waiting.pop()
                t.state = mesos_pb2.TASK_STARTING
                t.state_time = time.time()

                task = self.create_task(offer, t, cpus)
                tasks.append(task)

                self.task_launched[t.id] = t
                self.slaveTasks.setdefault(sid, set()).add(t.id)

                cpus -= self.cpus
                mem -= self.mem
                # gpus -= self.gpus

            operation = mesos_pb2.Offer.Operation()
            operation.type = mesos_pb2.Offer.Operation.LAUNCH
            operation.launch.task_infos.extend(tasks)
            driver.acceptOffers([offer.id], [operation])

    @safe
    def offerRescinded(self, driver, offer_id):
        logger.debug("rescinded offer: %s", offer_id)
        if self.task_waiting:
            self.requestMoreResources()

    def getResources(self, offer):
        cpus, mem, gpus = 0, 0, 0
        for r in offer.resources:
            if r.name == 'gpus':
                gpus = float(r.scalar.value)
            elif r.name == 'cpus':
                cpus = float(r.scalar.value)
            elif r.name == 'mem':
                mem = float(r.scalar.value)
        return cpus, mem, gpus

    def getResource(self, res, name):
        for r in res:
            if r.name == name:
                return r.scalar.value
        return 0

    def getAttribute(self, attrs, name):
        for r in attrs:
            if r.name == name:
                return r.scalar.value

    def getAttributes(self, offer):
        attrs = {}
        for a in offer.attributes:
            attrs[a.name] = a.scalar.value
        return attrs

    def create_task(self, offer, t, cpus):
        task = mesos_pb2.TaskInfo()

        task.task_id.value = t.id
        task.slave_id.value = offer.slave_id.value
        task.name = "task(%s/%d)" % (t.id, self.taskNum)
        task.executor.MergeFrom(self.executor)

        task.data = compress(cPickle.dumps((t, t.tried), -1))

        cpu = task.resources.add()
        cpu.name = "cpus"
        cpu.type = 0  # mesos_pb2.Value.SCALAR
        cpu.scalar.value = min(self.cpus, cpus)

        mem = task.resources.add()
        mem.name = "mem"
        mem.type = 0  # mesos_pb2.Value.SCALAR
        mem.scalar.value = self.mem
        #
        # gpu = task.resources.add()
        # gpu.name = "gpus"
        # gpu.type = 0 # mesos_pb2.Value.SCALAR
        # gpu.scalar.value = self.gpus

        return task

    @safe
    def statusUpdate(self, driver, update):
        logger.debug(
            "Task %s in state [%s]" %
            (update.task_id.value, mesos_pb2.TaskState.Name(update.state)))
        tid = str(update.task_id.value)

        if tid not in self.task_launched:
            # check failed after launched
            for t in self.task_waiting:
                if t.id == tid:
                    self.task_launched[tid] = t
                    self.task_waiting.remove(t)
                    break
            else:
                logger.debug("Task %s is finished, ignore it", tid)
                return

        t = self.task_launched[tid]
        t.state = update.state
        t.state_time = time.time()
        self.last_finish_time = t.state_time

        if update.state == mesos_pb2.TASK_RUNNING:
            self.started = True
            # to do task timeout handler
        elif update.state == mesos_pb2.TASK_LOST:
            self.task_launched.pop(tid)

            if t.tried < self.options.retry:
                t.tried += 1
                logger.warning("task %s lost, retry %s", t.id, update.state,
                               t.tried)
                self.task_waiting.append(t)  # try again
            else:
                self.taskEnded(
                    t, OtherFailure("task lost,exception:" + str(update.data)),
                    "task lost")

        elif update.state in (mesos_pb2.TASK_FINISHED, mesos_pb2.TASK_FAILED,
                              mesos_pb2.TASK_ERROR, mesos_pb2.TASK_KILLED):
            self.task_launched.pop(tid)

            slave = None
            for s in self.slaveTasks:
                if tid in self.slaveTasks[s]:
                    slave = s
                    self.slaveTasks[s].remove(tid)
                    break

            if update.state == mesos_pb2.TASK_FINISHED:
                self.taskEnded(t, Success(), update.data)

            if update.state == mesos_pb2.TASK_ERROR:
                logger.error(update.message)
                self.taskEnded(t, OtherFailure(update.message), update.message)
                driver.abort()
                self.shutdown()

            if update.state == mesos_pb2.TASK_FAILED or update.state == mesos_pb2.TASK_KILLED or update.state == mesos_pb2.TASK_LOST:
                if t.tried < self.options.retry:
                    t.tried += 1
                    logger.warning("task %s failed with %s, retry %s", t.id,
                                   update.state, t.tried)
                    self.task_waiting.append(t)  # try again
                else:
                    self.taskEnded(
                        t, OtherFailure("exception:" + str(update.data)), None)
                    logger.error("task %s failed on %s", t.id, slave)

        if not self.task_waiting:
            self.requestMoreResources()  # request more offers again

    @safe
    def check(self, driver):
        now = time.time()
        for tid, t in self.task_launched.items():
            if t.state == mesos_pb2.TASK_STARTING and t.state_time + 30 < now:
                logger.warning("task %s lauched failed, assign again", tid)
                if not self.task_waiting:
                    self.requestMoreResources()
                t.tried += 1
                t.state = -1
                self.task_launched.pop(tid)
                self.task_waiting.append(t)
            # TODO: check run time

    @safe
    def shutdown(self):
        if not self.started:
            return

        wait_started = datetime.datetime.now()
        while (len(self.task_launched) > 0) and \
          (SHUTDOWN_TIMEOUT > (datetime.datetime.now() - wait_started).seconds):
            time.sleep(1)

        logger.debug("total:%d, task finished: %d,task failed: %d",
                     self.taskNum, self.finished_count, self.fail_count)

        self.shuttingdown = True
        # self.driver.join()
        self.driver.stop(False)

        #self.driver = None
        logger.debug("scheduler stop!!!")
        self.stopped = True
        self.started = False

    @safe
    def error(self, driver, code):
        logger.warning("Mesos error message: %s", code)

    def defaultParallelism(self):
        return 16

    def frameworkMessage(self, driver, executor, slave, data):
        logger.warning("[slave %s] %s", slave.value, data)

    def executorLost(self, driver, executorId, slaveId, status):
        logger.warning("executor at %s %s lost: %s", slaveId.value,
                       executorId.value, status)
        self.slaveTasks.pop(slaveId.value, None)

    def slaveLost(self, driver, slaveId):
        logger.warning("slave %s lost", slaveId.value)
        self.slaveTasks.pop(slaveId.value, None)

    def killTask(self, job_id, task_id, tried):
        tid = mesos_pb2.TaskID()
        tid.value = "%s:%s:%s" % (job_id, task_id, tried)
        self.driver.killTask(tid)
Example #10
0
def run(api_url,
        mesos_master,
        user,
        config_dir,
        state_file,
        changes_request_limit,
        http_port,
        stats=None):
    scheduler = ChangesScheduler(state_file,
                                 api=ChangesAPI(api_url),
                                 stats=stats,
                                 blacklist=FileBlacklist(
                                     os.path.join(config_dir, 'blacklist')),
                                 changes_request_limit=changes_request_limit)

    executor = mesos_pb2.ExecutorInfo()
    executor.executor_id.value = "default"
    executor.command.value = os.path.abspath("./executor.py")
    executor.name = "Changes Executor"
    executor.source = "changes"

    framework = mesos_pb2.FrameworkInfo()
    framework.user = user
    framework.name = "Changes Scheduler"
    framework.principal = "changes"
    # Give the scheduler 1 week to restart before mesos cancels the tasks.
    # this is the setting recommended by the docs.
    framework.failover_timeout = 3600 * 24 * 7

    if scheduler.framework_id:
        framework.id.value = scheduler.framework_id
        executor.framework_id.value = scheduler.framework_id

    driver = MesosSchedulerDriver(scheduler, framework, mesos_master)

    stopped = threading.Event()

    def handle_interrupt(signal, frame):
        stopped.set()
        logging.info("Received interrupt, shutting down")
        logging.warning(
            "Not saving state. Will wait for running tasks to finish.")
        scheduler.shuttingDown.set()
        while scheduler.activeTasks > 0:
            logging.info("Waiting for %d tasks to finish running",
                         scheduler.activeTasks)
            sleep(5)
        driver.stop()

    def handle_sigterm(signal, frame):
        # TODO: Avoid save_state race conditions by having handle_sigterm()
        # only set shuttingDown, then do the actual save-state and driver.stop()
        # in the main thread after all other threads are join()ed.
        # Also, stopped doesn't appear to be used.
        stopped.set()
        logging.info("Received sigterm, shutting down")
        scheduler.shuttingDown.set()
        if scheduler.state_file:
            try:
                scheduler.save_state()
                logging.info("Successfully saved state to %s.", state_file)
            except Exception:
                logging.exception("Failed to save state")
                driver.stop()
                return
            # With `failover` set to true, we do not tell Mesos to stop the existing tasks
            # started by this framework. Instead, the tasks will run for
            # `fail_timeout` more seconds set above or we start a scheduler with
            # the same framework id.
            driver.stop(True)
        else:
            logging.warning(
                "State file location not set. Not saving state. Existing builds will be cancelled."
            )
            driver.stop()

    signal.signal(signal.SIGINT, handle_interrupt)
    signal.signal(signal.SIGTERM, handle_sigterm)

    driver.start()
    logging.info("Driver started")

    app = Flask("Changes Mesos Scheduler")
    app.add_url_rule('/api/state_json', 'state_json',
                     json_handler(scheduler.state_json))
    http_thread = threading.Thread(target=app.run, kwargs={'port': http_port})
    http_thread.start()

    scheduler.poll_changes_until_shutdown(driver, 5)
    status = 0
    if driver.join() == mesos_pb2.DRIVER_STOPPED:
        logging.info("Driver stopped cleanly.")
    else:
        # Ensure that the driver process terminates.
        status = 1
        logging.info("Stopping driver forcibly.")
        driver.stop()

    logging.info("Stopping HTTP server.")
    http_thread.terminate()
    http_thread.join()

    logging.info("Clean shutdown complete. Exiting status %d.", status)
    sys.exit(status)