def __init__(self, name, options, command):
self.framework_id = None
self.executor = None
self.framework = Dict()
self.framework.user = getuser()
if self.framework.user == 'root':
raise Exception('drun is not allowed to run as \'root\'')
self.framework.name = name
self.framework.hostname = socket.gethostname()
self.cpus = options.cpus
self.gpus = options.gpus
self.mem = memory_str_to_mb(options.mem)
self.options = options
self.command = command
self.started = False
self.stopped = False
self.status = 0
self.next_try = 0
self.lock = threading.RLock()
self.last_offer_time = time.time()
self.task_launched = {}
self.agentTasks = {}
# threads
self.stdout_t = None
self.stderr_t = None
def start_driver(self):
name = '[dpark] ' + os.path.abspath(sys.argv[0]) + ' ' + ' '.join(
sys.argv[1:])
if len(name) > 256:
name = name[:256] + '...'
framework = mesos_pb2.FrameworkInfo()
framework.user = getuser()
if framework.user == 'root':
raise Exception("dpark is not allowed to run as 'root'")
framework.name = name
framework.hostname = socket.gethostname()
self.driver = mesos.MesosSchedulerDriver(self, framework, self.master)
self.driver.start()
logger.debug("Mesos Scheudler driver started")
self.started = True
self.last_finish_time = time.time()
def check():
while self.started:
now = time.time()
if not self.activeJobs and now - self.last_finish_time > MAX_IDLE_TIME:
logger.info("stop mesos scheduler after %d seconds idle",
now - self.last_finish_time)
self.stop()
break
time.sleep(1)
spawn(check)
def __init__(self, options, command):
self.framework_id = None
self.executor = None
self.framework = mesos_pb2.FrameworkInfo()
self.framework.user = getuser()
if self.framework.user == 'root':
raise Exception("drun is not allowed to run as 'root'")
name = '[drun] ' + ' '.join(sys.argv[1:])
if len(name) > 256:
name = name[:256] + '...'
self.framework.name = name
self.framework.hostname = socket.gethostname()
self.cpus = options.cpus
self.mem = parse_mem(options.mem)
self.options = options
self.command = command
self.total_tasks = list(reversed([Task(i)
for i in range(options.start, options.tasks)]))
self.task_launched = {}
self.slaveTasks = {}
self.started = False
self.stopped = False
self.status = 0
self.next_try = 0
self.lock = threading.RLock()
self.last_offer_time = time.time()
def start_driver(self):
name = '[dpark] ' + os.path.abspath(sys.argv[0]) + ' ' + ' '.join(sys.argv[1:])
if len(name) > 256:
name = name[:256] + '...'
framework = mesos_pb2.FrameworkInfo()
framework.user = getuser()
if framework.user == 'root':
raise Exception("dpark is not allowed to run as 'root'")
framework.name = name
framework.hostname = socket.gethostname()
self.driver = mesos.MesosSchedulerDriver(self, framework,
self.master)
self.driver.start()
logger.debug("Mesos Scheudler driver started")
self.started = True
self.last_finish_time = time.time()
def check():
while self.started:
now = time.time()
if not self.activeJobs and now - self.last_finish_time > MAX_IDLE_TIME:
logger.info("stop mesos scheduler after %d seconds idle",
now - self.last_finish_time)
self.stop()
break
time.sleep(1)
spawn(check)
def __init__(self, options, command):
self.framework_id = None
self.executor = None
self.framework = mesos_pb2.FrameworkInfo()
self.framework.user = getuser()
if self.framework.user == 'root':
raise Exception("drun is not allowed to run as 'root'")
name = '[drun] ' + ' '.join(sys.argv[1:])
if len(name) > 256:
name = name[:256] + '...'
self.framework.name = name
self.framework.hostname = socket.gethostname()
self.cpus = options.cpus
self.mem = parse_mem(options.mem)
self.options = options
self.command = command
self.total_tasks = list(
reversed([Task(i) for i in range(options.start, options.tasks)]))
self.task_launched = {}
self.slaveTasks = {}
self.started = False
self.stopped = False
self.status = 0
self.next_try = 0
self.lock = threading.RLock()
self.last_offer_time = time.time()
def start_driver(self):
name = '[dpark] ' + \
os.path.abspath(sys.argv[0]) + ' ' + ' '.join(sys.argv[1:])
if len(name) > 256:
name = name[:256] + '...'
framework = Dict()
framework.user = getuser()
if framework.user == 'root':
raise Exception('dpark is not allowed to run as \'root\'')
framework.name = name
framework.hostname = socket.gethostname()
framework.webui_url = self.options.webui_url
self.driver = MesosSchedulerDriver(self,
framework,
self.master,
use_addict=True)
self.driver.start()
logger.debug('Mesos Scheudler driver started')
self.started = True
self.last_finish_time = time.time()
def check():
while self.started:
now = time.time()
if (not self.activeJobs
and now - self.last_finish_time > MAX_IDLE_TIME):
logger.info('stop mesos scheduler after %d seconds idle',
now - self.last_finish_time)
self.stop()
break
time.sleep(1)
spawn(check)
def start_driver(self):
name = '[dpark] ' + \
os.path.abspath(sys.argv[0]) + ' ' + ' '.join(sys.argv[1:])
if len(name) > 256:
name = name[:256] + '...'
framework = Dict()
framework.user = getuser()
if framework.user == 'root':
raise Exception('dpark is not allowed to run as \'root\'')
framework.name = name
framework.hostname = socket.gethostname()
framework.webui_url = self.options.webui_url
self.driver = MesosSchedulerDriver(
self, framework, self.master, use_addict=True
)
self.driver.start()
logger.debug('Mesos Scheudler driver started')
self.started = True
self.last_finish_time = time.time()
def check():
while self.started:
now = time.time()
if (not self.activeJobs and
now - self.last_finish_time > MAX_IDLE_TIME):
logger.info('stop mesos scheduler after %d seconds idle',
now - self.last_finish_time)
self.stop()
break
time.sleep(1)
spawn(check)
def __init__(self, name, options, command):
self.framework_id = None
self.executor = None
self.framework = Dict()
self.framework.user = getuser()
if self.framework.user == 'root':
raise Exception('drun is not allowed to run as \'root\'')
self.framework.name = name
self.framework.hostname = socket.gethostname()
self.cpus = options.cpus
self.mem = memory_str_to_mb(options.mem)
self.options = options
self.command = command
self.started = False
self.stopped = False
self.status = 0
self.next_try = 0
self.lock = threading.RLock()
self.last_offer_time = time.time()
self.task_launched = {}
self.agentTasks = {}
# threads
self.stdout_t = None
self.stderr_t = None
def __init__(self, name, options, command):
self.framework_id = None
self.executor = None
self.framework = mesos_pb2.FrameworkInfo()
self.framework.user = getuser()
if self.framework.user == 'root':
raise Exception("drun is not allowed to run as 'root'")
self.framework.name = name
self.framework.hostname = socket.gethostname()
self.cpus = options.cpus
self.mem = memory_str_to_mb(options.mem)
self.options = options
self.command = command
self.started = False
self.stopped = False
self.status = 0
self.next_try = 0
self.lock = threading.RLock()
self.last_offer_time = time.time()
self.task_launched = {}
self.slaveTasks = {}
def add_loghub(framework_id):
logger = get_logger('dpark')
try:
import dpark
from dpark.conf import LOGHUB
from dpark.util import getuser
date_str = datetime.now().strftime("%Y/%m/%d/%H")
dir_path = os.path.join(LOGHUB, date_str)
if not os.path.exists(dir_path):
logger.error("loghub dir not ready: %s", dir_path)
return
dpark_mtime = datetime.fromtimestamp(os.stat(
dpark.__file__).st_mtime).strftime('%Y-%m-%dT%H:%M:%S')
infos = [("CMD", ' '.join(sys.argv)), ("USER", getuser()),
("DPARK", dpark.__file__), ("DPARK_MTIME", dpark_mtime),
("PYTHONPATH", os.environ.get("PYTHONPATH", ""))]
log_path = os.path.join(dir_path, framework_id + ".log")
try:
with open(log_path, "a") as f:
for i in infos:
f.write("DPARK_{} = {}\n".format(i[0], i[1]))
f.write("\n")
except Exception as e:
logger.exception("fail to write loghub: %s", log_path)
return
file_handler = logging.FileHandler(filename=log_path)
file_handler.setFormatter(
ColoredFormatter(LOG_FORMAT, DATE_FORMAT, True))
file_handler.setLevel(logging.INFO)
logger.addHandler(file_handler)
except Exception as e:
logger.exception("add_loghub fail")