def __init__(self, name, options, command, role=None):
self.framework_id = None
self.executor = None
self.framework = Dict()
self.framework.user = getuser()
if self.framework.user == 'root':
raise Exception('drun is not allowed to run as \'root\'')
self.framework.name = name
self.framework.hostname = socket.gethostname()
if role is not None:
self.framework.role = role
self.cpus = options.cpus
self.gpus = options.gpus
self.mem = memory_str_to_mb(options.mem)
self.options = options
self.command = command
self.started = False
self.stopped = False
self.status = 0
self.next_try = 0
self.lock = threading.RLock()
self.last_offer_time = time.time()
self.task_launched = {}
self.agentTasks = {}
# threads
self.stdout_t = None
self.stderr_t = None
def add_loghub(framework_id):
logger = get_logger('dpark')
try:
import dpark
from dpark.conf import LOGHUB, ENABLE_ES_LOGHUB, ES_HOST, ES_INDEX, ES_TYPE, LOGHUB_PATH_FORMAT
from dpark.utils import getuser
date_str = datetime.now().strftime(LOGHUB_PATH_FORMAT)
date_dir_path = os.path.join(LOGHUB, date_str)
if not os.path.exists(date_dir_path):
logger.error("loghub dir not ready: %s", date_dir_path)
return
dir_path = os.path.join(date_dir_path, framework_id)
os.mkdir(dir_path)
infos = [
("CMD", ' '.join(sys.argv)),
("USER", getuser()),
("PWD", os.getcwd()),
("CTIME", datetime.strftime(datetime.now(), "%Y-%m-%d %H:%M:%S")),
("DPARK", dpark.__file__),
("PYTHONPATH", os.environ.get("PYTHONPATH", ""))
]
log_path = os.path.join(dir_path, "log")
try:
with open(log_path, "a") as f:
for i in infos:
f.write("DPARK_{} = {}\n".format(i[0], i[1]))
f.write("\n")
except IOError:
logger.exception("fail to write loghub: %s", log_path)
return
if ENABLE_ES_LOGHUB:
es_handler = ElasticSearchHandler(ES_HOST, ES_INDEX, ES_TYPE,
infos, log_path)
es_handler.setLevel(logging.WARNING)
logger.addHandler(es_handler)
file_handler = logging.FileHandler(filename=log_path)
file_handler.setFormatter(ColoredFormatter(LOG_FORMAT, DATE_FORMAT, True))
file_handler.setLevel(logging.INFO)
logger.addHandler(file_handler)
logger.info("logging/prof to %s", dir_path)
return file_handler, dir_path
except Exception:
logger.exception("add_loghub fail")
def add_loghub(framework_id):
logger = get_logger('dpark')
try:
import dpark
from dpark.conf import LOGHUB, ENABLE_ES_LOGHUB, ES_HOST, ES_INDEX, ES_TYPE, LOGHUB_PATH_FORMAT
from dpark.utils import getuser
date_str = datetime.now().strftime(LOGHUB_PATH_FORMAT)
date_dir_path = os.path.join(LOGHUB, date_str)
if not os.path.exists(date_dir_path):
logger.error("loghub dir not ready: %s", date_dir_path)
return
dir_path = os.path.join(date_dir_path, framework_id)
os.mkdir(dir_path)
infos = [("CMD", ' '.join(sys.argv)), ("USER", getuser()),
("PWD", os.getcwd()),
("CTIME",
datetime.strftime(datetime.now(), "%Y-%m-%d %H:%M:%S")),
("DPARK", dpark.__file__),
("PYTHONPATH", os.environ.get("PYTHONPATH", ""))]
log_path = os.path.join(dir_path, "log")
try:
with open(log_path, "a") as f:
for i in infos:
f.write("DPARK_{} = {}\n".format(i[0], i[1]))
f.write("\n")
except IOError:
logger.exception("fail to write loghub: %s", log_path)
return
if ENABLE_ES_LOGHUB:
es_handler = ElasticSearchHandler(ES_HOST, ES_INDEX, ES_TYPE,
infos, log_path)
es_handler.setLevel(logging.WARNING)
logger.addHandler(es_handler)
file_handler = logging.FileHandler(filename=log_path)
file_handler.setFormatter(
ColoredFormatter(LOG_FORMAT, DATE_FORMAT, True))
file_handler.setLevel(logging.INFO)
logger.addHandler(file_handler)
logger.info("logging/prof to %s", dir_path)
return file_handler, dir_path
except Exception:
logger.exception("add_loghub fail")
def __init__(self, name, options, command, role=None):
self.framework_id = None
self.executor = None
self.framework = Dict()
self.framework.user = getuser()
if self.framework.user == 'root':
raise Exception('drun is not allowed to run as \'root\'')
self.framework.name = name
self.framework.hostname = socket.gethostname()
if role is not None:
self.framework.role = role
self.cpus = options.cpus
self.gpus = options.gpus
self.mem = memory_str_to_mb(options.mem)
self.options = options
self.command = command
self.started = False
self.stopped = False
self.ec = EXIT_UNEXPECTED
self.next_try = 0
self.lock = threading.RLock()
self.last_offer_time = time.time()
self.task_launched = {}
self.agentTasks = {}
# threads
self.stdout_t = None
self.stderr_t = None
self.loghub_dir = None
self.stats = {
'ec': EXIT_UNEXPECTED,
'init_time': time.time(),
'submit_times': {}, # bind offers
'start_time': None, # mrun only: get all resources
'slots': {}, # empty for drun
'finish_times': {},
'stop_time': None,
'num_task': options.tasks,
'mem_mb': self.mem,
'scheduler': self.short_name,
'cmd': os.path.abspath(sys.argv[0]) + ' ' + ' '.join(sys.argv[1:])
}
def start_driver(self):
name = '[dpark] ' + \
os.path.abspath(sys.argv[0]) + ' ' + ' '.join(sys.argv[1:])
if len(name) > 256:
name = name[:256] + '...'
framework = Dict()
framework.user = getuser()
if framework.user == 'root':
raise Exception('dpark is not allowed to run as \'root\'')
framework.name = name
if self.role:
framework.role = self.role
framework.hostname = socket.gethostname()
if self.webui_url:
framework.webui_url = self.webui_url
self.driver = MesosSchedulerDriver(self,
framework,
self.master,
use_addict=True)
self.driver.start()
logger.debug('Mesos Scheudler driver started')
self.started = True
self.last_finish_time = time.time()
def check():
while self.started:
with self.lock:
now = time.time()
if (not self.active_tasksets
and now - self.last_finish_time > MAX_IDLE_TIME):
logger.info(
'stop mesos scheduler after %d seconds idle',
now - self.last_finish_time)
self.stop()
break
for taskset in self.active_tasksets.values():
if taskset.check_task_timeout():
self.requestMoreResources()
time.sleep(1)
spawn(check)
def add_loghub(framework_id):
logger = get_logger('dpark')
try:
import dpark
from dpark.conf import LOGHUB
from dpark.utils import getuser
date_str = datetime.now().strftime("%Y/%m/%d/%H")
date_dir_path = os.path.join(LOGHUB, date_str)
if not os.path.exists(date_dir_path):
logger.error("loghub dir not ready: %s", date_dir_path)
return
dir_path = os.path.join(date_dir_path, framework_id)
os.mkdir(dir_path)
dpark_mtime = datetime.fromtimestamp(os.stat(dpark.__file__).st_mtime).strftime('%Y-%m-%dT%H:%M:%S')
infos = [
("CMD", ' '.join(sys.argv)),
("USER", getuser()),
("PWD", os.getcwd()),
("DPARK", dpark.__file__),
("DPARK_MTIME", dpark_mtime),
("PYTHONPATH", os.environ.get("PYTHONPATH", ""))
]
log_path = os.path.join(dir_path, "log")
try:
with open(log_path, "a") as f:
for i in infos:
f.write("DPARK_{} = {}\n".format(i[0], i[1]))
f.write("\n")
except:
logger.exception("fail to write loghub: %s", log_path)
return
file_handler = logging.FileHandler(filename=log_path)
file_handler.setFormatter(ColoredFormatter(LOG_FORMAT, DATE_FORMAT, True))
file_handler.setLevel(logging.INFO)
logger.addHandler(file_handler)
logger.info("logging/prof to %s", dir_path)
return dir_path
except:
logger.exception("add_loghub fail")
def __init__(self, name, options, command, role=None):
self.framework_id = None
self.executor = None
self.framework = Dict()
self.framework.user = getuser()
if self.framework.user == 'root':
raise Exception('drun is not allowed to run as \'root\'')
self.framework.name = name
self.framework.hostname = socket.gethostname()
if role is not None:
self.framework.role = role
self.cpus = options.cpus
self.gpus = options.gpus
self.mem = memory_str_to_mb(options.mem)
self.options = options
self.command = command
self.started = False
self.stopped = False
self.ec = EXIT_UNEXPECTED
self.next_try = 0
self.lock = threading.RLock()
self.last_offer_time = time.time()
self.task_launched = {}
self.agentTasks = {}
# threads
self.stdout_t = None
self.stderr_t = None
self.loghub_dir = None
self.stats = {'ec': EXIT_UNEXPECTED,
'init_time': time.time(),
'submit_times': {}, # bind offers
'start_time': None, # mrun only: get all resources
'slots': {}, # empty for drun
'finish_times': {},
'stop_time': None,
'num_task': options.tasks,
'mem_mb': self.mem,
'scheduler': self.short_name,
'cmd': os.path.abspath(sys.argv[0]) + ' ' + ' '.join(sys.argv[1:])
}