def __init__(self, url="http://127.0.0.1:5050"):
logging.basicConfig()
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.DEBUG)
self.launched_tasks = 0
self.running_tasks = 0
self.finished_tasks = 0
self.killed_tasks = 0
self.lost_tasks = []
# self.failed_tasks = []
#signal.signal(signal.SIGINT, signal.SIG_IGN)
logging.getLogger('mesoshttp').setLevel(logging.DEBUG)
self.driver = None
self.client = MesosClient(mesos_urls=[url],
frameworkId=None,
frameworkName='Python HTTP framework',
frameworkUser=getpass.getuser())
self.client.on(MesosClient.SUBSCRIBED, self.subscribed)
self.client.on(MesosClient.OFFERS, self.offer_received)
self.client.on(MesosClient.UPDATE, self.status_update)
self.th = Test.MesosFramework(self.client)
self.th.start()
while True and self.th.isAlive():
try:
self.th.join(1)
except KeyboardInterrupt:
self.shutdown()
break
def __init__(self, cfg, espa_api, worklist):
master = cfg.get('mesos_master')
principal = cfg.get('mesos_principal')
secret = cfg.get('mesos_secret')
self.workList = worklist
self.runningList = {}
self.max_cpus = cfg.get('max_cpu')
self.required_cpus = cfg.get('task_cpu')
self.required_memory = cfg.get('task_mem')
self.required_disk = cfg.get('task_disk')
self.task_image = cfg.get('task_image')
self.refuse_seconds = cfg.get('offer_refuse_seconds')
self.request_count = cfg.get('product_request_count')
self.products = cfg.get('product_frequency')
self.healthy_states = [
"TASK_STAGING", "TASK_STARTING", "TASK_RUNNING", "TASK_FINISHED"
]
self.espa = espa_api
self.cfg = cfg
self.client = MesosClient(mesos_urls=[master],
frameworkName='ESPA Mesos Framework')
self.client.verify = False
self.client.set_credentials(principal, secret)
self.client.on(MesosClient.SUBSCRIBED, self.subscribed)
self.client.on(MesosClient.OFFERS, self.offer_received)
self.client.on(MesosClient.UPDATE, self.status_update)
# put some work on the queue
get_products_to_process(cfg, self.espa, self.workList)
def __init__(self):
logging.basicConfig()
self.logger = logging.getLogger(__name__)
logging.getLogger('mesoshttp').setLevel(logging.DEBUG)
self.driver = None
# Note: leader.mesos address requires Mesos DNS
#self.client = MesosClient(mesos_urls=['zk://leader.mesos:2181/mesos'])
# If you are purely using Mesos, you should use explicit address of Master
# Example: Zookeeper master discovery
#self.client = MesosClient(mesos_urls=['zk://127.0.0.1:2181/mesos'])
# Example: Directly address Mesos
#self.client = MesosClient(mesos_urls=['http://127.0.0.1:5050'])
# By default, use direct master addressing
# Allow for comma delimited URLs to be passed in via MASTER_URLS
# environment variable
master_urls = os.getenv('MESOS_URLS', 'http://127.0.0.1:5050')
self.client = MesosClient(mesos_urls=master_urls.split(','))
secret = os.getenv('SERVICE_SECRET')
if secret:
self.client.set_service_account(json.loads(secret))
self.client.on(MesosClient.SUBSCRIBED, self.subscribed)
self.client.on(MesosClient.OFFERS, self.offer_received)
self.client.on(MesosClient.UPDATE, self.status_update)
self.th = Test.MesosFramework(self.client)
self.th.start()
while True and self.th.isAlive():
try:
self.th.join(1)
except KeyboardInterrupt:
self.shutdown()
break
def __init__(self, url="http://127.0.0.1:5050", name="HTTP framework", user="******"):
logging.basicConfig()
self.logger = logging.getLogger(__name__)
self.tasks = 0
#signal.signal(signal.SIGINT, signal.SIG_IGN)
logging.getLogger('mesoshttp').setLevel(logging.DEBUG)
self.driver = None
self.mesos_offer = None # to store only one offer
self.update = None
self.task_id = None
self.agent_id = None
self.client = MesosClient(mesos_urls = [url], frameworkName = name, frameworkUser = user)
self.client.on(MesosClient.SUBSCRIBED, self.subscribed)
self.client.on(MesosClient.OFFERS, self.offer_received)
self.client.on(MesosClient.UPDATE, self.status_update)
self.th = Test.MesosFramework(self.client)
def __init__(self):
logging.basicConfig()
self.logger = logging.getLogger(__name__)
logging.getLogger('mesoshttp').setLevel(logging.DEBUG)
self.driver = None
self.client = MesosClient(mesos_urls=['http://52.87.159.219:5050'])
self.client.on(MesosClient.SUBSCRIBED, self.subscribed)
self.client.on(MesosClient.OFFERS, self.offer_received)
self.client.on(MesosClient.UPDATE, self.status_update)
self.th = Test.MesosFramework(self.client)
self.th.start()
while True and self.th.isAlive():
try:
self.th.join(1)
except KeyboardInterrupt:
self.shutdown()
break
def run_scheduler(self, mesos_master):
logger.info("Scale rising...")
self.scheduler = ScaleScheduler()
self.scheduler.initialize()
scheduler_mgr.hostname = socket.getfqdn()
logger.info('Connecting to Mesos master at %s:', mesos_master)
# By default use ZK for master detection
self.client = MesosClient(
mesos_urls=[settings.MESOS_MASTER],
# We have to run tasks as root, so docker commands may be executed
frameworkUser='******',
frameworkName=settings.FRAMEWORK_NAME,
frameworkHostname=scheduler_mgr.hostname,
frameworkWebUI=settings.WEBSERVER_ADDRESS)
if settings.SERVICE_SECRET:
# We are in Enterprise mode and using service account
self.client.set_service_account(json.loads(
settings.SERVICE_SECRET))
elif settings.PRINCIPAL and settings.SECRET:
self.client.set_credentials(settings.PRINCIPAL, settings.SECRET)
mesos_role = settings.MESOS_ROLE
logger.info('Launching scheduler with role: %s' % mesos_role)
self.client.set_role(settings.MESOS_ROLE)
logger.info('Accepting offers from role: %s' %
settings.ACCEPTED_RESOURCE_ROLE)
self.client.add_capability('GPU_RESOURCES')
try:
self.scheduler.run(self.client)
status = 0
except:
status = 1
logger.exception('Mesos Scheduler Driver returned an exception')
#Perform a shut down and return any non-zero status
shutdown_status = self._shutdown()
status = status or shutdown_status
logger.info('Exiting...')
sys.exit(status)
def start(self):
self.fprint("Start, connecting to:" + MESOS_MASTER)
self.driver = None
self.client = MesosClient(mesos_urls=[MESOS_MASTER])
self.client.on(MesosClient.SUBSCRIBED, self.subscribed)
self.client.on(MesosClient.OFFERS, self.offer_received)
self.client.on(MesosClient.UPDATE, self.status_update)
self.client.frameworkName = "HPC Framework"
self.th = HpcFramework.MesosFramework(self.client)
self.th.start()
while self.th.isAlive():
try:
self.running = True
self.th.join(1)
except Exception as e:
self.fprint("mesos framework exception" + str(e))
self.running = False
self.fprint("mesos framework stopped")
def __init__(self,
script_path,
setup_path,
headnode,
ssl_thumbprint,
client_cert,
node_group=""):
logging.basicConfig()
self.logger = logging_aux.init_logger_aux("hpcframework",
"hpcframework.log")
# signal.signal(signal.SIGINT, signal.SIG_IGN)
logging.getLogger('mesoshttp').setLevel(logging.DEBUG)
self.node_idle_check_table = {}
self.script_path = script_path
self.setup_path = setup_path
self.headnode = headnode
self.ssl_thumbprint = ssl_thumbprint
self.node_group = node_group
self.hpc_client = HpcRestClient(client_cert)
self.heartbeat_table = hpc_cluster_manager.HpcClusterManager(
self.hpc_client, node_group=self.node_group)
self.heartbeat_table.subscribe_node_closed_callback(
lambda l: map(self._kill_task_by_hostname, l))
self.heartbeat_table.start()
self.core_provisioning = 0.0
self.driver = None # type: MesosClient.SchedulerDriver
framework_suffix = self.headnode.replace(',', '_')
if self.node_group != "":
framework_suffix = framework_suffix + '-' + self.node_group
self.mesos_client = MesosClient(
mesos_urls=['http://172.16.1.4:5050'],
# mesos_urls=['zk://127.0.0.1:2181/mesos'],
frameworkName="HPC-Pack-Framework-{}".format(framework_suffix))
self.mesos_client.on(MesosClient.SUBSCRIBED, self.subscribed)
self.mesos_client.on(MesosClient.OFFERS, self.offer_received)
self.mesos_client.on(MesosClient.UPDATE, self.status_update)
self.th = HpcpackFramwork.MesosFramework(self.mesos_client)
self.stop = False
