import binascii
import random
import socket
import struct
import six.moves.cPickle
import marshal
from dpark.util import compress, decompress, spawn, get_logger
from dpark.cache import Cache
from dpark.serialize import marshalable
from dpark.env import env
import six
from six.moves import map
from six.moves import range
logger = get_logger(__name__)
MARSHAL_TYPE, PICKLE_TYPE = list(range(2))
BLOCK_SHIFT = 20
BLOCK_SIZE = 1 << BLOCK_SHIFT
class BroadcastManager:
header_fmt = '>BI'
header_len = struct.calcsize(header_fmt)
def start(self, is_master):
raise NotImplementedError
def shutdown(self):
raise NotImplementedError
import os
import socket
from cStringIO import StringIO
from consts import *
from master import MasterConn
from cs import read_chunk, read_chunk_from_local
from dpark.util import get_logger
MFS_ROOT_INODE = 1
logger = get_logger(__name__)
class CrossSystemSymlink(Exception):
def __init__(self, src, dst):
self.src = src
self.dst = dst
def __str__(self):
return '%s -> %s' % (self.src, self.dst)
class MooseFS(object):
def __init__(self, host='mfsmaster', port=9421, mountpoint='/mfs'):
self.host = host
self.mountpoint = mountpoint
self.mc = MasterConn(host, port)
self.symlink_cache = {}
def setLogLevel(level):
get_logger('dpark').setLevel(level)
import zmq
from addict import Dict
from pymesos import Executor, MesosExecutorDriver, encode_data, decode_data
import six
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
from dpark.util import compress, decompress, spawn, mkdir_p, get_logger
from dpark.serialize import marshalable
from dpark.accumulator import Accumulator
from dpark.schedule import Success, FetchFailed, OtherFailure
from dpark.env import env
from dpark.shuffle import LocalFileShuffle
from dpark.mutable_dict import MutableDict
from dpark.serialize import loads
logger = get_logger('dpark.executor@%s' % socket.gethostname())
TASK_RESULT_LIMIT = 1024 * 256
DEFAULT_WEB_PORT = 5055
MAX_EXECUTOR_IDLE_TIME = 60 * 60 * 24
KILL_TIME_OUT = 0.1 # 0.1 sec
CLEAN_ZOMBIE_TIME_OUT = 10
Script = ''
def setproctitle(x):
try:
from setproctitle import setproctitle as _setproctitle
_setproctitle(x)
except ImportError:
pass
from addict import Dict
from pymesos import Executor, MesosExecutorDriver, encode_data, decode_data
import six
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
from dpark.util import (compress, decompress, spawn, mkdir_p, get_logger,
init_dpark_logger, formatter_message)
from dpark.serialize import marshalable
from dpark.accumulator import Accumulator
from dpark.schedule import Success, FetchFailed, OtherFailure
from dpark.env import env
from dpark.shuffle import LocalFileShuffle
from dpark.mutable_dict import MutableDict
from dpark.serialize import loads
logger = get_logger('dpark.executor')
TASK_RESULT_LIMIT = 1024 * 256
DEFAULT_WEB_PORT = 5055
MAX_EXECUTOR_IDLE_TIME = 60 * 60 * 24
KILL_TIME_OUT = 0.1 # 0.1 sec
CLEAN_ZOMBIE_TIME_OUT = 10
Script = ''
def setproctitle(x):
try:
from setproctitle import setproctitle as _setproctitle
_setproctitle(x)
except ImportError:
pass
from __future__ import absolute_import
import os
import socket
import shutil
import uuid
import tempfile
import zmq
from dpark import util
import dpark.conf as conf
logger = util.get_logger(__name__)
class TaskStats(object):
def __init__(self):
self._reset()
def _reset(self):
self.bytes_max_rss = 0
self.bytes_shuffle_read = 0
self.bytes_shuffle_write = 0
self.secs_reduce_merge = 0
class DparkEnv:
environ = {}
trackerServer = None
import pymesos as mesos
from mesos.interface import mesos_pb2
from mesos.interface import Executor
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
from dpark.util import compress, decompress, spawn, mkdir_p, get_logger
from dpark.serialize import marshalable
from dpark.accumulator import Accumulator
from dpark.schedule import Success, FetchFailed, OtherFailure
from dpark.env import env
from dpark.shuffle import LocalFileShuffle
from dpark.mutable_dict import MutableDict
from dpark.serialize import loads
from dpark.moosefs import close_mfs
logger = get_logger("dpark.executor@%s" % socket.gethostname())
TASK_RESULT_LIMIT = 1024 * 256
DEFAULT_WEB_PORT = 5055
MAX_EXECUTOR_IDLE_TIME = 60 * 60 * 24
KILL_TIME_OUT = 0.1 # 0.1 sec
Script = ''
def setproctitle(x):
try:
from setproctitle import setproctitle as _setproctitle
_setproctitle(x)
except ImportError:
pass
import os
import socket
import shutil
import uuid
import zmq
from dpark import util
import dpark.conf as conf
logger = util.get_logger(__name__)
class DparkEnv:
environ = {}
@classmethod
def register(cls, name, value):
cls.environ[name] = value
@classmethod
def get(cls, name, default=None):
return cls.environ.get(name, default)
def __init__(self):
self.started = False
def start(self, isMaster, environ={}):
if self.started:
return
logger.debug("start env in %s: %s %s", os.getpid(), isMaster, environ)
import zmq
from addict import Dict
from pymesos import Executor, MesosExecutorDriver, encode_data, decode_data
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
from dpark.util import compress, decompress, spawn, mkdir_p, get_logger
from dpark.serialize import marshalable
from dpark.accumulator import Accumulator
from dpark.schedule import Success, FetchFailed, OtherFailure
from dpark.env import env
from dpark.shuffle import LocalFileShuffle
from dpark.mutable_dict import MutableDict
from dpark.serialize import loads
from dpark.moosefs import close_mfs
logger = get_logger('dpark.executor@%s' % socket.gethostname())
TASK_RESULT_LIMIT = 1024 * 256
DEFAULT_WEB_PORT = 5055
MAX_EXECUTOR_IDLE_TIME = 60 * 60 * 24
KILL_TIME_OUT = 0.1 # 0.1 sec
Script = ''
def setproctitle(x):
try:
from setproctitle import setproctitle as _setproctitle
_setproctitle(x)
except ImportError:
pass
def setLogLevel(level):
get_logger('dpark').setLevel(level)
