import binascii import random import socket import struct import six.moves.cPickle import marshal from dpark.util import compress, decompress, spawn, get_logger from dpark.cache import Cache from dpark.serialize import marshalable from dpark.env import env import six from six.moves import map from six.moves import range logger = get_logger(__name__) MARSHAL_TYPE, PICKLE_TYPE = list(range(2)) BLOCK_SHIFT = 20 BLOCK_SIZE = 1 << BLOCK_SHIFT class BroadcastManager: header_fmt = '>BI' header_len = struct.calcsize(header_fmt) def start(self, is_master): raise NotImplementedError def shutdown(self): raise NotImplementedError
import os import socket from cStringIO import StringIO from consts import * from master import MasterConn from cs import read_chunk, read_chunk_from_local from dpark.util import get_logger MFS_ROOT_INODE = 1 logger = get_logger(__name__) class CrossSystemSymlink(Exception): def __init__(self, src, dst): self.src = src self.dst = dst def __str__(self): return '%s -> %s' % (self.src, self.dst) class MooseFS(object): def __init__(self, host='mfsmaster', port=9421, mountpoint='/mfs'): self.host = host self.mountpoint = mountpoint self.mc = MasterConn(host, port) self.symlink_cache = {}
def setLogLevel(level): get_logger('dpark').setLevel(level)
import zmq from addict import Dict from pymesos import Executor, MesosExecutorDriver, encode_data, decode_data import six sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) from dpark.util import compress, decompress, spawn, mkdir_p, get_logger from dpark.serialize import marshalable from dpark.accumulator import Accumulator from dpark.schedule import Success, FetchFailed, OtherFailure from dpark.env import env from dpark.shuffle import LocalFileShuffle from dpark.mutable_dict import MutableDict from dpark.serialize import loads logger = get_logger('dpark.executor@%s' % socket.gethostname()) TASK_RESULT_LIMIT = 1024 * 256 DEFAULT_WEB_PORT = 5055 MAX_EXECUTOR_IDLE_TIME = 60 * 60 * 24 KILL_TIME_OUT = 0.1 # 0.1 sec CLEAN_ZOMBIE_TIME_OUT = 10 Script = '' def setproctitle(x): try: from setproctitle import setproctitle as _setproctitle _setproctitle(x) except ImportError: pass
from addict import Dict from pymesos import Executor, MesosExecutorDriver, encode_data, decode_data import six sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) from dpark.util import (compress, decompress, spawn, mkdir_p, get_logger, init_dpark_logger, formatter_message) from dpark.serialize import marshalable from dpark.accumulator import Accumulator from dpark.schedule import Success, FetchFailed, OtherFailure from dpark.env import env from dpark.shuffle import LocalFileShuffle from dpark.mutable_dict import MutableDict from dpark.serialize import loads logger = get_logger('dpark.executor') TASK_RESULT_LIMIT = 1024 * 256 DEFAULT_WEB_PORT = 5055 MAX_EXECUTOR_IDLE_TIME = 60 * 60 * 24 KILL_TIME_OUT = 0.1 # 0.1 sec CLEAN_ZOMBIE_TIME_OUT = 10 Script = '' def setproctitle(x): try: from setproctitle import setproctitle as _setproctitle _setproctitle(x) except ImportError: pass
from __future__ import absolute_import import os import socket import shutil import uuid import tempfile import zmq from dpark import util import dpark.conf as conf logger = util.get_logger(__name__) class TaskStats(object): def __init__(self): self._reset() def _reset(self): self.bytes_max_rss = 0 self.bytes_shuffle_read = 0 self.bytes_shuffle_write = 0 self.secs_reduce_merge = 0 class DparkEnv: environ = {} trackerServer = None
import pymesos as mesos from mesos.interface import mesos_pb2 from mesos.interface import Executor sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) from dpark.util import compress, decompress, spawn, mkdir_p, get_logger from dpark.serialize import marshalable from dpark.accumulator import Accumulator from dpark.schedule import Success, FetchFailed, OtherFailure from dpark.env import env from dpark.shuffle import LocalFileShuffle from dpark.mutable_dict import MutableDict from dpark.serialize import loads from dpark.moosefs import close_mfs logger = get_logger("dpark.executor@%s" % socket.gethostname()) TASK_RESULT_LIMIT = 1024 * 256 DEFAULT_WEB_PORT = 5055 MAX_EXECUTOR_IDLE_TIME = 60 * 60 * 24 KILL_TIME_OUT = 0.1 # 0.1 sec Script = '' def setproctitle(x): try: from setproctitle import setproctitle as _setproctitle _setproctitle(x) except ImportError: pass
import os import socket import shutil import uuid import zmq from dpark import util import dpark.conf as conf logger = util.get_logger(__name__) class DparkEnv: environ = {} @classmethod def register(cls, name, value): cls.environ[name] = value @classmethod def get(cls, name, default=None): return cls.environ.get(name, default) def __init__(self): self.started = False def start(self, isMaster, environ={}): if self.started: return logger.debug("start env in %s: %s %s", os.getpid(), isMaster, environ)
import zmq from addict import Dict from pymesos import Executor, MesosExecutorDriver, encode_data, decode_data sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) from dpark.util import compress, decompress, spawn, mkdir_p, get_logger from dpark.serialize import marshalable from dpark.accumulator import Accumulator from dpark.schedule import Success, FetchFailed, OtherFailure from dpark.env import env from dpark.shuffle import LocalFileShuffle from dpark.mutable_dict import MutableDict from dpark.serialize import loads from dpark.moosefs import close_mfs logger = get_logger('dpark.executor@%s' % socket.gethostname()) TASK_RESULT_LIMIT = 1024 * 256 DEFAULT_WEB_PORT = 5055 MAX_EXECUTOR_IDLE_TIME = 60 * 60 * 24 KILL_TIME_OUT = 0.1 # 0.1 sec Script = '' def setproctitle(x): try: from setproctitle import setproctitle as _setproctitle _setproctitle(x) except ImportError: pass