Python FLAGS 예제들

예제 #1

파일: cluster.py 프로젝트: MaggieQi/spartan

  def _str(self):
    return ','.join(['%s:%d' % (host, count) for host, count in self.val])

class AssignMode(object):
  BY_CORE = 1
  BY_NODE = 2

class AssignModeFlag(config.Flag):
  def parse(self, option_str):
    self.val = getattr(AssignMode, option_str)

  def _str(self):
    if self.val == AssignMode.BY_CORE: return 'BY_CORE'
    return 'BY_NODE'

FLAGS.add(HostListFlag('hosts', default=[('localhost', 8)]))
FLAGS.add(BoolFlag('xterm', default=False, help='Run workers in xterm'))
FLAGS.add(BoolFlag('oprofile', default=False, help='Run workers inside of operf'))
FLAGS.add(AssignModeFlag('assign_mode', default=AssignMode.BY_NODE))
FLAGS.add(BoolFlag('use_single_core', default=True))

FLAGS.add(BoolFlag(
  'use_threads',
  help='When running locally, use threads instead of forking. (slow, for debugging)',
  default=True))
FLAGS.add(IntFlag('heartbeat_interval', default=3, help='Heartbeat Interval in each worker'))
FLAGS.add(IntFlag('worker_failed_heartbeat_threshold', default=10, help='the max number of heartbeat that a worker can delay'))

def start_remote_worker(worker, st, ed):
  '''
  Start processes on a worker machine.

예제 #2

파일: util.py 프로젝트: MaggieQi/spartan

from contextlib import contextmanager
import logging
from math import ceil
import os
from os.path import basename
import socket
import sys
import threading
import time
import traceback
import numpy as np

from spartan.config import FLAGS, BoolFlag


FLAGS.add(BoolFlag('dump_timers', default=False))

HOSTNAME = socket.gethostname()
PID = os.getpid()

LOGGING_CONFIGURED = False

def _setup_logger():
  global LOGGING_CONFIGURED
  if logging.root is None:
    raise Exception, 'Log attempt before logging was configured.'
  
  logging.RootLogger.findCaller = findCaller
  LOGGING_CONFIGURED = True   

def log_debug(*args, **kw):

예제 #3

파일: test_common.py 프로젝트: EasonLiao/spartan

import atexit
import os
from os.path import basename, splitext
import signal
from spartan import util, config
from spartan.cluster import start_cluster
from spartan.config import FLAGS, StrFlag, BoolFlag
import cProfile
import imp
import spartan
import sys
import time
import types
import unittest

FLAGS.add(StrFlag('worker_list', default='4,8,16,32,64,80'))
FLAGS.add(BoolFlag('test_optimizations', default=False))

def millis(t1, t2):
  dt = t2 - t1
  ms = (dt.days * 24 * 60 * 60 + dt.seconds) * 1000 + dt.microseconds / 1000.0
  return ms

def sig_handler(sig, frame):
  import threading
  import sys
  import traceback

  for thread_id, stack in sys._current_frames().items():
    print '-' * 100
    traceback.print_stack(stack)

예제 #4

파일: cluster.py 프로젝트: GabrielWen/spartan

class AssignMode(object):
    BY_CORE = 1
    BY_NODE = 2


class AssignModeFlag(config.Flag):
    def parse(self, option_str):
        self.val = getattr(AssignMode, option_str)

    def _str(self):
        if self.val == AssignMode.BY_CORE:
            return "BY_CORE"
        return "BY_NODE"


FLAGS.add(HostListFlag("hosts", default=[("localhost", 8)]))
FLAGS.add(BoolFlag("xterm", default=False, help="Run workers in xterm"))
FLAGS.add(BoolFlag("oprofile", default=False, help="Run workers inside of operf"))
FLAGS.add(AssignModeFlag("assign_mode", default=AssignMode.BY_NODE))
FLAGS.add(BoolFlag("use_single_core", default=True))

FLAGS.add(
    BoolFlag(
        "use_threads", help="When running locally, use threads instead of forking. (slow, for debugging)", default=True
    )
)
FLAGS.add(IntFlag("heartbeat_interval", default=3, help="Heartbeat Interval in each worker"))
FLAGS.add(
    IntFlag("worker_failed_heartbeat_threshold", default=10, help="the max number of heartbeat that a worker can delay")
)

예제 #5

파일: util.py 프로젝트: rossparks/spartan

import collections
from contextlib import contextmanager
import logging
from math import ceil
import os
from os.path import basename
import socket
import sys
import threading
import time
import traceback
import numpy as np

from spartan.config import FLAGS, BoolFlag

FLAGS.add(BoolFlag('dump_timers', default=False))

HOSTNAME = socket.gethostname()
PID = os.getpid()

LOGGING_CONFIGURED = False


def _setup_logger():
    global LOGGING_CONFIGURED
    if logging.root is None:
        raise Exception, 'Log attempt before logging was configured.'

    logging.RootLogger.findCaller = findCaller
    LOGGING_CONFIGURED = True

예제 #6

파일: local.py 프로젝트: MaggieQi/spartan

  return module._jit_fn


class ParakeetExpr(LocalExpr):
  deps = PythonValue
  source = PythonValue

  def fn_name(self):
    return 'parakeet'

  def pretty_str(self):
    return 'parakeet_op'

  def evaluate(self, ctx):
    names = self.input_names()
    fn = compile_parakeet_source(self.source)

    kw_args = {}
    for var in names:
      value = ctx.inputs[var]
      kw_args[var] = value

    if FLAGS.use_cuda:
      return fn(_backend='cuda', **kw_args)
    else:
      return fn(**kw_args)

from spartan.config import FLAGS, BoolFlag
FLAGS.add(BoolFlag('use_cuda', default=False))

예제 #7

파일: cluster.py 프로젝트: rossparks/spartan

class AssignMode(object):
    BY_CORE = 1
    BY_NODE = 2


class AssignModeFlag(config.Flag):
    def parse(self, option_str):
        self.val = getattr(AssignMode, option_str)

    def _str(self):
        if self.val == AssignMode.BY_CORE: return 'BY_CORE'
        return 'BY_NODE'


FLAGS.add(HostListFlag('hosts', default=[('localhost', 8)]))
FLAGS.add(BoolFlag('xterm', default=False, help='Run workers in xterm'))
FLAGS.add(
    BoolFlag('oprofile', default=False, help='Run workers inside of operf'))
FLAGS.add(AssignModeFlag('assign_mode', default=AssignMode.BY_NODE))
FLAGS.add(BoolFlag('use_single_core', default=True))

FLAGS.add(
    BoolFlag(
        'use_threads',
        help=
        'When running locally, use threads instead of forking. (slow, for debugging)',
        default=True))
FLAGS.add(
    IntFlag('heartbeat_interval',
            default=3,

예제 #8

파일: local.py 프로젝트: muddimedia/spartan-1

    return module._jit_fn


class ParakeetExpr(LocalExpr):
    deps = PythonValue
    source = PythonValue

    def fn_name(self):
        return 'parakeet'

    def pretty_str(self):
        return 'parakeet_op'

    def evaluate(self, ctx):
        names = self.input_names()
        fn = compile_parakeet_source(self.source)

        kw_args = {}
        for var in names:
            value = ctx.inputs[var]
            kw_args[var] = value

        if FLAGS.use_cuda:
            return fn(_backend='cuda', **kw_args)
        else:
            return fn(**kw_args)


from spartan.config import FLAGS, BoolFlag
FLAGS.add(BoolFlag('use_cuda', default=False))

예제 #9

import atexit
import os
from os.path import basename, splitext
import signal
from spartan import util, config
from spartan.cluster import start_cluster
from spartan.config import FLAGS, StrFlag, BoolFlag
import cProfile
import imp
import spartan
import sys
import time
import types
import unittest

FLAGS.add(StrFlag('worker_list', default='4,8,16,32,64,80'))
FLAGS.add(BoolFlag('test_optimizations', default=False))


def millis(t1, t2):
    dt = t2 - t1
    ms = (dt.days * 24 * 60 * 60 +
          dt.seconds) * 1000 + dt.microseconds / 1000.0
    return ms


def sig_handler(sig, frame):
    import threading
    import sys
    import traceback

예제 #10

파일: local.py 프로젝트: GabrielWen/spartan

class ParakeetExpr(LocalExpr):
    deps = PythonValue
    source = PythonValue

    def fn_name(self):
        return "parakeet"

    def pretty_str(self):
        return "parakeet_op"

    def evaluate(self, ctx):
        names = self.input_names()
        fn = compile_parakeet_source(self.source)

        kw_args = {}
        for var in names:
            value = ctx.inputs[var]
            kw_args[var] = value

        if FLAGS.use_cuda:
            return fn(_backend="cuda", **kw_args)
        else:
            return fn(**kw_args)


from spartan.config import FLAGS, BoolFlag

FLAGS.add(BoolFlag("use_cuda", default=False))