コード例 #1
0
def main(gcm="", rcm="", out_folder=""):
    pc = None
    pf = None

    kwargs = {
        "start_year": 1970,
        "end_year": 1999,
        "rcm": rcm, "gcm": gcm,
        "out_folder": out_folder
    }
    in_folder = "data/narccap/{0}-{1}/current".format(gcm, rcm)

    if os.path.isdir(in_folder):
        pc = Process(target=interpolate_to_amno, args=(in_folder, ), kwargs=kwargs)
        pc.start()
    else:
        print "{0} does not exist, ignoring the period ...".format(in_folder)

    kwargs = {
        "start_year": 2041,
        "end_year": 2070,
        "rcm": rcm, "gcm": gcm,
        "out_folder": out_folder
    }
    in_folder = "data/narccap/{0}-{1}/future".format(gcm, rcm)
    if os.path.isdir(in_folder):
        pf = Process(target=interpolate_to_amno, args=(in_folder, ), kwargs=kwargs)
        pf.start()
    else:
        print "{0} does not exist, ignoring the period ...".format(in_folder)

    #do current and future climates in parallel
    if pc is not None: pc.join()
    if pf is not None: pf.join()
コード例 #2
0
ファイル: bmanager.py プロジェクト: MatiasBjorling/bisp
    def lookup(self, buildingID):
        from building.dummy import dummy
        '''
        @buildingID: int
        '''
        building = Building.objects.get(bid=buildingID)
        print building

        if building is None:
            return None

        if self.binstances.has_key(buildingID) is False:
            ins = None
            if building.bri == BUILDING_REPRESENTATION_DEBUG:
                ins = dummy.DummyBif(buildingID)
            elif building.bri == BUILDING_REPRESENTATION_IFC:
                raise NotImplementedError, 'Missing implementation of IFC plugin'
            elif building.bri == BUILDING_REPRESENTATION_REVIT:
                raise NotImplementedError, 'Missing implementation of Revit plugin'

            # Fire up the simulator for the building if necessary.
            self.comm_queues[buildingID] = Queue()
            p = Process(target=_Manager__init_simulator, args=(ins, self.comm_queues[buildingID]))
            p.start()

            self.binstances[buildingID] = ins

        return self.binstances[buildingID]
コード例 #3
0
def process_input_dir(args, input_path, output_path):
    patt = input_path + os.sep + "*" + args.extension
    files = glob.glob(patt)
    docs_num = len(files)
    if docs_num > args.threads:
        slice_size = docs_num / args.threads
    else:
        slice_size = 1
    print "Threads:", args.threads
    print "Documents number:", docs_num
    print "Documents per thread:", slice_size

    start = 0
    jobs = []
    for job_num in range(args.threads):
        print "Initializing process", job_num
        end = start + slice_size
        p = Process(target=lemmatize_files, args=(files[start:end], output_path, args))
        print files[start:end]
        jobs.append(p)
        p.start()
    	start += slice_size

    for p in jobs:
        p.join()
    
    if (docs_num % 2) == 1:
        lemmatize_files(files, output_path, args)
コード例 #4
0
ファイル: report_control.py プロジェクト: robertbetts/Kew
def process(host,port):
    from multiprocessing.process import Process
    
    p = Process(target=run, args=(host,port))  
    p.daemon = True
    p.start()     
    return p
コード例 #5
0
    def service_background_jobs(self):
        logger.info('service_background_jobs')
        
        # NOTE: paths must begin with a "/", indicating that the first part of 
        # the URI is a script name (which each app, i.e "reports" serves as).
        # see django.core.handlers.wsgi.__init__
        uri = '/' + '/'.join([SCHEMA.REPORTS_API_URI, JOB.resource_name ])
        data = {
            JOB.STATE: SCHEMA.VOCAB.job.state.PENDING }
        kwargs = {}

        logger.info('get jobs: %r', uri)
        resp = self.api_client.get(
            uri, data=data, **kwargs)
        
        job_listing = self.api_client.deserialize(resp)
        if API_RESULT_DATA in job_listing:
            job_listing = job_listing[API_RESULT_DATA]
        
        for job in job_listing:
            logger.info('found job: %r', job)
            job_id =job[JOB.ID]
            logger.info('Process the job: %r', job_id)
            p = Process(target=self.client_processor.service,args=(job_id,) )
            # make the parent process wait: 
            # p.daemon = True
            # if set to true, then the parent process won't wait.  
            logger.info('start')
            p.start();
            logger.info('started...')
            
        logger.debug('servicing completed')
コード例 #6
0
ファイル: stream.py プロジェクト: YunoHost/moulinette
 def __init__(self, fd, queue):
     assert hasattr(queue, 'put')
     assert hasattr(queue, 'empty')
     assert callable(fd.readline)
     Process.__init__(self)
     self._fd = fd
     self._queue = queue
コード例 #7
0
ファイル: stream.py プロジェクト: YunoHost/moulinette
def async_file_reading(fd, callback):
    """Helper which instantiate and run an AsynchronousFileReader."""
    queue = SimpleQueue()
    reader = AsynchronousFileReader(fd, queue)
    reader.start()
    consummer = Process(target=consume_queue, args=(queue, callback))
    consummer.start()
    return (reader, consummer)
コード例 #8
0
    def __init__(self, accessLevel, eightApi):
        self.accessLevel = accessLevel
        self._eightApi = eightApi

        pygame.init()
        self.check()
        thread = Process(target=self.inputHandler)
        thread.start()
コード例 #9
0
ファイル: objects.py プロジェクト: jonathanverner/qidle
 def __init__(self, pipe, object_class, *args, **kwargs):
     Process.__init__(self)
     self.object_class = object_class
     self.init_args = args
     self.init_kwargs = kwargs
     self.pipe = pipe
     self.command_queue = []
     self.message_priorities = {}
コード例 #10
0
 def add_server(self, app_name, service_name, host, port, processor, use_simple_server=True, wait=1,
                use_ssl=False, ca_certs=None, cert=None, key=None):
     self.sd_client.register_endpoint(app_name, service_name, host, port)
     server_process = Process(target=self.__thrift_server,
                              args=(processor, host, port, use_simple_server, use_ssl, ca_certs, cert, key))
     server_process.start()
     time.sleep(wait)
     self.server_processes.append(server_process)
コード例 #11
0
ファイル: SSHServer1.py プロジェクト: sys-git/stb-rebooter
 def __init__(self, sem, port=2022, protocol=ProtocolEnumerator.ECHO, users={}):
     Process.__init__(self, name = "SSHServer")
     self.port = port
     self.users = users
     self.privateKeys = {}
     self.publicKeys = {}
     self.protocol_ = protocol
     self.exitCmd = "exit"
     self.messages = []
     self.sem = sem
コード例 #12
0
ファイル: factory.py プロジェクト: jonathanverner/qidle
 def __init__(self, event_loop = ThreadedEventLoop()):
     Process.__init__(self)
     self.server_pipe, self.client_pipe = Pipe()
     self.server_router = Router(self.server_pipe)
     self.client_router = Router(self.client_pipe)
     self.srv_command_pipe = self.server_router.create()
     self.cli_command_pipe = self.client_router.create(self.srv_command_pipe.id)
     self.children = {}
     self.client_event_loop = event_loop
     self.client_event_loop.register_hook(self.client_router.eventloop_hook)
コード例 #13
0
	def __init__(self, **kwargs):
		"""
		@param init_target: unbound instance method to call at process start
		@param wrap_target: unbound instance method to wrap worker function
		@param term_target: unbound instance method to call at process finish
		"""
		self._init_target = kwargs.pop("init_target", lambda self: None).__get__(self, Process)
		self._wrap_target = kwargs.pop("wrap_target", lambda self: None).__get__(self, Process)
		self._term_target = kwargs.pop("term_target", lambda self: None).__get__(self, Process)
		Process.__init__(self, **kwargs)
コード例 #14
0
ファイル: test_server.py プロジェクト: AndreaCensi/mcdp
def test_mcdpweb_server(dirname):
    port = random.randint(11000, 15000)
    base = 'http://127.0.0.1:%s' % port

    p = Process(target=start_server, args=(dirname, port,))
    p.start()

    print('sleeping')
    time.sleep(5)

    try:
        url_wrong = base + '/not-existing'
        urllib2.urlopen(url_wrong).read()
    except HTTPError:
        pass
    else:
        raise Exception('Expected 404')

    # now run the spider
    tmpdir = tempfile.mkdtemp(prefix='wget-output')
    cwd = '.'
    cmd = ['wget', '-nv', '-P', tmpdir, '-m', base]
#     res = system_cmd_result(
#             cwd, cmd,
#             display_stdout=True,
#             display_stderr=True,
#             raise_on_error=True)
    sub = subprocess.Popen(
                cmd,
                bufsize=0,
                cwd=cwd)
    sub.wait()

    exc = get_exceptions(port)

    if len(exc) == 0:
        msg = 'Expected at least a not-found error'
        raise Exception(msg)

    if not 'not-existing' in exc[0]:
        raise Exception('Could not find 404 error')

    exc = exc[1:]

    if exc:
        msg = 'Execution raised errors:\n\n'
        msg += str("\n---\n".join(exc))
        raise_desc(Exception, msg)

    url_exit = base + '/exit'
    urllib2.urlopen(url_exit).read()

    print('waiting for start_server() process to exit...')
    p.join()
    print('...clean exit')
コード例 #15
0
 def __init__(self):
     self.__actualSpeed = Value('f', 0.0)
     self._targetSpeed = Value('f', 0.0)
     
     self.__actualDirection = Value('f', 0.0)
     self._targetDirection = Value('f', 0.0)
 
     self.__setupMotors()
     
     steering = Process(target=self.steer)
     #steering.daemon = True
     steering.start()
コード例 #16
0
 def __new__(self, type_, tId, qDistributor, q, model):
     if type_==SolverImplType.THREADED:
         thread = threading.Thread(target=SolverWorker, args=[tId, q, qDistributor, model])
         thread.setName("LoopingSolver_thread_%(I)s"%{"I":tId})
         thread.setDaemon(True)
     elif type_==SolverImplType.MULTIPROCESSOR:
         thread = Process(target=SolverWorker, args=[tId, q, qDistributor, model])
         thread.setName("LoopingSolver_process_%(I)s"%{"I":tId})
         thread.setDaemon(True)
     else:
         raise TypeError("Unknown SolverImplType: '%(T)s'."%{"T":type_})
     return thread
コード例 #17
0
ファイル: libraryfetcher.py プロジェクト: atthaboon/RIDE
def import_library_in_another_process(path, args):
    q = Queue(maxsize=1)
    p = Process(target=library_initializer, args=(q, path, args))
    p.start()
    while True:
        try:
            result = q.get(timeout=0.1)
            if isinstance(result, Exception):
                raise ImportError(result)
            return result
        except Empty:
            if not p.is_alive():
                raise ImportError()
コード例 #18
0
ファイル: remote_test.py プロジェクト: PierreBizouard/odemis
 def setUp(self):
     # Use Thread for debug:
     if USE_THREADS:
         self.server = Thread(target=ServerLoop, args=(self.container_name,))
     else:
         self.server = Process(target=ServerLoop, args=(self.container_name,))
     self.server.start()
     
     self.count = 0
     self.data_arrays_sent = 0
     time.sleep(0.1) # give it some time to start
     self.rdaemon = Pyro4.Proxy("PYRO:Pyro.Daemon@./u:"+self.container_name)
     self.comp = self.rdaemon.getObject("mycomp")
コード例 #19
0
ファイル: imagemngr.py プロジェクト: xyuan/shifter
    def __init__(self, config, logger=None, logname='imagemngr'):
        """
        Create an instance of the image manager.
        """
        if logger is None and logname is None:
            self.logger = logging.getLogger(logname)
            log_handler = logging.StreamHandler()
            logfmt = '%(asctime)s [%(name)s] %(levelname)s : %(message)s'
            log_handler.setFormatter(logging.Formatter(logfmt))
            log_handler.setLevel(logging.INFO)
            self.logger.addHandler(log_handler)
        elif logname is not None:
            self.logger = logging.getLogger(logname)
            self.logger.info('ImageMngr using logname %s' % (logname))
        else:
            print "Using upstream logger"
            self.logger = logger
            print logger
            self.logger.info('ImageMngr using upstream logger')

        self.logger.debug('Initializing image manager')
        self.config = config
        if 'Platforms' not in self.config:
            raise NameError('Platforms not defined')
        self.systems = []
        # Time before another pull can be attempted
        self.pullupdatetimeout = 300
        if 'PullUpdateTime' in self.config:
            self.pullupdatetimeout = self.config['PullUpdateTimeout']
        # Max amount of time to allow for a pull
        self.pulltimeout = self.pullupdatetimeout
        # This is not intended to provide security, but just
        # provide a basic check that a session object is correct
        self.magic = 'imagemngrmagic'
        if 'Authentication' not in self.config:
            self.config['Authentication'] = "munge"
        self.auth = Authentication(self.config)
        self.platforms = self.config['Platforms']

        for system in self.config['Platforms']:
            self.systems.append(system)
        # Connect to database
        if 'MongoDBURI' not in self.config:
            raise NameError('MongoDBURI not defined')
        self.workers = WorkerThreads()
        self.status_queue = self.workers.get_updater_queue()
        self.status_proc = Process(target=self.status_thread,
                                   name='StatusThread')
        self.status_proc.start()
        atexit.register(self.shutdown)
        self.mongo_init()
コード例 #20
0
def main():
    
    ''' parse the command line - new up the appl and listen on port '''
    if os.path.isfile("../kew_pe.conf"):
        print ("Loading config file ../kew_pe.conf")
        options.parse_config_file("../kew_pe.conf")
    options.parse_command_line()
        
    logging.basicConfig(level=logging.DEBUG)
    #report_control.process('localhost',8081)
    
    process = Process(target=report_control.run, name="report_control", kwargs={'host':'localhost', 'port':8081})
    process.daemon = True
    process.start()           
コード例 #21
0
ファイル: test_litmus.py プロジェクト: Nonolost/wsgidav
    def test_litmus_with_authentication(self):
        """Run litmus test suite on HTTP with authentification.

        This test passes
        """
        try:
            proc = Process(target=run_wsgidav_server, args=(True, False))
            proc.daemon = True
            proc.start()
            time.sleep(1)

            try:
                self.assertEqual(subprocess.call(["litmus", "http://127.0.0.1:8080/", "tester", "secret"]),
                                 0,
                                 "litmus suite failed: check the log")
            except OSError:
                print "*" * 70
                print "This test requires the litmus test suite."
                print "See http://www.webdav.org/neon/litmus/"
                print "*" * 70
                raise

        finally:
            proc.terminate()
            proc.join()
コード例 #22
0
def start_sched():
    j_logger.info(u'starting job scheduler ...')
    jobs = get_all_jobs()
    for job in jobs:
        j_logger.info(u'starting job %s ' % job.name)
        job.run_status = Value('i', 0) #job的状态值
        try:
            p = Process(target=_inner_job, name=job.name,args=(job,))
            p.start()
            job.process_id = p.pid
            j_logger.info(u'job %s started !' % job.name)
        except Exception as e:
            j_logger.error(u'job %s fail to start,due to [%s]!' % (job.name,e))
    register_signal_notify()
    j_logger.info(u'job scheduler started !')
コード例 #23
0
def start_sched():
    j_logger.info(u'starting job scheduler ...')
    jobs = get_all_jobs()
    for job in jobs:
        j_logger.info(u'starting job %s ' % job.name)
        job.run_status = Value('i', 0)  #job的状态值
        try:
            p = Process(target=_inner_job, name=job.name, args=(job, ))
            p.start()
            job.process_id = p.pid
            j_logger.info(u'job %s started !' % job.name)
        except Exception as e:
            j_logger.error(u'job %s fail to start,due to [%s]!' %
                           (job.name, e))
    register_signal_notify()
    j_logger.info(u'job scheduler started !')
コード例 #24
0
    def setUp(self):
        # Use Thread for debug:
        if USE_THREADS:
            self.server = threading.Thread(target=ServerLoop,
                                           args=(self.container_name, ))
        else:
            self.server = Process(target=ServerLoop,
                                  args=(self.container_name, ))
        self.server.start()

        self.count = 0
        self.data_arrays_sent = 0
        time.sleep(0.1)  # give it some time to start
        self.rdaemon = Pyro4.Proxy("PYRO:Pyro.Daemon@./u:" +
                                   self.container_name)
        self.comp = self.rdaemon.getObject("mycomp")
コード例 #25
0
    def test_wsgi_404(self):
        def run_provider(queue):
            try:
                provider = create_provider()

                app = Application(provider=provider)

                httpd = make_server('', 15486, app,
                                    handler_class=NoLoggingHandler)

                queue.put({"result": 0})

                httpd.serve_forever()
            except Exception as e:
                queue.put({"result": 1, "error_message": str(e)})

        ready_queue = Queue()

        self.server = Process(target=run_provider, args=(ready_queue,))
        self.server.start()

        provider_started = ready_queue.get()

        if provider_started["result"] != 0:
            raise Exception("Error starting Provider process with message"
                            "'{0}'".format(provider_started["error_message"]))

        try:
            urlopen("http://127.0.0.1:15486/invalid-path").read()
        except HTTPError as e:
            self.assertEqual(404, e.code)
コード例 #26
0
def main():
    auth_server = Process(target=run_auth_server)
    auth_server.start()
    print("To test getting an auth token, execute the following curl command:")
    print(
        "curl --ipv4 -v -X POST"
        " -d 'grant_type=client_credentials&client_id=abc&client_secret=xyz' "
        "http://localhost:8080/token"
    )

    def sigint_handler(signal, frame):
        print("Terminating server...")
        auth_server.terminate()
        auth_server.join()

    signal.signal(signal.SIGINT, sigint_handler)
コード例 #27
0
	def run(self):
		self._init_target()
		if self._wrap_target:
			self._target = self._wrap_target(self._target)
		try:
			return Process.run(self)
		finally:
			self._term_target()
コード例 #28
0
ファイル: smp.py プロジェクト: jucabot/polymr
 def _start(self,name,cpu, module_name, class_name, params):
     fn = None
     
     self._processes = []
     self._in_queue = Queue()
     self._out_queue = Queue()
     self._log_queue = Queue()
     
     if name == "mapper":
         fn = q_run_mapper
     elif name == "reducer":
         fn = q_run_reducer
     
     for i in range(cpu):
         process = Process(target=fn,args=(module_name, class_name ,params, self._in_queue, self._out_queue, self._log_queue))
         self._processes.append(process)
         process.start()
コード例 #29
0
ファイル: namespace.py プロジェクト: pxnj63/RIDE
 def set_from_file(self, varfile_path, args):
     q = Queue()
     p = Process(target=set_from_file, args=(q, varfile_path, args))
     p.start()
     p.join()
     there_are_results = False
     while True:
         try:
             results = q.get_nowait()
             there_are_results  = True
             if len(results) == 1:
                 raise DataError(results[0])
             self.set(*results)
         except Empty:
             if not there_are_results:
                 raise DataError('No variables')
             return
コード例 #30
0
    def __enter__(self):
        self.smtp_process_queue = Queue()
        self.smtp_process = Process(
            target=get_otp_mail, args=(self.smtp_process_queue, self.timeout))
        self.smtp_process.start()
        self.port = self.smtp_process_queue.get(True, 5)
        self._do_lintop_config()

        return self
コード例 #31
0
def multiprocess_map(target,params, *args):
    mgr = multiprocessing.Manager()
    dict_threadsafe = mgr.dict()

    jobs = [Process(target=target_wrapper, args=(target,param,index,dict_threadsafe,args)) for index, param in enumerate(params)]
    for job in jobs: job.start()
    for job in jobs: job.join()

    return dict_threadsafe.values()
コード例 #32
0
class FileResultsLogger(ResultsLogger):

    process = None
    queue = None
    logFileName = None

    def __init__(self, args):
        self.queue = Queue()
        if args.no_p2p:
            fileName = '_'.join([args.mem_algo, str(args.mem_size), 'no_p2p'])
        elif args.no_mem:
            fileName = '_'.join(
                [args.p2p_net, 'no_mem', args.p2p_algo,
                 str(args.p2p_size)])
        else:
            fileName = '_'.join([
                args.p2p_net, args.mem_algo,
                str(args.mem_size), args.p2p_algo,
                str(args.p2p_size)
            ])
        self.logFileName = 'logs/' + fileName + '.logs'

    def logRequest(self, nodeId, address, latency, cacheLevel=-1):
        ResultsLogger.logRequest(self,
                                 nodeId,
                                 address,
                                 latency,
                                 cacheLevel=cacheLevel)
        self.queue.put("{}\t{}\t{}\t{}\n".format(nodeId, address, latency,
                                                 cacheLevel))

    def finish(self):
        self.queue.put("STOP")
        self.process.join()

    def writerProcess(self):
        fileObject = open(self.logFileName, 'w+')
        for logEntry in iter(self.queue.get, "STOP"):
            fileObject.write(logEntry)
        fileObject.close()

    def start(self):
        self.process = Process(target=self.writerProcess)
        self.process.start()
コード例 #33
0
 def load_models(self, pmodels):
     """assumes the pmodels length is equivalent to the number of desired processes."""
     if self.use_lock:
         lock = Lock()
     else:
         lock = None
     processes = [
         Process(target=self.loader, args=(pm, lock)) for pm in pmodels
     ]
     self.run(processes)
コード例 #34
0
ファイル: SSH.py プロジェクト: sys-git/YATES
    def __init__(self, username, password, host, cmdsToSend, port = 22, exitCmd = "exit", timeout = None):
        Process.__init__(self, name = "SSHClient")

        self.logger = LogManager().getLogger('SSHClient-%s' % host)
        self.username = username
        self.password = password
        self.host = host
        self.port = int(port)
        self.cmdsToSend = cmdsToSend if isinstance(cmdsToSend, list) else [cmdsToSend]
        self.exitCmd = exitCmd

        self.queue = Queue()
        self.msg = ""
        self.status = Status.FAILURE
        self.startTime = Value('d', 0.0)
        self.endTime = Value('d', 0.0)
        self.timeout = timeout or SSHClient.TIMEOUT
        self.cmdsSend = False
        self.start()
コード例 #35
0
    def forecast_demands(self):
        """ See :meth:`StatisticalForecast.forecast_demands`.
        This method uses processes, to speed up the calculation. 
        This drives the cpu to full load for a short time."""
        cached = StatisticalForecast.read_from_cache(self)
        if cached != None:
            return cached

        split_results = [[] for i in range(7)]
        #multi processed forecasting
        ## WARNING: hangups:
        #v.1 :  pool.map
        #v.2  use multiple processes instead of pool.map to circumvent a hangup caused
        # by a multiprocessing/django incompabatility
        #share results in a multiaccess queue
        #note: this queue can not hold unlimited elements and will hang up with no warning if there are too many elements
        #12 weeks 1hourly will work, 20 not
        #result_queue = multiprocessing.Queue()
        #v.3 now uses a shared multiprocessing dict, to circumvent hang up problems with queue on windows

        mgr = multiprocessing.Manager()
        dict_threadsafe = mgr.dict()
        logger.debug("forecasting demands with daytype strategy..")

        #call class as Functor because class methods are not pickeable
        jobs = [
            Process(target=self, args=(demand, index, dict_threadsafe))
            for index, demand in enumerate(self.demands)
        ]
        for job in jobs:
            job.start()
        for job in jobs:
            job.join()

        for index in dict_threadsafe.keys():
            split_results[index] = dict_threadsafe[index]

        forecasted_demands = []
        self.calculated_parameters = []
        for fc_tuple in split_results:
            forecasted_demands.append(list(fc_tuple[0]))
            self.calculated_parameters.append(fc_tuple[1])

        #cache forecasts
        pickle.dump(
            {
                "forecasts": forecasted_demands,
                "parameters": self.calculated_parameters,
                "date": self.env.now
            },
            open(os.path.join(BASE_DIR, "cache", "cached_forecasts.cache"),
                 "wb"))
        logger.debug("forecasting completed")

        return forecasted_demands
コード例 #36
0
 def run_rule(self, rule, params=None):
     try:
         log.info('trying job %s' % rule.name)
         job_settings = self.settings.copy()
         if params:
             job_settings.update(params)
         name = rule.qualified_name
         args = (name, job_settings)
         Process(target=run_rule_async, args=args).start()
     except Exception:
         log.error('Error running rule: %s' % rule.name)
コード例 #37
0
    def startNode(self, requests, nodesIds, idx, factory, resultsLogger,
                  knownHosts):
        nodeName = nodesIds[idx]
        nodeRequests = requests[nodeName]
        client = SimulatorClientProcess(factory.port, nodeName, idx,
                                        resultsLogger, knownHosts)

        process = Process(target=client.startNode,
                          args=(factory, nodeRequests))

        self.clients[nodeName] = process
コード例 #38
0
 def __init__(self, name=None):
     if name is None: name = 'default'
     if name in _workers.keys():
         return
     _workers[name] = self
     super(Worker, self).__init__()
     self.daemon = True
     self.name = name
     self._queue = _ver.queue.Queue(1)
     self.last_exception = None
     self._pon = _mp.Value('b', True)
     tsk, self.task = _mp.Pipe(False)
     self.out, res = _mp.Pipe(False)
     self.process = Process(target=process,
                            args=(self._pon, tsk, res),
                            name=name)
     self.process.start()
     self._on = True
     self.start()
     _time.sleep(1)
コード例 #39
0
 def load(self):
     indices_groups = array_split(self.indices, self.procs)
     if self.use_lock:
         lock = Lock()
     else:
         lock = None
     processes = [
         Process(target=self.loader,
                 args=(indices_group, self.f, self.fkwds, self.Model, lock))
         for indices_group in indices_groups
     ]
     self.run(processes)
コード例 #40
0
    def setUp(self):
        #setup game server to run on a seperate process
        self.game_server = Process(target=start_server)
        self.game_server.start()

        #create the game client
        self.client = GameClient(host="127.0.0.1", port="5000")

        self.player_one = "Arthur"
        self.player_one_suspect = game_state.PEACOCK
        self.player_two = "Steven"
        self.player_two_suspect = game_state.PLUM
コード例 #41
0
ファイル: mongoredis.py プロジェクト: pnegahdar/MongoRedis
    def start(self):
        """
        Starts the background process that prunes expired items
        """

        def task():
            while True:
                self.prune_expired()
                pytime.sleep(.5)

        self.processs = Process(target=task)
        self.processs.start()
コード例 #42
0
    def test_tornado(self):
        def run_provider(queue):
            try:
                provider = create_provider()

                app = TornadoApplication([
                    url(r"/authorize", OAuth2Handler, dict(provider=provider)),
                    url(r"/token", OAuth2Handler, dict(provider=provider))
                ],
                                         debug=True)
                app.listen(15486)

                queue.put({"result": 0})

                IOLoop.current().start()
            except Exception as e:
                queue.put({"result": 1, "error_message": str(e)})

        ready_queue = Queue()

        self.server = Process(target=run_provider, args=(ready_queue, ))
        self.server.start()

        provider_started = ready_queue.get()

        if provider_started["result"] != 0:
            raise Exception("Error starting Provider process with message"
                            "'{0}'".format(provider_started["error_message"]))

        self.client = Process(target=run_client, args=(ready_queue, ))
        self.client.start()

        client_started = ready_queue.get()

        if client_started["result"] != 0:
            raise Exception("Error starting Client Application process with "
                            "message '{0}'".format(
                                client_started["error_message"]))

        self.access_token()
コード例 #43
0
    def test_wsgi(self):
        def run_provider(queue):
            try:
                provider = create_provider()

                app = Application(provider=provider)

                httpd = make_server('',
                                    15486,
                                    app,
                                    handler_class=NoLoggingHandler)

                queue.put({"result": 0})

                httpd.serve_forever()
            except Exception as e:
                queue.put({"result": 1, "error_message": str(e)})

        ready_queue = Queue()

        self.server = Process(target=run_provider, args=(ready_queue, ))
        self.server.start()

        provider_started = ready_queue.get()

        if provider_started["result"] != 0:
            raise Exception("Error starting Provider process with message"
                            "'{0}'".format(provider_started["error_message"]))

        self.client = Process(target=run_client, args=(ready_queue, ))
        self.client.start()

        client_started = ready_queue.get()

        if client_started["result"] != 0:
            raise Exception("Error starting Client Application process with "
                            "message '{0}'".format(
                                client_started["error_message"]))

        self.access_token()
コード例 #44
0
ファイル: chord.py プロジェクト: tomusdrw/mgr.p2p.proxy
class ChordNode:

    process = None
    node = None

    def createNode(self, nodeId, port, cacheStorage, knownHosts):
        BlockingChordNode(nodeId, knownHosts, port, cacheStorage)

    def __init__(self, knownHosts, port=4000, cacheStorage=None):
        # TODO id?
        nodeId = random.randrange(NETWORK_SIZE)

        self.process = Process(target=self.createNode,
                               args=(nodeId, port, cacheStorage, knownHosts))
        self.process.start()
        self.node = ServerProxy(('', port), nodeId, ('localhost', port))

    def terminate(self):
        """
        Kill node process
        """
        self.process.terminate()
コード例 #45
0
def cli(config):
    _configure_logging()

    config = yaml.load(config)

    for plugin_config in config['plugins']:
        plugin_config['redis'] = config['redis']
        _launch(plugin_config['type'], plugin_config, False)

    sleep(3)  # wait for plugins to subscribe to queues

    from triviabot.daemon import Daemon
    core_config = config['core']
    core_config['redis'] = config['redis']
    daemon = Daemon(core_config)

    p = Process(target=daemon.run)
    p.daemon = True
    p.run()

    while True:
        sleep(1)
コード例 #46
0
 def __init__(self, settings):
     logging.debug("Initializing Router")
     self.dbhost = settings['DBHOST']
     self.poll = float(settings['POLLINTERVAL'])
     self.agent = settings['JOBSURL']
     user = settings['RTRUSER']
     mapfile = settings['MAPFILE']
     key = None
     if 'RTRKEY' in settings:
         key = settings['RTRKEY']
     self.map = self._load_mapfile(mapfile)
     self.vyos = vyos_interface.vyosInterface(user, key=key)
     self.ddns = self._init_dns(settings)
     self.shutdown_file = settings.get('SHUTFILE', "/tmp/shutdown_sdn")
     self.collection = settings.get('COLLECTION', 'routes')
     self.cleanup_proc = Process(target=self.cleanup, name='CleanupThread')
     self.cleanup_proc.start()
     client = MongoClient(self.dbhost)
     self.routes = client.sdn[self.collection]
     if self.routes.find_one() is None:
         self.cleanup_proc.terminate()
         raise OSError('DB not initialized')
コード例 #47
0
def main():
    auth_server = Process(target=run_auth_server)
    auth_server.start()
    app_server = Process(target=run_app_server)
    app_server.start()
    print("Access http://localhost:8081/app in your browser")

    def sigint_handler(signal, frame):
        print("Terminating servers...")
        auth_server.terminate()
        auth_server.join()
        app_server.terminate()
        app_server.join()

    signal.signal(signal.SIGINT, sigint_handler)
コード例 #48
0
 def run(self):
     items = get_file_list(self.path_list)     #取得所有文件列表,读取文件交给了map_workers
     self.file_list = items
     self.total_count = len(items)
     item_step_count = self.total_count / self.map_worker_count  #每个worker分得的任务数量
     print u'文件总数:%d' % self.total_count
     
     for i in range(self.map_worker_count):
         name = 'pre_worker@%d' % (i + 1)
         p = Process(target=add_worker,args=(name,self.task_queue,self.result_queue))
         self.worker_process_list.append(p)
     
     self.start_time = time.clock()
     self.last_refresh_time = time.clock()
     print u'开始时间:',datetime.now()
     for p in self.worker_process_list:
         p.start()
     
     t_reader = threading.Thread(target=self.put_task_thread)
     t_writer = threading.Thread(target=self.get_result_thread)
     t_reader.start()
     t_writer.start()
     print '------------------------'
コード例 #49
0
ファイル: utils.py プロジェクト: Hamid3010/robotasks
def get_stream(hdf5_file, which_set, batch_size=None):
    dataset = TrajectoryDataset(which_sets=(which_set, ))
    if batch_size == None:
        batch_size = dataset.num_examples
    data_stream = DataStream(dataset=dataset,
                             iteration_scheme=ShuffledScheme(
                                 examples=dataset.num_examples,
                                 batch_size=batch_size))

    load_in_memory = os.path.getsize(
        hdf5_file) < 14 * 10**9 or which_set == 'test'
    if not load_in_memory:
        port = 5557 if which_set == 'train' else 5558
        print port
        server_process = Process(target=start_server,
                                 args=(data_stream, port, 10))
        server_process.start()
        data_stream = ServerDataStream(dataset.sources,
                                       False,
                                       host='localhost',
                                       port=port,
                                       hwm=10)

    return data_stream
コード例 #50
0
ファイル: test_server.py プロジェクト: rusi/mcdp
def test_mcdpweb_server(dirname):
    port = random.randint(11000, 15000)
    base = 'http://127.0.0.1:%s' % port

    p = Process(target=start_server, args=(
        dirname,
        port,
    ))
    p.start()

    print('sleeping')
    time.sleep(5)

    try:
        url_wrong = base + '/not-existing'
        urllib2.urlopen(url_wrong).read()
    except HTTPError:
        pass
    else:
        raise Exception('Expected 404')

    # now run the spider
    tmpdir = tempfile.mkdtemp(prefix='wget-output')
    cwd = '.'
    cmd = ['wget', '-nv', '-P', tmpdir, '-m', base]
    #     res = system_cmd_result(
    #             cwd, cmd,
    #             display_stdout=True,
    #             display_stderr=True,
    #             raise_on_error=True)
    sub = subprocess.Popen(cmd, bufsize=0, cwd=cwd)
    sub.wait()

    exc = get_exceptions(port)

    if len(exc) == 0:
        msg = 'Expected at least a not-found error'
        raise Exception(msg)

    if not 'not-existing' in exc[0]:
        raise Exception('Could not find 404 error')

    exc = exc[1:]

    if exc:
        msg = 'Execution raised errors:\n\n'
        msg += str("\n---\n".join(exc))
        raise_desc(Exception, msg)

    url_exit = base + '/exit'
    urllib2.urlopen(url_exit).read()

    print('waiting for start_server() process to exit...')
    p.join()
    print('...clean exit')
コード例 #51
0
def _launch(type_name, config, is_important):
    plugin_clazz = _get_type(type_name)
    if not plugin_clazz:
        logging.warn('could not find %s plugin' % type_name)

    d = plugin_clazz(config)

    p = Process(target=d.start)
    p.daemon = not is_important
    p.name = 'plugin: %s' % d.name
    p.start()
コード例 #52
0
ファイル: stat.py プロジェクト: netconstructor/OpenClimateGIS
 def calculate(self, funcs):
     ## always count the data
     base_funcs = [{'function': len, 'name': 'count_agg', 'raw': False}]
     ## check if there are raw calculation functions
     if any([f['raw'] for f in funcs]):
         has_raw = True
         if self.sub.value_set == {}:
             raise (ValueError(
                 'Raw aggregate statistics requested with no "value_set_coll"!!'
             ))
     else:
         has_raw = False
     ## need to count the raw data values if raw value are present
     if has_raw:
         base_funcs.append({
             'function': len,
             'name': 'count_raw',
             'raw': True
         })
     ## append the rest of the functions
     funcs = base_funcs + funcs
     ## check the function definition dictionary for common problems
     check_function_dictionary(funcs)
     ## convert the time vector for faster referencing
     time_conv = [[getattr(time, grp) for grp in self.time_grouping]
                  for time in self.sub.timevec]
     ## the shared list
     all_attrs = Manager().list()
     if self.procs > 1:
         processes = [
             Process(target=self.f_calculate,
                     args=(all_attrs, self.sub, groups, funcs, time_conv,
                           self.time_grouping, self.grouping, has_raw))
             for groups in self.groups
         ]
         pmanager = ProcessManager(processes, self.procs)
         pmanager.run()
     else:
         self.f_calculate(all_attrs, self.sub, self.groups[0], funcs,
                          time_conv, self.time_grouping, self.grouping,
                          has_raw)
     self.stats = merge_dict_list(list(all_attrs))
コード例 #53
0
    def calculate(self, funcs):
        ## always count the data
        funcs = [{'function': len, 'name': 'count'}] + funcs
        ## check the function definition dictionary for common problems
        check_function_dictionary(funcs)
        ## convert the time vector for faster referencing
        time_conv = [[getattr(time, grp) for grp in self.time_grouping]
                     for time in self.sub.timevec]
        ## the shared list
        all_attrs = Manager().list()
        if self.procs > 1:
            processes = [
                Process(target=self.f_calculate,
                        args=(all_attrs, self.sub, groups, funcs, time_conv,
                              self.time_grouping, self.grouping))
                for groups in self.groups
            ]
            self.run_parallel(processes)
        else:
            self.f_calculate(all_attrs, self.sub, self.groups[0], funcs,
                             time_conv, self.time_grouping, self.grouping)

        return (all_attrs)
コード例 #54
0
 def test_cleanup(self):
     """
     Test the cleanup thread
     """
     # We want the cleanup thread to run in this context
     # but it is an infitite loop.  So let's delay a shtudown.
     sd2 = "/tmp/shut2"
     rec = {
         'address': '1.2.3.4',
         'ip': '10.128.0.1',
         'router': 'router',
         'last_associated': time(),
         'end_time': 0,
         'uid': 501,
         'user': '******',
         'jobid': '1233',
         'status': 'used'
     }
     self.db.routes2.insert(rec)
     # We want to shutdown the thread that is started on init
     with open(sd2, 'w') as f:
         f.write('1')
     settings = self.settings.copy()
     settings['COLLECTION'] = 'routes2'
     settings['SHUTFILE'] = sd2
     rt = router.Router(settings)
     # Wait for the init thread to shutdown
     sleep(0.2)
     # Now let's start our own
     if os.path.exists(sd2):
         os.remove(sd2)
     shut = Process(target=self._shutdown)
     shut.start()
     rt.cleanup()
     shut.terminate()
     r = self.db.routes2.find_one({'address': '1.2.3.4'})
     self.assertEquals(r['status'], 'available')
     self.db.routes2.remove({})
     rv = rt.cleanup()
     self.assertEquals(-1, rv)
コード例 #55
0
def start_example_app_process3():
    # XXX DO NOT FORGET TO KILL THE PROCESS IF THE TEST DOES NOT SUCCEED
    p = Process(target=start_example_server3)
    p.start()
    return p
コード例 #56
0
        'density_configs': ["sgde_zero"],  #density_configs,
        "function_configs": [("two_moons", 2), ("mult_beta", 2)]
    }
    processes = []
    for estimationMethod in scenarions['density_configs']:
        for functionName, numDims in scenarions["function_configs"]:
            print("-" * 80)
            print("scenario: %s, d=%i, %s" %
                  (functionName, numDims, estimationMethod))
            print("-" * 80)

            if args.parallel:
                myargs = (functionName, estimationMethod, 10, numDims, 20000,
                          "join", "ml", args.out, args.plot)
                processes.append(
                    Process(target=run_densityEstimation, args=myargs))
            else:
                run_densityEstimation(functionName,
                                      estimationMethod,
                                      kfold=10,
                                      numDims=numDims,
                                      numSamples=20000,
                                      candidates="join",
                                      bandwidthOptimizationType="ml",
                                      out=args.out,
                                      plot=args.plot)

    # run applications in parallel if there are any available
    for process in processes:
        process.start()
コード例 #57
0
def action_import_apks(storage, apk_paths,
                       copy_apk = False, copy_to_mongodb = False,
                       update = False, tag = None,
                       # shared memory
                       cnt_imported_apks = None, total_apk_count = None, import_finished = None,
                       # concurrent settings
                       concurrency = None
                       ):

    ''' Import the apks from the `apk_paths` and create the file system structure
    where the results will be kept, specified by `storage`.

    Parameters
    ----------
    storage : RedundantStorage
        The store to use.
    apk_paths : iterable<str>
        The apk files and/or directories.
    copy_apk : bool
        Import the apk file to the `import_dir` (copy it).
    copy_to_mongodb : bool, optional (default is False)
        Also import into MongoDB. Useful for the distributed analysis.
    update : bool
        Update apks that have already been imported.
    tag : str, optional (default is None)
        Some tag
    cnt_imported_apks : multiprocessing.Value<int>, optional (default is None)
        If given, use for progress updating.
    total_apk_count : multiprocessing.Value<int>, optional (default is None)
        If given, use for total count of apks.
    import_finished : multiprocessing.Value<byte>, optional (default is None)
        If given, use to signal that import has been completed.
    concurrency : int, optional (default is number of cpus)
        Number of processes to use for the import.
    '''
    from androlyze.loader.ApkImporter import ApkImporter

    # get single paths to apks so we get the correct total count of apks
    clilog.info("looking for apks in given paths ... ")
    apk_paths = ApkImporter.get_apks_from_list_or_dir(apk_paths)

    if total_apk_count is not None:
        # may be time consuming for recursive lookup
        apk_paths, total_apk_count.value = Util.count_iterable_n_clone(apk_paths)

    # create count if not given
    if cnt_imported_apks is None:
        cnt_imported_apks = Value('i', 0, lock = RLock())

    # set concurrency
    if concurrency is None:
        concurrency = cpu_count()
    log.warn("Using %d processes", concurrency)

    clilog.info("Storage dir is %s" % storage.fs_storage.store_root_dir)
    if copy_apk:
        clilog.info("Copying APKs to %s ..." % storage.fs_storage.store_root_dir)

    def import_apks(apk_paths):
        apk_importer = ApkImporter(apk_paths, storage)
        for apk in apk_importer.import_apks(copy_apk = copy_apk, copy_to_mongodb = copy_to_mongodb,
                                                update = update, tag = tag):

            clilog.info("imported %s", apk.short_description())

            # use shared memory counter if given
            if cnt_imported_apks is not None:
                with cnt_imported_apks.get_lock():
                    cnt_imported_apks.value += 1

    pool = []


    # don't convert generator to list if only 1 process wanted
    apk_paths = [apk_paths] if concurrency == 1 else Util.split_n_uniform_distri(list(apk_paths), concurrency)

    # start parallel import
    # multiprocessing's pool causes pickle errors
    for i in range(concurrency):
        p = Process(target = import_apks, args = (apk_paths[i], ))
        log.debug("starting process %s", p)
        pool.append(p)
        p.start()

    for it in pool:
        log.debug("joined on process %s", p)
        it.join()

    apks_imported = cnt_imported_apks.value != 0
    # show some message that no APK has been imported
    if not apks_imported:
        log.warn("No .apk file has been imported! This means no .apk file has been found or they already have been imported.")
    else:
        clilog.info("done")

    # because not all apks may be importable, we cannot use we count for signal that the import is done
    if import_finished is not None:
        import_finished.value = 1

    clilog.info("Imported %d apks", cnt_imported_apks.value)
コード例 #58
0
ファイル: imagemngr.py プロジェクト: psykok/shifter
class ImageMngr(object):
    """
    This class handles most of the backend work for the image gateway.
    It uses a Mongo Database to track state, uses threads to dispatch work,
    and has public functions to lookup, pull and expire images.
    """
    def __init__(self, config, logger=None, logname='imagemngr'):
        """
        Create an instance of the image manager.
        """
        if logger is None and logname is None:
            self.logger = logging.getLogger(logname)
            log_handler = logging.StreamHandler()
            logfmt = '%(asctime)s [%(name)s] %(levelname)s : %(message)s'
            log_handler.setFormatter(logging.Formatter(logfmt))
            log_handler.setLevel(logging.INFO)
            self.logger.addHandler(log_handler)
        elif logname is not None:
            self.logger = logging.getLogger(logname)
            self.logger.info('ImageMngr using logname %s' % (logname))
        else:
            print "Using upstream logger"
            self.logger = logger
            print logger
            self.logger.info('ImageMngr using upstream logger')

        self.logger.debug('Initializing image manager')
        self.config = config
        if 'Platforms' not in self.config:
            raise NameError('Platforms not defined')
        self.systems = []
        # Time before another pull can be attempted
        self.pullupdatetimeout = 300
        if 'PullUpdateTime' in self.config:
            self.pullupdatetimeout = self.config['PullUpdateTimeout']
        # Max amount of time to allow for a pull
        self.pulltimeout = self.pullupdatetimeout
        # This is not intended to provide security, but just
        # provide a basic check that a session object is correct
        self.magic = 'imagemngrmagic'
        if 'Authentication' not in self.config:
            self.config['Authentication'] = "munge"
        self.auth = Authentication(self.config)
        self.platforms = self.config['Platforms']

        for system in self.config['Platforms']:
            self.systems.append(system)
        # Connect to database
        if 'MongoDBURI' not in self.config:
            raise NameError('MongoDBURI not defined')
        self.workers = WorkerThreads()
        self.status_queue = self.workers.get_updater_queue()
        self.status_proc = Process(target=self.status_thread,
                                   name='StatusThread')
        self.status_proc.start()
        atexit.register(self.shutdown)
        self.mongo_init()

    def shutdown(self):
        self.status_queue.put('stop')

    def mongo_init(self):
        client = MongoClient(self.config['MongoDBURI'])
        db_ = self.config['MongoDB']
        self.images = client[db_].images
        self.metrics = None
        if 'Metrics' in self.config and self.config['Metrics'] is True:
            self.metrics = client[db_].metrics

    def status_thread(self):
        """
        This listens for update messages from a queue.
        """
        self.mongo_init()
        while True:
            message = self.status_queue.get()
            if message == 'stop':
                self.logger.info("Shutting down Status Thread")
                break
            ident = message['id']
            state = message['state']
            meta = message['meta']
            # TODO: Handle a failed expire
            if state == "FAILURE":
                self.logger.warn("Operation failed for %s", ident)

            # print "Status: %s" % (state)
            # A response message
            if state != 'READY':
                self.update_mongo_state(ident, state, meta)
                continue
            if 'response' in meta and meta['response']:
                response = meta['response']
                self.logger.debug(response)
                if 'meta_only' in response:
                    self.logger.debug('Updating ACLs')
                    self.update_acls(ident, response)
                else:
                    self.complete_pull(ident, response)
                self.logger.debug('meta=%s', str(response))

    def check_session(self, session, system=None):
        """Check if this is a valid session
        session is a session handle
        """
        if 'magic' not in session:
            self.logger.warn("request recieved with no magic")
            return False
        elif session['magic'] is not self.magic:
            self.logger.warn("request received with bad magic %s",
                             session['magic'])
            return False
        if system is not None and session['system'] != system:
            self.logger.warn("request received with a bad system %s!=%s",
                             session['system'], system)
            return False
        return True

    def _isadmin(self, session, system=None):
        """
        Check if this is an admin user.
        Returns true if is an admin or false if not.
        """
        if 'admins' not in self.platforms[system]:
            return False
        admins = self.platforms[system]['admins']
        user = session['user']
        if user in admins:
            self.logger.info('user %s is an admin', user)
            return True
        return False

    def _isasystem(self, system):
        """Check if system is a valid platform."""
        return bool(system in self.systems)

    def _get_groups(self, uid, gid):
        """Look up auxilary groups. """
        proc = Popen(['id', '-G', '%d' % (uid)], stdout=PIPE, stderr=PIPE)
        if proc is None:
            self.logger.warn("Group lookup failed")
            return []
        stdout, stderr = proc.communicate()
        groups = []
        for group in stdout.split():
            groups.append(int(group))
        return groups

    def _checkread(self, session, rec):
        """
        Checks if the user has read permissions to the image.
        """

        # Start by checking if the image is public (no ACLs)
        if 'private' in rec and rec['private'] is False:
            return True
        iUACL = None
        iGACL = None
        if 'userACL' in rec:
            iUACL = rec['userACL']
        if 'groupACL' in rec:
            iGACL = rec['groupACL']
        if iUACL is None and iGACL is None:
            return True
        if iUACL == [] and iGACL == []:
            return True
        uid = session['uid']
        gid = session['gid']
        self.logger.debug('uid=%s iUACL=%s' % (uid, str(iUACL)))
        self.logger.debug('sessions = ' + str(session))
        groups = self._get_groups(uid, gid)
        if iUACL is not None and uid in iUACL:
            return True
        if iGACL is not None and gid in iGACL:
            return True
        for group in groups:
            if iGACL is not None and group in iGACL:
                return True
        return False

    def _resetexpire(self, ident):
        """Reset the expire time.  (Not fully implemented)."""
        # Change expire time for image
        # TODO shore up expire-time parsing
        expire_timeout = self.config['ImageExpirationTimeout']
        (days, hours, minutes, secs) = expire_timeout.split(':')
        expire = time() + int(secs) + 60 * (int(minutes) + 60 *
                                            (int(hours) + 24 * int(days)))
        self._images_update({'_id': ident}, {'$set': {'expiration': expire}})
        return expire

    def _make_acl(self, acllist, id):
        if id not in acllist:
            acllist.append(id)
        return acllist

    def _compare_list(self, a, b, key):
        """"
        look at the key element of two objects
        and compare the list of ids.

        return True if everything matches
        return False if anything is different
        """

        # If the key isn't in the objects or
        # something else fails, then it must
        # have changed.
        try:
            if key not in a:
                return False
            if key not in b:
                return False
        except:
            return True
        aitems = a[key]
        bitems = b[key]
        if len(aitems) != len(bitems):
            return False
        for item in aitems:
            if item not in bitems:
                return False
        return True

    def _add_metrics(self, session, request, record):
        """
        Add a row to mongo for this lookup request.
        """
        try:
            r = {
                'user': session['user'],
                'uid': session['uid'],
                'system': request['system'],
                'type': request['itype'],
                'tag': request['tag'],
                'id': record['id'],
                'time': time()
            }
            self._metrics_insert(r)
        except:
            self.logger.warn('Failed to log lookup.')

    def get_metrics(self, session, system, limit):
        """
        Return the last <limit> lookup records.
        """
        recs = []
        if not self._isadmin(session, system):
            return recs
        if self.metrics is None:
            return recs
        count = self.metrics.count()
        skip = count - limit
        if skip < 0:
            skip = 0
        for r in self.metrics.find().skip(skip):
            r.pop('_id', None)
            recs.append(r)
        return recs

    def new_session(self, auth_string, system):
        """
        Creates a session context that can be used for multiple transactions.
        auth is an auth string that will be passed to the authenication layer.
        Returns a context that can be used for subsequent operations.
        """
        if auth_string is None:
            return {'magic': self.magic, 'system': system}
        arec = self.auth.authenticate(auth_string, system)
        if arec is None and isinstance(arec, dict):
            raise OSError("Authenication returned None")
        else:
            if 'user' not in arec:
                raise OSError("Authentication returned invalid response")
            session = arec
            session['magic'] = self.magic
            session['system'] = system
            return session

    def lookup(self, session, image):
        """
        Lookup an image.
        Image is dictionary with system,itype and tag defined.
        """
        if not self.check_session(session, image['system']):
            raise OSError("Invalid Session")
        query = {
            'status': 'READY',
            'system': image['system'],
            'itype': image['itype'],
            'tag': {
                '$in': [image['tag']]
            }
        }
        self.update_states()
        rec = self._images_find_one(query)
        if rec is not None:
            if self._checkread(session, rec) is False:
                return None
            self._resetexpire(rec['_id'])

        if self.metrics is not None:
            self._add_metrics(session, image, rec)
        return rec

    def imglist(self, session, system):
        """
        list images for a system.
        Image is dictionary with system defined.
        """
        if not self.check_session(session, system):
            raise OSError("Invalid Session")
        if self._isasystem(system) is False:
            raise OSError("Invalid System")
        query = {'status': 'READY', 'system': system}
        self.update_states()
        records = self._images_find(query)
        resp = []
        for record in records:
            if self._checkread(session, record):
                resp.append(record)
        # verify access
        return resp

    def show_queue(self, session, system):
        """
        list queue for a system.
        Image is dictionary with system defined.
        """
        if not self.check_session(session, system):
            raise OSError("Invalid Session")
        query = {'status': {'$ne': 'READY'}, 'system': system}
        self.update_states()
        records = self._images_find(query)
        resp = []
        for record in records:
            resp.append({
                'status': record['status'],
                'image': record['pulltag']
            })
        return resp

    def _isready(self, image):
        """Helper function to determine if an image is READY."""
        query = {
            'status': 'READY',
            'system': image['system'],
            'itype': image['itype'],
            'tag': {
                '$in': [image['tag']]
            }
        }
        rec = self._images_find_one(query)
        if rec is not None:
            return True
        return False

    def _pullable(self, rec):
        """
        An image is pullable when:
        -There is no existing record
        -The status is a FAILURE
        -The status is READY and it is past the update time
        -The state is something else and the pull has expired
        """

        # if rec is None then do a pull
        if rec is None:
            return True

        # Okay there has been a pull before
        # If the status flag is missing just repull (shouldn't happen)
        if 'status' not in rec:
            return True
        status = rec['status']

        # EXPIRED images can be pulled
        if status == 'EXPIRED':
            return True

        # Need to deal with last_pull for a READY record
        if 'last_pull' not in rec:
            return True
        nextpull = self.pullupdatetimeout + rec['last_pull']

        # It has been a while, so re-pull to see if it is fresh
        if status == 'READY' and (time() > nextpull):
            return True

        # Repull failed pulls
        if status == 'FAILURE' and (time() > nextpull):
            return True

        # Last thing... What if the pull somehow got hung or died in the middle
        # See if heartbeat is old
        # TODO: add pull timeout.  For now use 1 hour
        if status != 'READY' and 'last_heartbeat' in rec:
            if (time() - rec['last_heartbeat']) > 3600:
                return True

        return False

    def new_pull_record(self, image):
        """
        Creates a new image in mongo.  If the pull already exist it removes
        it first.
        """
        # Clean out any existing records
        for rec in self._images_find(image):
            if rec['status'] == 'READY':
                continue
            else:
                self._images_remove({'_id': rec['_id']})
        newimage = {
            'format': 'invalid',  # <ext4|squashfs|vfs>
            'arch': 'amd64',  # <amd64|...>
            'os': 'linux',  # <linux|...>
            'location': '',  # urlencoded location
            'remotetype': 'dockerv2',  # <file|dockerv2|amazonec2>
            'ostcount': '0',  # integer, number of OSTs (future)
            'replication': '1',  # integer, number of copies to deploy
            'userACL': [],
            'groupACL': [],
            'private': None,
            'tag': [],
            'status': 'INIT'
        }
        if 'DefaultImageFormat' in self.config:
            newimage['format'] = self.config['DefaultImageFormat']
        for param in image:
            if param is 'tag':
                continue
            newimage[param] = image[param]
        self._images_insert(newimage)
        return newimage

    def pull(self, session, image, testmode=0):
        """
        pull the image
        Takes an auth token, a request object
        Optional: testmode={0,1,2} See below...
        """
        request = {
            'system': image['system'],
            'itype': image['itype'],
            'pulltag': image['tag']
        }
        self.logger.debug('Pull called Test Mode=%d', testmode)
        if not self.check_session(session, request['system']):
            self.logger.warn('Invalid session on system %s', request['system'])
            raise OSError("Invalid Session")
        # If a pull request exist for this tag
        #  check to see if it is expired or a failure, if so remove it
        # otherwise
        #  return the record
        rec = None
        # find any pull record
        self.update_states()
        # let's lookup the active image
        query = {
            'status': 'READY',
            'system': image['system'],
            'itype': image['itype'],
            'tag': {
                '$in': [image['tag']]
            }
        }
        rec = self._images_find_one(query)
        for record in self._images_find(request):
            status = record['status']
            if status == 'READY' or status == 'SUCCESS':
                continue
            rec = record
            break
        inflight = False
        recent = False
        if rec is not None and rec['status'] != 'READY':
            inflight = True
        elif rec is not None:
            # if an image has been pulled in the last 60 seconds
            # let's consider that "recent"
            if (time() - rec['last_pull']) < 10:
                recent = True
        request['userACL'] = []
        request['groupACL'] = []
        if 'userACL' in image and image['userACL'] != []:
            request['userACL'] = self._make_acl(image['userACL'],
                                                session['uid'])
        if 'groupACL' in image and image['groupACL'] != []:
            request['groupACL'] = self._make_acl(image['groupACL'],
                                                 session['gid'])
        if self._compare_list(request, rec, 'userACL') and \
                self._compare_list(request, rec, 'groupACL'):
            acl_changed = False
        else:
            self.logger.debug("No ACL change detected.")
            acl_changed = True

        # We could hit a key error or some other edge case
        # so just do our best and update if there are problems
        update = False
        if not recent and not inflight and acl_changed:
            self.logger.debug("ACL change detected.")
            update = True

        if self._pullable(rec):
            self.logger.debug("Pullable image")
            update = True

        if update:
            self.logger.debug("Creating New Pull Record")
            rec = self.new_pull_record(request)
            ident = rec['_id']
            self.logger.debug("ENQUEUEING Request")
            self.update_mongo_state(ident, 'ENQUEUED')
            request['tag'] = request['pulltag']
            request['session'] = session
            self.logger.debug("Calling do pull with queue=%s",
                              request['system'])
            self.workers.dopull(ident, request, testmode=testmode)

            memo = "pull request queued s=%s t=%s" \
                % (request['system'], request['tag'])
            self.logger.info(memo)

            self.update_mongo(ident, {'last_pull': time()})

        return rec

    def mngrimport(self, session, image, testmode=0):
        """
        import the image directly from a file
        Only for allowed users
        Takes an auth token, a request object
        """
        meta = {}
        fp = image['filepath']
        request = {
            'system': image['system'],
            'itype': image['itype'],
            'pulltag': image['tag'],
            'filepath': image['filepath'],
            'format': image['format'],
            'meta': meta
        }
        self.logger.debug('mngrmport called for file %s' % (fp))
        # self.logger.debug(image)
        if not self.check_session(session, request['system']):
            self.logger.warn('Invalid session on system %s', request['system'])
            raise OSError("Invalid Session")
        # Skip checks about previous requests for now
        # Future work could check the fasthash and
        # not import if they're the same
        q = {
            'system': image['system'],
            'itype': image['itype'],
            'pulltag': image['tag']
        }
        rec = self._images_find_one(q)
        if not self._pullable(rec):
            return rec
        # rec = None
        # request['userACL'] = []
        # request['groupACL'] = []
        # if 'userACL' in image and image['userACL'] != []:
        #     request['userACL'] = self._make_acl(image['userACL'],
        #                                         session['uid'])
        # if 'groupACL' in image and image['groupACL'] != []:
        #     request['groupACL'] = self._make_acl(image['groupACL'],
        #                                          session['gid'])
        # if self._compare_list(request, rec, 'userACL') and \
        #         self._compare_list(request, rec, 'groupACL'):
        #     acl_changed = False
        # else:
        #     self.logger.debug("No ACL change detected.")
        #     acl_changed = True

        # We could hit a key error or some other edge case
        # so just do our best and update if there are problems

        self.logger.debug("Creating New Import Record")
        # new_pull_record works for import too
        rec = self.new_pull_record(request)
        ident = rec['_id']
        self.logger.debug("ENQUEUEING Request, ident %s" % (ident))
        self.update_mongo_state(ident, 'ENQUEUED')
        request['tag'] = request['pulltag']
        request['session'] = session
        self.logger.debug("Calling wrkimport with queue=%s", request['system'])
        self.workers.dowrkimport(ident, request, testmode=testmode)

        memo = "import request queued s=%s t=%s" \
            % (request['system'], request['tag'])
        self.logger.info(memo)

        self.update_mongo(ident, {'last_pull': time()})

        return rec

    def update_mongo_state(self, ident, state, info=None):
        """
        Helper function to set the mongo state for an image with _id==ident
        to state=state.
        """
        if state == 'SUCCESS':
            state = 'READY'
        set_list = {'status': state, 'status_message': ''}
        if info is not None and isinstance(info, dict):
            if 'heartbeat' in info:
                set_list['last_heartbeat'] = info['heartbeat']
            if 'message' in info:
                set_list['status_message'] = info['message']
        self._images_update({'_id': ident}, {'$set': set_list})

    def add_tag(self, ident, system, tag):
        """
        Helper function to add a tag to an image.
        ident is the mongo id (not image id)
        """
        # Remove the tag first
        self.remove_tag(system, tag)
        # see if tag isn't a list
        rec = self._images_find_one({'_id': ident})
        if rec is not None and 'tag' in rec and \
                not isinstance(rec['tag'], (list)):
            memo = 'Fixing tag for non-list %s %s' % (ident, str(rec['tag']))
            self.logger.info(memo)
            curtag = rec['tag']
            self._images_update({'_id': ident}, {'$set': {'tag': [curtag]}})
        self._images_update({'_id': ident}, {'$addToSet': {'tag': tag}})
        return True

    def remove_tag(self, system, tag):
        """
        Helper function to remove a tag to an image.
        """
        self._images_update({
            'system': system,
            'tag': {
                '$in': [tag]
            }
        }, {'$pull': {
            'tag': tag
        }},
                            multi=True)
        return True

    def update_acls(self, ident, response):
        self.logger.debug("Update ACLs called for %s %s", ident, str(response))
        pullrec = self._images_find_one({'_id': ident})
        if pullrec is None:
            self.logger.error('ERROR: Missing pull request (r=%s)',
                              str(response))
            return
        # Check that this image ident doesn't already exist for this system
        rec = self._images_find_one({
            'id': response['id'],
            'status': 'READY',
            'system': pullrec['system']
        })
        if rec is None:
            # This means the image already existed, but we didn't have a
            # record of it.  That seems odd (it happens in tests).  Let's
            # note it and power on through.
            msg = "WARNING: No image record found for an ACL update"
            self.logger.warn(msg)
            response['last_pull'] = time()
            response['status'] = 'READY'
            self.update_mongo(ident, response)
            self.add_tag(ident, pullrec['system'], pullrec['pulltag'])
        else:
            updates = {
                'userACL': response['userACL'],
                'groupACL': response['groupACL'],
                'private': response['private'],
                'last_pull': time()
            }
            self.logger.debug("Doing ACLs update")
            response['last_pull'] = time()
            response['status'] = 'READY'
            self.update_mongo(rec['_id'], updates)
            self._images_remove({'_id': ident})

    def complete_pull(self, ident, response):
        """
        Transition a completed pull request to an available image.
        """

        self.logger.debug("Complete called for %s %s", ident, str(response))
        pullrec = self._images_find_one({'_id': ident})
        if pullrec is None:
            self.logger.warn('Missing pull request (r=%s)', str(response))
            return
        # Check that this image ident doesn't already exist for this system
        rec = self._images_find_one({
            'id': response['id'],
            'system': pullrec['system'],
            'status': 'READY'
        })
        tag = pullrec['pulltag']
        if rec is not None:
            # So we already had this image.
            # Let's delete the pull record.
            # TODO: update the pull time of the matching id
            self.logger.warn('Duplicate image')
            update_rec = {'last_pull': time()}
            self.update_mongo(rec['_id'], update_rec)

            self._images_remove({'_id': ident})
            # However it could be a new tag.  So let's update the tag
            try:
                rec['tag'].index(response['tag'])
            except:
                self.add_tag(rec['_id'], pullrec['system'], tag)
            return True
        else:
            response['last_pull'] = time()
            response['status'] = 'READY'
            self.update_mongo(ident, response)
            self.add_tag(ident, pullrec['system'], tag)

    def update_mongo(self, ident, resp):
        """
        Helper function to set the mongo values for an image with _id==ident.
        """
        setline = dict()
        # This maps from the key name in the response to the
        # key name used in mongo
        mappings = {
            'id': 'id',
            'entrypoint': 'ENTRY',
            'env': 'ENV',
            'workdir': 'WORKDIR',
            'last_pull': 'last_pull',
            'userACL': 'userACL',
            'groupACL': 'groupACL',
            'private': 'private',
            'status': 'status'
        }
        if 'private' in resp and resp['private'] is False:
            resp['userACL'] = []
            resp['groupACL'] = []

        for key in mappings.keys():
            if key in resp:
                setline[mappings[key]] = resp[key]

        self._images_update({'_id': ident}, {'$set': setline})

    def get_state(self, ident):
        """
        Lookup the state of the image with _id==ident in Mongo.
        Returns the state.
        """
        self.update_states()
        rec = self._images_find_one({'_id': ident}, {'status': 1})
        if rec is None:
            return None
        elif 'status' not in rec:
            return None
        return rec['status']

    def update_states(self):
        """
        Cleanup failed transcations after a period
        """
        for rec in self._images_find({'status': 'FAILURE'}):
            nextpull = self.pullupdatetimeout + rec['last_pull']
            # It it has been a while then let's clean up
            if time() > nextpull:
                self._images_remove({'_id': rec['_id']})

    def autoexpire(self, session, system, testmode=0):
        """Auto expire images and do cleanup"""
        # While this should be safe, let's restrict this to admins
        if not self._isadmin(session, system):
            return False
        # Cleanup - Lookup for things stuck in non-READY state
        self.update_states()
        removed = []
        for rec in self._images_find({
                'status': {
                    '$ne': 'READY'
                },
                'system': system
        }):
            if 'last_pull' not in rec:
                self.logger.warning('Image missing last_pull for pulltag:' +
                                    rec['pulltag'])
                continue
            if time() > rec['last_pull'] + self.pulltimeout:
                removed.append(rec['_id'])
                self._images_remove({'_id': rec['_id']})

        expired = []
        # Look for READY images that haven't been pulled recently
        for rec in self._images_find({'status': 'READY', 'system': system}):
            if 'expiration' not in rec:
                continue
            elif rec['expiration'] < time():
                self.logger.debug("expiring %s", rec['id'])
                ident = rec.pop('_id')
                self.expire_id(rec, ident)
                if 'id' in rec:
                    expired.append(rec['id'])
                else:
                    expired.append('unknown')
            self.logger.debug(rec['expiration'] > time())
        return expired

    def expire_id(self, rec, ident, testmode=0):
        """ Helper function to expire by id """
        memo = "Calling do expire id=%s TM=%d" \
            % (ident, testmode)
        self.logger.debug(memo)

        self.workers.doexpire(ident, rec)
        self.logger.info("expire request queued s=%s t=%s", rec['system'],
                         ident)

    def expire(self, session, image, testmode=0):
        """Expire an image.  (Not Implemented)"""
        if not self._isadmin(session, image['system']):
            return False
        query = {
            'system': image['system'],
            'itype': image['itype'],
            'tag': {
                '$in': [image['tag']]
            }
        }
        rec = self._images_find_one(query)
        if rec is None:
            return None
        ident = rec.pop('_id')
        memo = "Calling do expire with queue=%s id=%s TM=%d" \
            % (image['system'], ident, testmode)
        self.logger.debug(memo)
        self.workers.doexpire(ident, rec)

        memo = "expire request queued s=%s t=%s" \
            % (image['system'], image['tag'])
        self.logger.info(memo)

        return True

    @mongo_reconnect_reattempt
    def _images_remove(self, *args, **kwargs):
        """ Decorated function to remove images from mongo """
        return self.images.remove(*args, **kwargs)

    @mongo_reconnect_reattempt
    def _images_update(self, *args, **kwargs):
        """ Decorated function to updates images in mongo """
        return self.images.update(*args, **kwargs)

    @mongo_reconnect_reattempt
    def _images_find(self, *args, **kwargs):
        """ Decorated function to find images in mongo """
        return self.images.find(*args, **kwargs)

    @mongo_reconnect_reattempt
    def _images_find_one(self, *args, **kwargs):
        """ Decorated function to find one image in mongo """
        return self.images.find_one(*args, **kwargs)

    @mongo_reconnect_reattempt
    def _images_insert(self, *args, **kwargs):
        """ Decorated function to insert an image in mongo """
        return self.images.insert(*args, **kwargs)

    @mongo_reconnect_reattempt
    def _metrics_insert(self, *args, **kwargs):
        """ Decorated function to insert an image in mongo """
        if self.metrics is not None:
            return self.metrics.insert(*args, **kwargs)
コード例 #59
0
ファイル: longevity.py プロジェクト: umang-cb/Jython
                toBeAdded.append(server)
            if len(toBeAdded) == how_many:
                break

        for server in toBeAdded:
            rest.add_node('Administrator', 'password', server.ip)
            #check if its added ?
        nodes = rest.node_statuses()
        otpNodes = [node.id for node in nodes]
        started = rest.rebalance(otpNodes, [])
        msg = "rebalance operation started ? {0}"
        self.log.info(msg.format(started))
        if started:
            result = rest.monitorRebalance()
            msg = "successfully rebalanced out selected nodes from the cluster ? {0}"
            self.log.info(msg.format(result))
            return result
        return False


if __name__ == "__main__":
    process1 = Process(target=start_load, args=(sys.argv,))
    process1.start()
    process2 = Process(target=start_combo, args=(sys.argv,))
    process2.start()
    process3 = Process(target=start_backup, args=(sys.argv,))
    process3.start()
    process1.join()
    process2.join()
    process3.join()