def mq_server(): beanstalk = beanstalkc.Connection( host=config.get('beanstalkd_interface', 'localhost'), port=int(config.get('beanstalkd_port', 11300))) beanstalk.watch('taobao_stream') beanstalk.ignore('default') while True: try: job = beanstalk.reserve() except: import traceback exc = traceback.format_exc() _logger.error(exc) time.sleep(1 / 1000) continue try: name, args, kwds = cPickle.loads(job.body) func = NAME2FUNC.get(name) except: import traceback exc = traceback.format_exc() _logger.error(exc) job.delete() continue try: func(*args, **kwds) job.delete() except TOPException: import traceback exc = traceback.format_exc() _logger.error(exc) job.delete() except osv.except_osv: import traceback exc = traceback.format_exc() _logger.error(exc) job.delete() except TransactionRollbackError: #TransactionRollbackError: 错误: 由于同步更新而无法串行访问 job.release(delay=1) except DataError: #DataError: 错误: 无效的 "UTF8" 编码字节顺序: 0xad import traceback exc = traceback.format_exc() _logger.error(exc) job.delete() except: import traceback exc = traceback.format_exc() _logger.error(exc) job.release(delay=1) finally: time.sleep(1 / 1000)
def __init__(self, config_file): #super(DNNManager, self).__init__(parent=parent) host = platform.node() CONFIG = json.load(open(config_file, 'r')) self.config = CONFIG['universal'] if host in CONFIG['machine_specific']: local = CONFIG['machine_specific'][host] for k in local.keys(): self.config[k] = local[k] targ = self.config['beanstalk_host'] port = self.config['beanstalk_port'] if not targ.upper() == host.upper(): print( f'\rERROR!!! {config_file} indicates that manager should be run on {targ.upper()} not {host.upper()}' ) sys.exit(-1) else: # FORK the beanstalk daemon, if not already running if self.__is_port_in_use__(port): print( f'\rERROR!!! It appears that port {port} is already in use!!!' ) sys.exit(-1) else: ps = subprocess.Popen(['beanstalkd', '-V', '-p', f'{port}']) self.bs_conn = BSC.Connection( host=self.config['beanstalk_host'], port=self.config['beanstalk_port'])
def mq_server(): print 'mq_server on port %s' % MQ_PORT beanstalk = beanstalkc.Connection(host='localhost', port=MQ_PORT, parse_yaml=True) beanstalk.watch(MQ_USE) beanstalk.ignore('default') while True: job = beanstalk.reserve() #print job try: name, args, kwds = loads(job.body) except: job.delete() continue func = NAME2FUNC.get(name) #print name, args, kwds try: func(*args, **kwds) except: import traceback exc = traceback.format_exc() cursor = cursor_by_table('failed_mq') cursor.execute( 'insert into failed_mq (body,exc,func) values (%s,%s,%s)', (job.body, exc, name)) logging.error(exc) job.delete()
def _open(self): conninfo = self.connection.client host = conninfo.hostname or 'localhost' port = conninfo.port or DEFAULT_PORT conn = beanstalkc.Connection(host=host, port=port) conn.connect() return conn
def __init__(self, config, **kwargs): """Create a new Worker. This uses the config dictionary to connect to the queue and worker. That dictionary must contain a cache dictionary with the keys connection to indicate the connection to use, timeout for the time in seconds until an object is expired in the cache. :config: A configuration dictionary. This must specify the cache and queue to use. :kwargs: Keyword arguments to set as attributes of this worker. If no name is given then a random UUID is generated as a name. """ self.config = dict(config) self.cache = redis.Redis(**config['cache']['connection']) self.timeout = config['cache']['timeout'] self.persist = set(config['cache'].get('persist', [])) self.beanstalk = beanstalkc.Connection(**config['queue']['connection']) self.beanstalk.watch(config['queue']['name']) self.beanstalk.ignore('default') self.name = kwargs.get('name', str(uuid.uuid4())) self.logger = logging.getLogger('queue.Worker:%s' % self.name) for key, value in kwargs.items(): setattr(self, key, value)
def __init__(self, config_path, config, notifier, websocket_notifier, authentificator, health): app = bottle.app() app.config.load_config(config_path) engine = database.engine() plugin = sabottle.Plugin( engine, None, keyword='db', use_kwargs=True, create=False, ) app.install(plugin) self._check_for_index_html(app) conn = beanstalkc.Connection(host=config.get("general", "beanstalk_host"), port=11300) conn.use('deployer-deployments') app.config["deployer.engine"] = engine app.config["deployer.beanstalk"] = conn app.config["deployer.notifier"] = notifier app.config["deployer.websocket_notifier"] = websocket_notifier app.config["deployer.bcrypt_log_rounds"] = 12 app.config["deployer.authentificator"] = authentificator app.config["health"] = health self.httpd = make_server("0.0.0.0", config.getint('general', 'api_port'), app, server_class=ThreadingWSGIServer, handler_class=LoggingWSGIRequestHandler)
def __init__(self, gpus, config_file): hostname = platform.node() ngpus = count_gpus() gpu_val = validate_gpus(gpus, ngpus) if not gpu_val[0]: print( f'\rERROR! Only found {ngpu} GPUs [0..{ngpus-1}], cannot assign GPU {gpu_val[1]}' ) sys.exit(-1) else: gpustr = ",".join(str(x) for x in gpus) os.environ['CUDA_VISIBLE_DEVICES'] = gpustr self.gpu_info = (hostname, gpus, ngpus) self.my_id = f'{hostname}-gpu{gpustr}' CONFIG = json.load(open(config_file, 'r')) self.config = CONFIG['universal'] local = CONFIG['machine_specific'][hostname] for k in local.keys(): self.config[k] = local[k] self.current_jid = None self.bs_conn = BSC.Connection(host=self.config['beanstalk_host'], port=self.config['beanstalk_port'])
def remove_beanstalk_jobs(run_name, tube_name): qhost = config.queue_host qport = config.queue_port if qhost is None or qport is None: raise RuntimeError( 'Beanstalk queue information not found in {conf_path}'.format( conf_path=config.teuthology_yaml)) log.info("Checking Beanstalk Queue...") beanstalk = beanstalkc.Connection(host=qhost, port=qport) beanstalk.watch(tube_name) beanstalk.ignore('default') curjobs = beanstalk.stats_tube(tube_name)['current-jobs-ready'] if curjobs != 0: x = 1 while x != curjobs: x += 1 job = beanstalk.reserve(timeout=20) if job is None: continue job_config = yaml.safe_load(job.body) if run_name == job_config['name']: job_id = job.stats()['id'] msg = "Deleting job from queue. ID: " + \ "{id} Name: {name} Desc: {desc}".format( id=str(job_id), name=job_config['name'], desc=job_config['description'], ) log.info(msg) job.delete() else: print "No jobs in Beanstalk Queue" beanstalk.close()
def beanstalk_connect(self): try: self.beanstalk = beanstalkc.Connection(host=self.beanstalk_host, port=self.beanstalk_port) print 'system: Beanstalkd connected on ' + str(self.beanstalk_host) + ' on port ' + str(self.beanstalk_port) except socket.error, (value,message): print "Fatal: Unable to connect to beanstalkd" raise
def main(): bs_connection = False while True: try: if not bs_connection: bs = beanstalkc.Connection(connect_timeout=2) bs.watch('tosign') bs_connection = True print 'Waiting for jobs in queue tosign' job = bs.reserve() jbody = json.loads(job.body) job.delete() src_dir = reimzul_repo_basedir + jbody['target'] + '/' + jbody[ 'pkgname'] + '/' + jbody['timestamp'] + '/' target_dir = reimzul_tosign_basedir + jbody['target'] + '/' print "Copying RPM pkgs from %s to %s" % (src_dir, target_dir) copy_cmd = "test -d %s || mkdir -p %s ; find %s -iname '*.rpm' -exec cp {} %s \;" % ( target_dir, target_dir, src_dir, target_dir) process = subprocess.call(copy_cmd, shell=True) except beanstalkc.SocketError: bs_connection = False time.sleep(2) continue
def init_route_beanstalk_dict(): route_beanstalk_dict = {} for tube_name in [HOOK_TUBE_NAME]: beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST, port=BEANSTALK_PORT) EventManager.switch(beanstalk, tube_name) route_beanstalk_dict[tube_name] = beanstalk return route_beanstalk_dict
def connect(): host = config.queue_host port = config.queue_port if host is None or port is None: raise RuntimeError( 'Beanstalk queue information not found in {conf_path}'.format( conf_path=config.teuthology_yaml)) return beanstalkc.Connection(host=host, port=port)
def _get_stats(self): stats = {} try: connection = beanstalkc.Connection(self.config['host'], self.config['port']) except beanstalkc.BeanstalkcException, e: self.log.error("Couldn't connect to beanstalkd: %s", e) return {}
def worker(): beanstalk = beanstalkc.Connection(host='127.0.0.1', port=11300) while True: job = beanstalk.reserve() mid = int(job.body) print 'Got job:', mid work(mid) job.delete()
def send_message(msg, user): # username = user.username peer = user b = beanstalkc.Connection() b.use(settings.POLLS_TUBE) tube_message = json.dumps({'message': str(msg), 'username': peer}) b.put(tube_message) b.close()
def rebeanstalk(): global beanstalk if not beanstalk: error_log('Re-establishing connection to beanstalkd') beanstalk = beanstalkc.Connection(host=BSHOST, port=BSPORT) beanstalk.watch(BSMSBOXQ) beanstalk.use(BSMSBOXQ) beanstalk.ignore('default')
def __init__(self, tube, worker_id, host="localhost"): threading.Thread.__init__(self) self.beanstalk = beanstalkc.Connection(host=host) self.beanstalk.connect() self.tube = tube # self.beanstalk.watch(tube) self.worker_id = worker_id self.watchTube(tube)
def monitor_polls(self, polls=None): b = beanstalkc.Connection() b.watch(settings.POLLS_TUBE) while True: job = b.reserve() msg = json.loads(job.body) job.bury() self.message_new(msg)
def get_job(): beanstalk = beanstalkc.Connection(host='localhost', port=11300) while True: job = beanstalk.reserve() if job: print str(os.getpid()) + ': ' + job.body time.sleep(.1) job.delete()
def start_operation(self): global _RENUM try: bean = beanstalkc.Connection('192.168.8.55', 11300) bean.watch('test') bean.ignore('default') except: sys.stderr.write('Can not connect to beanstalk server') return while True: try: job_msg = bean.reserve(timeout=60) assert job_msg except: self.thread_stop = True continue try: self.thread_stop = False self.repeat_num = 0 self.job_body = json.loads(job_msg.body, object_hook=self._decode_dict) if self.thread_running is False: self.start_thread_work() if 'method' not in self.job_body: sys.stderr.write('Abort message without method') elif self.job_body['method'] == 'manual': try: engine = QtShot(self.job_body, qtshot, self.write_process_pid) engine.start() engine.join() while self.flags == 1 and self.repeat_num <= _RENUM: self.flags = 0 print 'rerere' if self.thread_running is False: self.start_thread_work() engine = QtShot(self.job_body, qtshot, self.write_process_pid) engine.start() engine.join() if self.repeat_num > _RENUM: print 222222 else: self.handle_pic() except: traceback.print_exc() else: sys.stderr.write('Abort message with method = ' + job_body['method']) except: traceback.print_exc() finally: try: job_msg.delete() except beanstalkc.CommandFailed: pass
def monitor_polls(self): b = beanstalkc.Connection() b.watch(settings.POLLS_TUBE) while True: job = b.reserve() msg = json.loads(job.body) job.bury() logger.info("Got New message") self.message_new(msg)
def __init__(self, db_path, wl, bl): self.bs = beanstalkc.Connection(host='127.0.0.1', port=11300) self.sql = sqlite3.connect(db_path, check_same_thread=False) self.c = self.sql.cursor() self.wl = wl self.bl = bl self.job = None
def __init__(self, host='localhost', port=11300): self.host = host self.port = port self.__watch_tube = [] self.__use_tube = "" self.__conn = beanstalkc.Connection(host, port) self.__use_tube = self.__conn.using() for w in self.__conn.watching(): self.__watch_tube.append(w)
def main(): parser = ArgumentParser() parser.add_argument("-m", "--model", required=True, help="Check this model") args = parser.parse_args() model = args.model modelClass = getModelObj(model) modelLinks = modelClass.getRun() constants = Constants() db = NgwipsDB(constants.dbConnStr) redisConn = redis.Redis(constants.redisHost) beanstalkdConn = beanstalkc.Connection(host=constants.beanstalkdHost, port=11300) resetHourKeys(modelClass, model, redisConn) # redisConn.set(model + '-complete', db.getCurrentRun(model)) currentlyProcessing = redisConn.get(model) # If key is not set in Redis... Set it. if currentlyProcessing is None: redisConn.set(model, "0") print "Currently Processing: " + currentlyProcessing while True: modelClass = getModelObj(model) modelLinks = modelClass.getRun() currentlyProcessing = redisConn.get(model) print "Currently Processing: " + currentlyProcessing if int(currentlyProcessing) == 0: # If it has new hourly data, or it is a new run altogether. Put it in the Queue. if hasNewData(modelLinks, model, modelClass, redisConn) or db.isNewRun(model, modelClass.runTime): # Run has been updated. Put into queue. print "Putting into Queue..." # Set Currently processing # 1 = Currently processing # 0 = Not currently processing redisConn.set(model, "1") print "Currently processing : " + redisConn.get(model) cmd = "python dataGetter.py -b --model=" + model + " --clean" cmdObj = {"model": model, "command": cmd} beanstalkdConn.put(json.dumps(cmdObj)) print "doing nothing... Waiting 5 mins." # Sleep for 3 mins. time.sleep(180) return
def main(): stalk = beanstalkc.Connection() stalk.watch('game-requests') championships = list(Match.objects.filter(root=True)) while not all([x.winner for x in championships]): for championship in championships: maintain_bracket(championship) if stalk.stats_tube('game-requests')['current-jobs-ready'] < 1: generate_speculative_game(random.choice(championships))
def announce(messg, user): messg = str(messg) # username = user.username username = user.get_profile().peer.domain_name b = beanstalkc.Connection() b.use(settings.POLLS_TUBE) tube_message = json.dumps({'message': messg, 'username': username}) b.put(tube_message) b.close()
def _get_connection(): global _connection if not _connection: try: host, port = g.DEFAULT_BEANSTALK.split(':') except TypeError: host, port = DEFAULT_BEANSTALK.split(':') _connection = beanstalkc.Connection(host=host, port=int(port)) return _connection
def main(): stalk = beanstalkc.Connection() if 'game-requests' not in stalk.tubes(): schedule_a_game() while True: if stalk.stats_tube('game-requests')['current-jobs-ready'] < 1: schedule_a_game() sleep(0.2) stalk.close()
def handle(self): proxy_data = self.rfile.readline().strip() data = self.rfile.readline().strip() for trigger, handler in TRIGGERS.items(): if trigger in data: job = handler(proxy_data.split()[2], str(data)) print(job) beanstalk = beanstalkc.Connection(host=BHOST, port=BPORT) beanstalk.put(json.dumps(job))
def monitor_jobs(): # We have to open one socket per Greenlet, as currently socket sharing is # not allowed try: b = beanstalkc.Connection() except Exception, err: logger.error("Error connecting to beanstalkd: %s" % str(err)) sleep(5) return