def mq_server():
beanstalk = beanstalkc.Connection(
host=config.get('beanstalkd_interface', 'localhost'),
port=int(config.get('beanstalkd_port', 11300)))
beanstalk.watch('taobao_stream')
beanstalk.ignore('default')
while True:
try:
job = beanstalk.reserve()
except:
import traceback
exc = traceback.format_exc()
_logger.error(exc)
time.sleep(1 / 1000)
continue
try:
name, args, kwds = cPickle.loads(job.body)
func = NAME2FUNC.get(name)
except:
import traceback
exc = traceback.format_exc()
_logger.error(exc)
job.delete()
continue
try:
func(*args, **kwds)
job.delete()
except TOPException:
import traceback
exc = traceback.format_exc()
_logger.error(exc)
job.delete()
except osv.except_osv:
import traceback
exc = traceback.format_exc()
_logger.error(exc)
job.delete()
except TransactionRollbackError: #TransactionRollbackError: 错误: 由于同步更新而无法串行访问
job.release(delay=1)
except DataError: #DataError: 错误: 无效的 "UTF8" 编码字节顺序: 0xad
import traceback
exc = traceback.format_exc()
_logger.error(exc)
job.delete()
except:
import traceback
exc = traceback.format_exc()
_logger.error(exc)
job.release(delay=1)
finally:
time.sleep(1 / 1000)
def __init__(self, config_file):
#super(DNNManager, self).__init__(parent=parent)
host = platform.node()
CONFIG = json.load(open(config_file, 'r'))
self.config = CONFIG['universal']
if host in CONFIG['machine_specific']:
local = CONFIG['machine_specific'][host]
for k in local.keys():
self.config[k] = local[k]
targ = self.config['beanstalk_host']
port = self.config['beanstalk_port']
if not targ.upper() == host.upper():
print(
f'\rERROR!!! {config_file} indicates that manager should be run on {targ.upper()} not {host.upper()}'
)
sys.exit(-1)
else:
# FORK the beanstalk daemon, if not already running
if self.__is_port_in_use__(port):
print(
f'\rERROR!!! It appears that port {port} is already in use!!!'
)
sys.exit(-1)
else:
ps = subprocess.Popen(['beanstalkd', '-V', '-p', f'{port}'])
self.bs_conn = BSC.Connection(
host=self.config['beanstalk_host'],
port=self.config['beanstalk_port'])
def mq_server():
print 'mq_server on port %s' % MQ_PORT
beanstalk = beanstalkc.Connection(host='localhost',
port=MQ_PORT,
parse_yaml=True)
beanstalk.watch(MQ_USE)
beanstalk.ignore('default')
while True:
job = beanstalk.reserve()
#print job
try:
name, args, kwds = loads(job.body)
except:
job.delete()
continue
func = NAME2FUNC.get(name)
#print name, args, kwds
try:
func(*args, **kwds)
except:
import traceback
exc = traceback.format_exc()
cursor = cursor_by_table('failed_mq')
cursor.execute(
'insert into failed_mq (body,exc,func) values (%s,%s,%s)',
(job.body, exc, name))
logging.error(exc)
job.delete()
def _open(self):
conninfo = self.connection.client
host = conninfo.hostname or 'localhost'
port = conninfo.port or DEFAULT_PORT
conn = beanstalkc.Connection(host=host, port=port)
conn.connect()
return conn
def __init__(self, config, **kwargs):
"""Create a new Worker. This uses the config dictionary to connect to
the queue and worker. That dictionary must contain a cache dictionary
with the keys connection to indicate the connection to use, timeout for
the time in seconds until an object is expired in the cache.
:config: A configuration dictionary. This must specify the cache and
queue to use.
:kwargs: Keyword arguments to set as attributes of this worker. If no
name is given then a random UUID is generated as a name.
"""
self.config = dict(config)
self.cache = redis.Redis(**config['cache']['connection'])
self.timeout = config['cache']['timeout']
self.persist = set(config['cache'].get('persist', []))
self.beanstalk = beanstalkc.Connection(**config['queue']['connection'])
self.beanstalk.watch(config['queue']['name'])
self.beanstalk.ignore('default')
self.name = kwargs.get('name', str(uuid.uuid4()))
self.logger = logging.getLogger('queue.Worker:%s' % self.name)
for key, value in kwargs.items():
setattr(self, key, value)
def __init__(self, config_path, config, notifier, websocket_notifier,
authentificator, health):
app = bottle.app()
app.config.load_config(config_path)
engine = database.engine()
plugin = sabottle.Plugin(
engine,
None,
keyword='db',
use_kwargs=True,
create=False,
)
app.install(plugin)
self._check_for_index_html(app)
conn = beanstalkc.Connection(host=config.get("general",
"beanstalk_host"),
port=11300)
conn.use('deployer-deployments')
app.config["deployer.engine"] = engine
app.config["deployer.beanstalk"] = conn
app.config["deployer.notifier"] = notifier
app.config["deployer.websocket_notifier"] = websocket_notifier
app.config["deployer.bcrypt_log_rounds"] = 12
app.config["deployer.authentificator"] = authentificator
app.config["health"] = health
self.httpd = make_server("0.0.0.0",
config.getint('general', 'api_port'),
app,
server_class=ThreadingWSGIServer,
handler_class=LoggingWSGIRequestHandler)
def __init__(self, gpus, config_file):
hostname = platform.node()
ngpus = count_gpus()
gpu_val = validate_gpus(gpus, ngpus)
if not gpu_val[0]:
print(
f'\rERROR! Only found {ngpu} GPUs [0..{ngpus-1}], cannot assign GPU {gpu_val[1]}'
)
sys.exit(-1)
else:
gpustr = ",".join(str(x) for x in gpus)
os.environ['CUDA_VISIBLE_DEVICES'] = gpustr
self.gpu_info = (hostname, gpus, ngpus)
self.my_id = f'{hostname}-gpu{gpustr}'
CONFIG = json.load(open(config_file, 'r'))
self.config = CONFIG['universal']
local = CONFIG['machine_specific'][hostname]
for k in local.keys():
self.config[k] = local[k]
self.current_jid = None
self.bs_conn = BSC.Connection(host=self.config['beanstalk_host'],
port=self.config['beanstalk_port'])
def remove_beanstalk_jobs(run_name, tube_name):
qhost = config.queue_host
qport = config.queue_port
if qhost is None or qport is None:
raise RuntimeError(
'Beanstalk queue information not found in {conf_path}'.format(
conf_path=config.teuthology_yaml))
log.info("Checking Beanstalk Queue...")
beanstalk = beanstalkc.Connection(host=qhost, port=qport)
beanstalk.watch(tube_name)
beanstalk.ignore('default')
curjobs = beanstalk.stats_tube(tube_name)['current-jobs-ready']
if curjobs != 0:
x = 1
while x != curjobs:
x += 1
job = beanstalk.reserve(timeout=20)
if job is None:
continue
job_config = yaml.safe_load(job.body)
if run_name == job_config['name']:
job_id = job.stats()['id']
msg = "Deleting job from queue. ID: " + \
"{id} Name: {name} Desc: {desc}".format(
id=str(job_id),
name=job_config['name'],
desc=job_config['description'],
)
log.info(msg)
job.delete()
else:
print "No jobs in Beanstalk Queue"
beanstalk.close()
def beanstalk_connect(self):
try:
self.beanstalk = beanstalkc.Connection(host=self.beanstalk_host, port=self.beanstalk_port)
print 'system: Beanstalkd connected on ' + str(self.beanstalk_host) + ' on port ' + str(self.beanstalk_port)
except socket.error, (value,message):
print "Fatal: Unable to connect to beanstalkd"
raise
def main():
bs_connection = False
while True:
try:
if not bs_connection:
bs = beanstalkc.Connection(connect_timeout=2)
bs.watch('tosign')
bs_connection = True
print 'Waiting for jobs in queue tosign'
job = bs.reserve()
jbody = json.loads(job.body)
job.delete()
src_dir = reimzul_repo_basedir + jbody['target'] + '/' + jbody[
'pkgname'] + '/' + jbody['timestamp'] + '/'
target_dir = reimzul_tosign_basedir + jbody['target'] + '/'
print "Copying RPM pkgs from %s to %s" % (src_dir, target_dir)
copy_cmd = "test -d %s || mkdir -p %s ; find %s -iname '*.rpm' -exec cp {} %s \;" % (
target_dir, target_dir, src_dir, target_dir)
process = subprocess.call(copy_cmd, shell=True)
except beanstalkc.SocketError:
bs_connection = False
time.sleep(2)
continue
def init_route_beanstalk_dict():
route_beanstalk_dict = {}
for tube_name in [HOOK_TUBE_NAME]:
beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST,
port=BEANSTALK_PORT)
EventManager.switch(beanstalk, tube_name)
route_beanstalk_dict[tube_name] = beanstalk
return route_beanstalk_dict
def connect():
host = config.queue_host
port = config.queue_port
if host is None or port is None:
raise RuntimeError(
'Beanstalk queue information not found in {conf_path}'.format(
conf_path=config.teuthology_yaml))
return beanstalkc.Connection(host=host, port=port)
def _get_stats(self):
stats = {}
try:
connection = beanstalkc.Connection(self.config['host'],
self.config['port'])
except beanstalkc.BeanstalkcException, e:
self.log.error("Couldn't connect to beanstalkd: %s", e)
return {}
def worker():
beanstalk = beanstalkc.Connection(host='127.0.0.1', port=11300)
while True:
job = beanstalk.reserve()
mid = int(job.body)
print 'Got job:', mid
work(mid)
job.delete()
def send_message(msg, user):
# username = user.username
peer = user
b = beanstalkc.Connection()
b.use(settings.POLLS_TUBE)
tube_message = json.dumps({'message': str(msg), 'username': peer})
b.put(tube_message)
b.close()
def rebeanstalk():
global beanstalk
if not beanstalk:
error_log('Re-establishing connection to beanstalkd')
beanstalk = beanstalkc.Connection(host=BSHOST, port=BSPORT)
beanstalk.watch(BSMSBOXQ)
beanstalk.use(BSMSBOXQ)
beanstalk.ignore('default')
def __init__(self, tube, worker_id, host="localhost"):
threading.Thread.__init__(self)
self.beanstalk = beanstalkc.Connection(host=host)
self.beanstalk.connect()
self.tube = tube
# self.beanstalk.watch(tube)
self.worker_id = worker_id
self.watchTube(tube)
def monitor_polls(self, polls=None):
b = beanstalkc.Connection()
b.watch(settings.POLLS_TUBE)
while True:
job = b.reserve()
msg = json.loads(job.body)
job.bury()
self.message_new(msg)
def get_job():
beanstalk = beanstalkc.Connection(host='localhost', port=11300)
while True:
job = beanstalk.reserve()
if job:
print str(os.getpid()) + ': ' + job.body
time.sleep(.1)
job.delete()
def start_operation(self):
global _RENUM
try:
bean = beanstalkc.Connection('192.168.8.55', 11300)
bean.watch('test')
bean.ignore('default')
except:
sys.stderr.write('Can not connect to beanstalk server')
return
while True:
try:
job_msg = bean.reserve(timeout=60)
assert job_msg
except:
self.thread_stop = True
continue
try:
self.thread_stop = False
self.repeat_num = 0
self.job_body = json.loads(job_msg.body,
object_hook=self._decode_dict)
if self.thread_running is False:
self.start_thread_work()
if 'method' not in self.job_body:
sys.stderr.write('Abort message without method')
elif self.job_body['method'] == 'manual':
try:
engine = QtShot(self.job_body, qtshot,
self.write_process_pid)
engine.start()
engine.join()
while self.flags == 1 and self.repeat_num <= _RENUM:
self.flags = 0
print 'rerere'
if self.thread_running is False:
self.start_thread_work()
engine = QtShot(self.job_body, qtshot,
self.write_process_pid)
engine.start()
engine.join()
if self.repeat_num > _RENUM:
print 222222
else:
self.handle_pic()
except:
traceback.print_exc()
else:
sys.stderr.write('Abort message with method = ' +
job_body['method'])
except:
traceback.print_exc()
finally:
try:
job_msg.delete()
except beanstalkc.CommandFailed:
pass
def monitor_polls(self):
b = beanstalkc.Connection()
b.watch(settings.POLLS_TUBE)
while True:
job = b.reserve()
msg = json.loads(job.body)
job.bury()
logger.info("Got New message")
self.message_new(msg)
def __init__(self, db_path, wl, bl):
self.bs = beanstalkc.Connection(host='127.0.0.1', port=11300)
self.sql = sqlite3.connect(db_path, check_same_thread=False)
self.c = self.sql.cursor()
self.wl = wl
self.bl = bl
self.job = None
def __init__(self, host='localhost', port=11300):
self.host = host
self.port = port
self.__watch_tube = []
self.__use_tube = ""
self.__conn = beanstalkc.Connection(host, port)
self.__use_tube = self.__conn.using()
for w in self.__conn.watching():
self.__watch_tube.append(w)
def main():
parser = ArgumentParser()
parser.add_argument("-m",
"--model",
required=True,
help="Check this model")
args = parser.parse_args()
model = args.model
modelClass = getModelObj(model)
modelLinks = modelClass.getRun()
constants = Constants()
db = NgwipsDB(constants.dbConnStr)
redisConn = redis.Redis(constants.redisHost)
beanstalkdConn = beanstalkc.Connection(host=constants.beanstalkdHost,
port=11300)
resetHourKeys(modelClass, model, redisConn)
# redisConn.set(model + '-complete', db.getCurrentRun(model))
currentlyProcessing = redisConn.get(model)
# If key is not set in Redis... Set it.
if currentlyProcessing is None:
redisConn.set(model, "0")
print "Currently Processing: " + currentlyProcessing
while True:
modelClass = getModelObj(model)
modelLinks = modelClass.getRun()
currentlyProcessing = redisConn.get(model)
print "Currently Processing: " + currentlyProcessing
if int(currentlyProcessing) == 0:
# If it has new hourly data, or it is a new run altogether. Put it in the Queue.
if hasNewData(modelLinks, model, modelClass,
redisConn) or db.isNewRun(model, modelClass.runTime):
# Run has been updated. Put into queue.
print "Putting into Queue..."
# Set Currently processing
# 1 = Currently processing
# 0 = Not currently processing
redisConn.set(model, "1")
print "Currently processing : " + redisConn.get(model)
cmd = "python dataGetter.py -b --model=" + model + " --clean"
cmdObj = {"model": model, "command": cmd}
beanstalkdConn.put(json.dumps(cmdObj))
print "doing nothing... Waiting 5 mins."
# Sleep for 3 mins.
time.sleep(180)
return
def main():
stalk = beanstalkc.Connection()
stalk.watch('game-requests')
championships = list(Match.objects.filter(root=True))
while not all([x.winner for x in championships]):
for championship in championships:
maintain_bracket(championship)
if stalk.stats_tube('game-requests')['current-jobs-ready'] < 1:
generate_speculative_game(random.choice(championships))
def announce(messg, user):
messg = str(messg)
# username = user.username
username = user.get_profile().peer.domain_name
b = beanstalkc.Connection()
b.use(settings.POLLS_TUBE)
tube_message = json.dumps({'message': messg, 'username': username})
b.put(tube_message)
b.close()
def _get_connection():
global _connection
if not _connection:
try:
host, port = g.DEFAULT_BEANSTALK.split(':')
except TypeError:
host, port = DEFAULT_BEANSTALK.split(':')
_connection = beanstalkc.Connection(host=host, port=int(port))
return _connection
def main():
stalk = beanstalkc.Connection()
if 'game-requests' not in stalk.tubes():
schedule_a_game()
while True:
if stalk.stats_tube('game-requests')['current-jobs-ready'] < 1:
schedule_a_game()
sleep(0.2)
stalk.close()
def handle(self):
proxy_data = self.rfile.readline().strip()
data = self.rfile.readline().strip()
for trigger, handler in TRIGGERS.items():
if trigger in data:
job = handler(proxy_data.split()[2], str(data))
print(job)
beanstalk = beanstalkc.Connection(host=BHOST, port=BPORT)
beanstalk.put(json.dumps(job))
def monitor_jobs():
# We have to open one socket per Greenlet, as currently socket sharing is
# not allowed
try:
b = beanstalkc.Connection()
except Exception, err:
logger.error("Error connecting to beanstalkd: %s" % str(err))
sleep(5)
return
