def __init__(self, queues=[], server="localhost:6379", password=None):
self.queues = queues
self.validate_queues()
self._shutdown = False
self.child = None
self.pid = os.getpid()
self.hostname = os.uname()[1]
if isinstance(server, basestring):
self.resq = ResQ(server=server, password=password)
elif isinstance(server, ResQ):
self.resq = server
else:
raise Exception("Bad server argument")
def pyres_web():
usage = "usage: %prog [options]"
parser = OptionParser(usage)
parser.add_option("--host",
dest="host",
default="localhost",
metavar="HOST")
parser.add_option("--port", dest="port", type="int", default=8080)
parser.add_option("--dsn", dest="dsn", help="redis server to display")
(options, args) = parser.parse_args()
if options.dsn:
from pyres import ResQ
server.HOST = ResQ(options.dsn)
run_itty(host=options.host, port=options.port)
def save(self, resq=None):
"""Saves the failed Job into a "failed" Redis queue preserving all its original enqueud info."""
if not resq:
resq = ResQ()
data = {
'failed_at': int(time.mktime(datetime.datetime.now().timetuple())),
'payload': self._payload,
'error': self._parse_message(self._exception),
'backtrace': self._parse_traceback(self._traceback),
'queue': self._queue
}
if self._worker:
data['worker'] = self._worker
data = ResQ.encode(data)
resq.redis.rpush('resque:failed', data)
def working(cls, host):
if isinstance(host, string_types):
resq = ResQ(host)
elif isinstance(host, ResQ):
resq = host
total = []
for key in Worker.all(host):
total.append('resque:worker:%s' % key)
names = []
for key in total:
value = resq.redis.get(key)
if value:
w = Worker.find(key[14:], resq) #resque:worker:
names.append(w)
return names
def save(self, resq=None):
"""Saves the failed Job into a "failed" Redis queue preserving all its original enqueud info."""
if not resq:
resq = ResQ()
data = {
'failed_at': datetime.datetime.now().strftime('%Y/%m/%d %H:%M:%S'),
'payload': self._payload,
'exception': self._exception.__class__.__name__,
'error': self._parse_message(self._exception),
'backtrace': self._parse_traceback(self._traceback),
'queue': self._queue
}
if self._worker:
data['worker'] = self._worker
data = ResQ.encode(data)
resq.redis.rpush('resque:failed', data)
def __init__(self, queues=(), server="localhost:6379", password=None, timeout=None, odoo_env=None):
self.queues = queues
self.validate_queues()
self._shutdown = False
self.child = None
self.pid = os.getpid()
self.hostname = os.uname()[1]
self.timeout = timeout
self.odoo_env = odoo_env
if isinstance(server, string_types):
self.resq = ResQ(server=server, password=password)
elif isinstance(server, ResQ):
self.resq = server
else:
raise Exception("Bad server argument")
def dispatch():
input = request.json
session_no = create_session(g.db,
input['build_id'],
parent=input['parent'],
labels=input['labels'],
run_info=input['run_info'],
state=SESSION_STATE_TO_BACKEND)
session_id = '%s-%s' % (input['build_id'], session_no)
ri = input['run_info'] or {}
args = ", ".join(ri.get('args', []))
title = "%s(%s)" % (ri.get('step_name', 'main'), args)
item = RunAsync(session_no, title)
add_slog(g.db, input['parent'], item)
r = ResQ()
r.enqueue(DispatchSession, session_id)
return jsonify(session_id=session_id)
def perform(args):
try:
sc2reader_to_esdb = SC2ReaderToEsdb()
filename = args['hash'] + '.s2gs'
gateway = args['gateway']
if gateway == 'sea':
gateway = 'sg'
# retrieve it from battlenet
depoturl = 'http://{0}.depot.battle.net:1119/{1}'.format(
gateway, filename)
try:
s2gsfile = urllib2.urlopen(depoturl).read()
except:
logging.getLogger("jobs").info(
"couldnt retrieve {} s2gs hash {}. maybe its bad.".format(
gateway, args['hash']))
return None
# save it in S3 because we are pack rats
bucket = boto.connect_s3(settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY)\
.get_bucket(settings.S2GS_BUCKET_NAME)
k = Key(bucket)
k.key = filename
k.set_contents_from_string(s2gsfile)
# parse it and write stuff to DB
summaryDB = sc2reader_to_esdb.processSummary(
StringIO(s2gsfile), args['hash'])
except Exception as e:
tb = traceback.format_exc()
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logging.getLogger("jobs").info(
"parsing failed for s2gs {}. oh well. exception={}. {} {} {} {}"
.format(args['hash'], e, exc_type, fname, exc_tb.tb_lineno,
tb))
pass
finally:
# Enqueue ruby PostParse job, always!
ResQ(server=settings.REDIS_SERVER).enqueue_from_string(
'ESDB::Jobs::Sc2::Summary::PostParse', 'summaries-high',
{'hash': args['hash']})
def check_in_available(agent_id):
session_id = request.json['session_id']
build_id, num = session_id.split('-')
with g.db.pipeline() as pipe:
set_session_done(pipe, session_id, request.json['result'],
request.json['output'], request.json['log_file'])
if int(num) == 0:
Build.set_done(build_id, request.json['result'], pipe=pipe)
add_slog(pipe, session_id, SessionDone(request.json['result']))
pipe.hmset(jdb.KEY_AGENT % agent_id,
dict(state=jdb.AGENT_STATE_AVAIL, seen=get_ts()))
pipe.execute()
r = ResQ()
r.enqueue(AgentAvailable, agent_id)
return jsonify()
def pyres_web():
usage = "usage: %prog [options]"
parser = OptionParser(usage)
parser.add_option("--host",
dest="host",
default="localhost",
metavar="HOST")
parser.add_option("--port", dest="port", type="int", default=8080)
parser.add_option("--dsn", dest="dsn", help="Redis server to display")
parser.add_option("--server",
dest="server",
help="Server for itty to run under.",
default='wsgiref')
(options, args) = parser.parse_args()
if options.dsn:
from pyres import ResQ
resweb_server.HOST = ResQ(options.dsn)
run_itty(host=options.host, port=options.port, server=options.server)
def main(params=None):
if params is None:
params = sys.argv[1:]
parser = argparse.ArgumentParser(description='Runs pyres web console.')
conn_group = parser.add_argument_group('Connection arguments')
conn_group.add_argument('--host', default='localhost', help='Binding host')
conn_group.add_argument('--port',
default=8080,
type=int,
help='Binding port')
conn_group = parser.add_argument_group('Redis arguments')
conn_group.add_argument('--redis-host',
default='localhost',
help='Redis host')
conn_group.add_argument('--redis-port',
default=6379,
type=int,
help='Redis port')
conn_group.add_argument('--redis-database',
default=0,
type=int,
help='Redis database')
conn_group.add_argument('--redis-password',
default=None,
help='Redis password')
other_group = parser.add_argument_group('Other arguments')
other_group.add_argument('-l',
'--level',
default='debug',
help='Logging level')
arguments = parser.parse_args(params)
logging.basicConfig(level=getattr(logging, arguments.level.upper()))
redis = Redis(host=arguments.redis_host,
port=arguments.redis_port,
db=arguments.redis_database,
password=arguments.redis_password)
resweb_server.HOST = ResQ(redis)
run_itty(host=arguments.host, port=arguments.port, server='wsgiref')
def run(self):
setproctitle('pyres_minion:%s: Starting' % (os.getppid(), ))
if self.log_path:
if special_log_file(self.log_path):
self.log_file = self.log_path
else:
self.log_file = os.path.join(self.log_path,
'minion-%s.log' % self.pid)
namespace = 'minion:%s' % self.pid
self.logger = setup_logging('minion', namespace, self.log_level,
self.log_file)
#self.clear_logger()
if isinstance(self.server, string_types):
self.resq = ResQ(server=self.server, password=self.password)
elif isinstance(self.server, ResQ):
self.resq = self.server
else:
raise Exception("Bad server argument")
self.work()
def details():
r = ResQ()
with closing(get_connection()) as connection:
total = 0
with closing(connection.cursor()) as cursor:
query = '''
SELECT COUNT(id) AS count
FROM records
WHERE details IS NULL
'''
cursor.execute(query)
total = cursor.fetchone()['count']
with closing(connection.cursor('cursor')) as cursor:
query = '''
SELECT *
FROM records
WHERE details IS NULL
'''
cursor.execute(query)
for record in tqdm(cursor, total=total):
r.enqueue(Record, record['id'])
def do_register():
agent_id = request.json['id']
info = {
"ip": request.remote_addr,
'nick': request.json.get('nick', ''),
"port": request.json["port"],
"state": jdb.AGENT_STATE_AVAIL,
"seen": get_ts(),
"labels": ",".join(request.json["labels"])
}
with g.db.pipeline() as pipe:
pipe.hmset(jdb.KEY_AGENT % agent_id, info)
pipe.sadd(jdb.KEY_ALL, agent_id)
for label in request.json["labels"]:
pipe.sadd(jdb.KEY_LABEL % label, agent_id)
pipe.execute()
r = ResQ()
r.enqueue(AgentAvailable, agent_id)
return jsonify()
def main(args=None):
if args is None:
args = sys.argv[1:]
parser = argparse.ArgumentParser()
parser.add_argument('--conf', '-c', help="Path to configuration file.")
parser.add_argument('--verbose', '-v', action='count', default=0, help='Log level: v=warning, vv=info, vvv=debug.')
options = parser.parse_args(args)
log_level = LOGS[options.verbose].upper()
logging.basicConfig(
level=getattr(logging, log_level),
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
if options.conf:
cfg = Config.load(abspath(expanduser(options.conf)))
else:
cfg = Config()
conn = ResQ(server="%s:%s" % (cfg.REDIS_HOST, cfg.REDIS_PORT), password=cfg.REDIS_PASSWORD)
conn.config = cfg
connect(
cfg.MONGO_DB,
host=cfg.MONGO_HOST,
port=cfg.MONGO_PORT,
username=cfg.MONGO_USER,
password=cfg.MONGO_PASS
)
print
print("--- Wight worker started ---")
print
Worker.run([WorkerJob.queue], conn)
print
print "--- Wight worker killed ---"
print
def before_request():
"""Make sure we are connected to the database each request."""
g.pyres = ResQ(app.config['RESWEB_HOST'], password=app.config.get('RESWEB_PASSWORD', None))
def setUp(self):
self.resq = ResQ()
self.redis = self.resq.redis
self.redis.flushall()
def perform(args):
performStart = datetime.now()
md5 = None
replayDB = None
try:
sc2reader_to_esdb = SC2ReaderToEsdb()
#
# at this point the 'hash' may actually be an S3 key like '/uploads/1234-5667-1234234/filename.sc2replay'
# or simply '{md5}'
#
# not to worry, in a few lines, we'll rename the S3 key to be md5.sc2replay
#
filename = args['hash']
if re.search('.sc2replay', filename, re.IGNORECASE) is None:
filename = filename + ".SC2Replay"
bucket = boto.connect_s3(settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY)\
.get_bucket(settings.REPLAY_BUCKET_NAME)
# logging.getLogger("jobs").info("trying to get key {}".format(filename));
k = bucket.get_key(filename)
replaystring = k.get_contents_as_string()
md5 = hashlib.md5(replaystring).hexdigest()
#
# rename the S3 key to simply be md5.SC2Replay, so it's easier for us to find it
# when we need it.
#
# http://stackoverflow.com/questions/2481685/amazon-s3-boto-how-to-rename-a-file-in-a-bucket
k.copy(settings.REPLAY_BUCKET_NAME, md5 + ".SC2Replay",
metadata=None, preserve_acl=False)
replayDB, blob = sc2reader_to_esdb.processReplay(StringIO(replaystring), args['channel'])
if len(blob) > 0:
blobbucket = boto.connect_s3(settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY)\
.get_bucket(settings.BLOB_BUCKET_NAME)
k = Key(blobbucket)
k.key = "%i" % (replayDB.match.id)
blobdump = json.dumps(blob)
k.set_contents_from_string(blobdump)
except Exception as e:
tb = traceback.format_exc()
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logging.getLogger("jobs").info("parsing failed for replay {}. oh well. exception={}. {} {} {} {}".format(args['hash'].encode('ascii'), e, exc_type, fname, exc_tb.tb_lineno, tb))
pass
finally:
alldone = datetime.now()
# Enqueue ruby PostParse job, always.
ResQ(server=settings.REDIS_SERVER).enqueue_from_string('ESDB::Jobs::Sc2::Replay::PostParse', 'replays-high', {
'uuid': args['uuid'],
'hash': md5,
'provider_id': str(args['provider_id']),
'ggtracker_received_at': args['ggtracker_received_at'],
'esdb_received_at': args['esdb_received_at'],
'preparse_received_at': args['preparse_received_at'],
'jobspy_received_at': performStart.strftime('%s.%f'),
'jobspy_done_at': alldone.strftime('%s.%f'),
})
# regarding converting times to floating point seconds since the
# epoch, using %s above is dangerous because its not python, it
# calls the underlying OS. i tried using the solution here:
# http://stackoverflow.com/questions/6999726/python-getting-millis-since-epoch-from-datetime/11111177#11111177
# but i ran into timezone issues and did the lazy thing instead.
matchId = 0
if replayDB and hasattr(replayDB, "match") and replayDB.match.id:
matchId = replayDB.match.id
logging.getLogger("jobs").info("all done with match {}. total time in ParseReplay.perform() = {}".format(matchId, alldone - performStart))
def queue_job(input_coordinate=None, \
hg18_option='off', \
transcript_type='NA', \
refseq_correspondence='no', \
sample_id='yes', \
analysis_type=None, \
analysis_programs=None):
if analysis_type == None:
print 'Choose an analysis type.'
return
input_basename = input_coordinate + '_' + hg18_option + '_' + transcript_type + '_' + refseq_correspondence + '_' + sample_id + '.txt'
job_name = input_basename[:-4] + '_' + analysis_type + '_' + analysis_programs + '_20121231_010101'
job_dir = os.path.join(diagnostic_tests_dir, job_name)
if os.path.exists(job_dir) == False:
os.mkdir(job_dir)
input_filename = os.path.join(job_dir, input_basename)
shutil.copy(os.path.join(diagnostic_tests_dir, input_basename), input_filename)
job_id = job_name
mutation_filename = input_filename
mutation_filename_fix = mutation_filename + '.fix'
mutation_filename_error = mutation_filename +'.error'
email = '*****@*****.**'
classifier = '_Other'
upload_filename = input_basename
user_upload_dir = job_dir
chosen_db_str = analysis_programs.strip('_')
chosen_dbs = chosen_db_str.split('_')
tsv_report = 'on'
gene_annot = 'on'
hg18 = hg18_option
functional_annot = 'on'
f = open(input_filename)
no_input_line = 0
for line in f:
if len(line) > 0:
if not line[0] in ['#', '>', '!']:
no_input_line += 1
f.close()
if no_input_line >= 5000:
queue_name = queue_name_prefix + '_LARGE'
else:
queue_name = queue_name_prefix + '_SMALL'
mupit_input = 'on'
resubmit = 'no'
wf = open(os.path.join(job_dir, 'job_info.txt'), 'w')
wf.write(str(no_input_line)+'\n')
wf.write(job_id+'\n')
wf.write(chosen_db_str+'\n')
wf.write('call_queuer_path\n')
wf.write('python_path\n')
wf.write('queuer_path\n')
wf.write(email+'\n')
wf.write(job_id+'\n')
wf.write(classifier+'\n')
wf.write(mutation_filename+'\n')
wf.write(upload_filename+'\n')
wf.write(user_upload_dir+'\n')
wf.write(chosen_db_str+'\n')
wf.write(tsv_report+'\n')
wf.write(functional_annot+'\n')
wf.write(hg18+'\n')
wf.write(analysis_type+'\n')
wf.write(str(no_input_line)+'\n')
wf.write('error_output_path\n')
wf.write(mupit_input+'\n')
wf.close()
r=ResQ()
argstr = 'dummy'
argstr += ' ' + 'yes'
argstr += ' ' + user_upload_dir
argstr += ' ' + resubmit
argstr += ' ' + job_id
argstr += ' -e ' + email
argstr += ' -i ' + job_id
argstr += ' -c ' + classifier
argstr += ' -m ' + mutation_filename
argstr += ' -u ' + upload_filename
argstr += ' -D ' + user_upload_dir
argstr += ' -d ' + chosen_db_str
argstr += ' -t ' + tsv_report
argstr += ' -f ' + functional_annot
argstr += ' -r ' + hg18
argstr += ' -y ' + analysis_type
argstr += ' -n ' + str(no_input_line)
argstr += ' -M ' + mupit_input
argstr += ' -R ' + resubmit
argstr += ' -Q ' + queue_name
argstr += ' -T ' + 'yes' # -T option for 'test'
r.enqueue(queue_name, masteranalyzer.MasterAnalyzer, argstr)
global job_ids
job_ids.append(job_id)
def test_redis_property(self):
from redis import Redis
rq = ResQ(server="localhost:6379")
red = Redis()
#rq2 = ResQ(server=red)
self.assertRaises(Exception, rq.redis, [Basic])
from pyres import ResQ
from django.conf import settings
pyres = ResQ(getattr(settings,'PYRES_HOST','localhost:6379'),getattr(settings,'PYRES_PASSWORD',None))
class PyresMixin(object):
@classmethod
def enqueue(cls, *args):
if settings.PYRES_USE_QUEUE:
pyres.enqueue(cls, *args)
else:
cls.perform(*args)
import os
from itty import *
from pyres import ResQ
from pyres import failure
from views import (Overview, Queues, Queue, Workers, Working, Failed, Stats,
Stat, Worker, Delayed, DelayedTimestamp)
from base64 import b64decode
HOST = ResQ("localhost:6379")
MY_ROOT = os.path.join(os.path.dirname(__file__), 'media')
#resq = ResQ(HOST)
@get("/")
def index(request):
return str(Overview(HOST).render())
@get("/working/")
def working(request):
return str(Working(HOST).render())
@get("/queues/")
def queues(request):
return str(Queues(HOST).render())
@get('/queues/(?P<queue_id>\w.+)/')
def queue(request, queue_id):
start = int(request.GET.get('start', 0))
def get_resq(request):
return ResQ()
def get_pyres():
'''
Connection getter.
'''
return ResQ('%s:%d' % (getattr(settings, 'REDIS_HOST', 'localhost'),
getattr(settings, 'REDIS_PORT', 6379)))
