def publish(subject, text):
'''
Publish cumulative post to Livejournal.
'''
r = ResQ()
for target in settings.PUBLISHING_LJ_TARGETS.iterkeys():
r.enqueue(LivejournalPost, target, subject, text)
class Jobs(object):
def __init__(self,host='localhost',port=6379):
HOST = 'localhost:6379'
self.host=host
self.port=port
self.queue='iOSPush'
self.resq = ResQ('%s:%i'%(self.host,self.port))
def push(self,queue,item):
self.resq.push(queue,item)
def pop(self,queue):
return self.resq.pop(queue)
def add(self):
pass
def send(self):
info=self.pop(self.queue) #will get (None, None) if none
info={'badge':1,'msg':'您有新的车票信息','token':'cb86b176ee99ae5f3387c79f1226d234599e91c7bc300e97afb034cc0009e192'}
deviceToken = binascii.unhexlify(info['token'])
message = APNSNotification()
message.token(deviceToken)
message.alert(info['msg'])
message.badge(info['badge'])
message.sound()
wrapper = APNSNotificationWrapper('ck.pem', True)
wrapper.append(message)
wrapper.notify()
def pyres_web():
usage = "usage: %prog [options]"
parser = OptionParser(usage)
parser.add_option("--host",
dest="host",
default="localhost",
metavar="HOST")
parser.add_option("--port", dest="port", type="int", default=8080)
parser.add_option("--dsn", dest="dsn", help="Redis server to display")
parser.add_option("--auth", dest="auth", help="Redis user:pass")
parser.add_option("--server",
dest="server",
help="Server for itty to run under.",
default='wsgiref')
(options, args) = parser.parse_args()
if options.dsn:
from pyres import ResQ
if options.auth is not None:
from redis import Redis
rhost, rport = options.dsn.split(':')
ruser, rpass = options.auth.split(':')
redis = Redis(host=rhost,
port=int(rport),
db=ruser,
password=rpass)
resweb_server.HOST = ResQ(redis)
else:
resweb_server.HOST = ResQ(options.dsn)
run_itty(host=options.host, port=options.port, server=options.server)
def publish(text, url=None):
'''
Publish status to social networks.
'''
r = ResQ()
r.enqueue(TwitterStatus, text, url)
r.enqueue(VkontakteStatus, text, url)
def all(cls, host="localhost:6379"):
if isinstance(host,string_types):
resq = ResQ(host)
elif isinstance(host, ResQ):
resq = host
return [Worker.find(w,resq) for w in resq.workers() or []]
def all(cls, host="localhost:6379"):
if isinstance(host, string_types):
resq = ResQ(host)
elif isinstance(host, ResQ):
resq = host
return [Worker.find(w, resq) for w in resq.workers() or []]
def test_push(self):
self.resq.push('pushq', 'content-newqueue')
self.resq.push('pushq', 'content2-newqueue')
assert self.redis.llen('resque:queue:pushq') == 2
assert self.redis.lindex('resque:queue:pushq',
0) == ResQ.encode('content-newqueue')
assert self.redis.lindex('resque:queue:pushq',
1) == ResQ.encode('content2-newqueue')
def perform(eventid):
print (eventid)
db = sqlitedb()
eventrec = db.getactiveevent()
for event in eventrec:
print event['eventid']
r = ResQ(server="%s:%s" % (resqserver, resqport))
r.enqueue(callQ, event['eventid'])
def __init__(self, queues, host, max_children=10):
self.queues = queues
self._host = host
self.max_children = max_children
self._shutdown = False
self.children = []
self.resq = ResQ(host)
self.validate_queues()
self.reports = {}
def trigger(self, timer):
intent_json = self.db.hget('timer:%s' % timer, 'intent')
if not intent_json:
logging.warning("Triggering timer %s, but found no intent" % timer)
return
from async .send_intent import SendIntent
r = ResQ()
r.enqueue(SendIntent, intent_json)
def test_get_job(self):
worker = Worker(['basic'])
self.resq.enqueue(Basic,"test1")
job = Job.reserve('basic', self.resq)
worker.working_on(job)
name = "%s:%s:%s" % (os.uname()[1],os.getpid(),'basic')
assert worker.job() == ResQ.decode(self.redis.get('resque:worker:%s' % name))
assert worker.processing() == ResQ.decode(self.redis.get('resque:worker:%s' % name))
worker.done_working()
w2 = Worker(['basic'])
print w2.job()
assert w2.job() == {}
def test_get_job(self):
worker = Worker(['basic'])
self.resq.enqueue(Basic,"test1")
job = Job.reserve('basic', self.resq)
worker.working_on(job)
name = "%s:%s:%s" % (os.uname()[1],os.getpid(),'basic')
assert worker.job() == ResQ.decode(self.redis.get('resque:worker:%s' % name))
assert worker.processing() == ResQ.decode(self.redis.get('resque:worker:%s' % name))
worker.done_working(job)
w2 = Worker(['basic'])
print w2.job()
assert w2.job() == {}
def do_start_build(job_name):
input = request.json
job = Job.load(job_name, input.get('job_ref'))
build = Build.create(job,
parameters=input.get('parameters', {}),
description=input.get('description', ''))
session_id = '%s-0' % build.uuid
set_session_queued(g.db, session_id)
r = ResQ()
r.enqueue(DispatchSession, session_id)
return jsonify(**build.as_dict())
def __init__(self, server="localhost:6379", password=None):
"""
>>> from pyres.scheduler import Scheduler
>>> scheduler = Scheduler('localhost:6379')
"""
self._shutdown = False
if isinstance(server, basestring):
self.resq = ResQ(server=server, password=password)
elif isinstance(server, ResQ):
self.resq = server
else:
raise Exception("Bad server argument")
def save(self, resq=None):
"""Saves the failed Job into a "failed" Redis queue preserving all its original enqueud info."""
if not resq:
resq = ResQ()
data = {
'failed_at': int(time.mktime(datetime.datetime.now().timetuple())),
'payload': self._payload,
'error': self._parse_message(self._exception),
'backtrace': self._parse_traceback(self._traceback),
'queue': self._queue
}
if self._worker:
data['worker'] = self._worker
data = ResQ.encode(data)
resq.redis.rpush('resque:failed', data)
def retry(self, payload_class, args):
"""This method provides a way to retry a job after a failure.
If the jobclass defined by the payload containes a ``retry_every``
attribute then pyres will attempt to retry the job until successful
or until timeout defined by ``retry_timeout`` on the payload class.
:param payload_class: the :class:`Job`-like class that needs
to be retried
:type payload_class: :class:`Job`-like
:param args: The args to be passed to the `payload_class.perform`
method when it is retried.
:type args: list
"""
retry_every = getattr(payload_class, 'retry_every', None)
retry_timeout = getattr(payload_class, 'retry_timeout', 0)
if retry_every:
now = ResQ._current_time()
first_attempt = self._payload.get("first_attempt", now)
retry_until = first_attempt + timedelta(seconds=retry_timeout)
retry_at = now + timedelta(seconds=retry_every)
if retry_at < retry_until:
self.resq.enqueue_at(retry_at, payload_class, *args,
**{'first_attempt':first_attempt})
return True
return False
def save(self, resq=None):
"""Saves the failed Job into a "failed" Redis queue preserving all its original enqueud info."""
if not resq:
resq = ResQ()
data = {
'failed_at': datetime.datetime.now().strftime('%Y/%m/%d %H:%M:%S'),
'payload': self._payload,
'exception': self._exception.__class__.__name__,
'error': self._parse_message(self._exception),
'backtrace': self._parse_traceback(self._traceback),
'queue': self._queue
}
if self._worker:
data['worker'] = self._worker
data = ResQ.encode(data)
resq.redis.rpush('resque:failed', data)
def dispatch():
input = request.json
session_no = create_session(g.db,
input['build_id'],
parent=input['parent'],
labels=input['labels'],
run_info=input['run_info'],
state=SESSION_STATE_TO_BACKEND)
session_id = '%s-%s' % (input['build_id'], session_no)
ri = input['run_info'] or {}
args = ", ".join(ri.get('args', []))
title = "%s(%s)" % (ri.get('step_name', 'main'), args)
item = RunAsync(session_no, title)
add_slog(g.db, input['parent'], item)
r = ResQ()
r.enqueue(DispatchSession, session_id)
return jsonify(session_id=session_id)
def check_in_available(agent_id):
session_id = request.json['session_id']
build_id, num = session_id.split('-')
with g.db.pipeline() as pipe:
set_session_done(pipe, session_id, request.json['result'],
request.json['output'], request.json['log_file'])
if int(num) == 0:
Build.set_done(build_id, request.json['result'], pipe=pipe)
add_slog(pipe, session_id, SessionDone(request.json['result']))
pipe.hmset(jdb.KEY_AGENT % agent_id,
dict(state=jdb.AGENT_STATE_AVAIL, seen=get_ts()))
pipe.execute()
r = ResQ()
r.enqueue(AgentAvailable, agent_id)
return jsonify()
def __init__(self, queues, host, max_children=10):
self.queues = queues
self._host = host
self.max_children = max_children
self._shutdown = False
self.children = []
self.resq = ResQ(host)
self.validate_queues()
self.reports = {}
def autodiscover():
"""
Auto-discover INSTALLED_APPS cron.py modules and fail silently when
not present. This forces an import on them to register any cron jobs they
may want.
"""
import imp
from django.conf import settings
for app in settings.INSTALLED_APPS:
# For each app, we need to look for an cron.py inside that app's
# package. We can't use os.path here -- recall that modules may be
# imported different ways (think zip files) -- so we need to get
# the app's __path__ and look for cron.py on that path.
# Step 1: find out the app's __path__ Import errors here will (and
# should) bubble up, but a missing __path__ (which is legal, but weird)
# fails silently -- apps that do weird things with __path__ might
# need to roll their own cron registration.
try:
app_path = __import__(app, {}, {}, [app.split('.')[-1]]).__path__
except AttributeError:
continue
# Step 2: use imp.find_module to find the app's admin.py. For some
# reason imp.find_module raises ImportError if the app can't be found
# but doesn't actually try to import the module. So skip this app if
# its admin.py doesn't exist
try:
fp, pathname, description = imp.find_module('tasks', app_path)
except ImportError:
continue
# Step 3: import the app's task file. If this has errors we want them
# to bubble up.
modulename = imp.load_module('tasks', fp, pathname, description)
# Step 4: iterate though tasks.py module and find all classes with their names
# starting with 'Periodic' and 'Interval'.
for item in dir(modulename):
if 'Periodic' or 'Interval' in item and hasattr(item, '__class__'):
pyres_sched = PyresScheduler()
resque = ResQ()
pyres_sched.add_resque(resque)
pyres_sched.start()
classname = getattr(modulename, item)
if hasattr(classname, 'run_every'):
run_every = getattr(classname, 'run_every')
if 'Periodic' in item:
print 'periodic'
pyres_sched.add_cron_job(classname, args=None, **run_every)
elif 'Interval' in item:
pyres_sched.add_interval_job(classname,
args=None,
**run_every)
def setup_resq(self):
if hasattr(self,'logger'):
self.logger.info('Connecting to redis server - %s' % self.server)
if isinstance(self.server,string_types):
self.resq = ResQ(server=self.server, password=self.password)
elif isinstance(self.server, ResQ):
self.resq = self.server
else:
raise Exception("Bad server argument")
def all(cls, resq, start=0, count=1):
items = resq.redis.lrange('resque:failed', start, count) or []
ret_list = []
for i in items:
failure = ResQ.decode(i)
failure['redis_value'] = b64encode(i)
ret_list.append(failure)
return ret_list
def all(cls, resq, start=0, count=1):
items = resq.redis.lrange('resque:failed', start, count) or []
ret_list = []
for i in items:
failure = ResQ.decode(i)
failure['redis_value'] = b64encode(i)
ret_list.append(failure)
return ret_list
def queues(request):
queues = ResQ.queues(HOST)
failure_count = Failure.count(ResQ(HOST))
template = env.get_template('queues.html')
dic = {
'queues':queues,
'failure_count':failure_count,
'resq': resq
}
return str(template.render(dic))
def failed_retry(request):
try:
import json
except ImportError:
import simplejson as json
failed_job = request.POST['failed_job']
job = b64decode(failed_job)
decoded = ResQ.decode(job)
failure.retry(HOST, decoded['queue'], job)
raise Redirect('/failed/')
def save(self, resq):
data = {
'failed_at' : str(datetime.datetime.now()),
'payload' : self._payload,
'error' : self._parse_message(self._exception),
'backtrace' : self._parse_traceback(self._traceback),
'queue' : self._queue
}
data = ResQ.encode(data)
resq.redis.push('failed', data)
def failed_retry(request):
try:
import json
except ImportError:
import simplejson as json
failed_job = request.POST['failed_job']
job = b64decode(failed_job)
decoded = ResQ.decode(job)
failure.retry(HOST, decoded['queue'], job)
raise Redirect('/failed/')
def all(cls, host="localhost:6379"):
if isinstance(host, basestring):
resq = ResQ(host)
elif isinstance(host, ResQ):
resq = host
return [
Worker.find(w, resq)
for w in resq.redis.smembers('resque:workers') or []
]
def __init__(self,
queues=(),
server="localhost:6379",
password=None,
timeout=None):
self.queues = queues
self.validate_queues()
self._shutdown = False
self.child = None
self.pid = os.getpid()
self.hostname = os.uname()[1]
self.timeout = timeout
if isinstance(server, string_types):
self.resq = ResQ(server=server, password=password)
elif isinstance(server, ResQ):
self.resq = server
else:
raise Exception("Bad server argument")
def failed_job(request, retry=False):
failed_job_ = request.POST['failed_job']
job = b64decode(failed_job_)
if retry:
# post /failed/retry
decoded = ResQ.decode(job)
failure.retry(get_pyres(), decoded['queue'], job)
else:
# post /failed/delete
failure.delete(get_pyres(), job)
return redirect('resweb-failed')
def __init__(self, server="localhost:6379", password=None):
"""
>>> from pyres.scheduler import Scheduler
>>> scheduler = Scheduler('localhost:6379')
"""
self._shutdown = False
if isinstance(server, basestring):
self.resq = ResQ(server=server, password=password)
elif isinstance(server, ResQ):
self.resq = server
else:
raise Exception("Bad server argument")
def details():
r = ResQ()
with closing(get_connection()) as connection:
total = 0
with closing(connection.cursor()) as cursor:
query = '''
SELECT COUNT(id) AS count
FROM records
WHERE details IS NULL
'''
cursor.execute(query)
total = cursor.fetchone()['count']
with closing(connection.cursor('cursor')) as cursor:
query = '''
SELECT *
FROM records
WHERE details IS NULL
'''
cursor.execute(query)
for record in tqdm(cursor, total=total):
r.enqueue(Record, record['id'])
def submit_feedback(request):
if request.method != 'POST':
raise Http404
if 'lang' in request.GET and request.GET['lang'] in ALLOWED_LANGS:
translation.activate(request.GET['lang'])
try:
form = FeedbackForm(request.POST or None)
if form.is_valid():
r = ResQ()
r.enqueue(FeedbackTask, form.cleaned_data['name'],
form.cleaned_data['email'], form.cleaned_data['text'])
return HttpResponse(status=200)
# form is invalid
return HttpResponse(
loader.render_to_string('feedback/feedback_form_partial.html',
{'feedback_form': form}),
status=409)
except Exception:
logger.exception('')
raise
def retry(self, payload_class, args):
retry_every = getattr(payload_class, "retry_every", None)
retry_timeout = getattr(payload_class, "retry_timeout", 0)
if retry_every:
now = ResQ._utcnow()
first_attempt = self._payload.get("first_attempt", now)
retry_until = first_attempt + timedelta(seconds=retry_timeout)
retry_at = now + timedelta(seconds=retry_every)
if retry_at < retry_until:
self.resq.enqueue_at(retry_at, payload_class, *args, **{"first_attempt": first_attempt})
return True
return False
def add_repository_to_queue(user, repo_owner, repository_name, repo_object=None):
res = ResQ()
repo = None
token = user.get_profile().extra_data['access_token']
if not repo_object:
gh = Github(login=user.email, token=token)
repo = gh.repos.get(repo_owner, repository_name)
else:
repo = repo_object
queue_data = {'email': user.email, 'token': token}
queue_data['repo'] = {
'name': repo.name,
'url': repo.url,
'git_url': repo.git_url,
'html_url': repo.html_url,
'language': repo.language
}
db_repo, created = Repository.objects.get_or_create(
git_url=repo.git_url, defaults=queue_data['repo'])
query_filter = {'user': user, 'repository': db_repo}
query_filter.update({'defaults': {
'user': user,
"repository": db_repo
}
})
as_contributor, created = Contributor.objects.get_or_create(**query_filter)
# put a timestamp field in repository model to verify if there is need to
# process again. so only put in queue if is not created and timestamp > x time
res.enqueue(RepositoryWorker, queue_data)
def save(self, resq=None):
if not resq:
resq = ResQ()
data = {
'failed_at' : str(datetime.datetime.now()),
'payload' : self._payload,
'error' : self._parse_message(self._exception),
'backtrace' : self._parse_traceback(self._traceback),
'queue' : self._queue
}
if self._worker:
data['worker'] = self._worker
data = ResQ.encode(data)
resq.redis.rpush('resque:failed', data)
def do_register():
agent_id = request.json['id']
info = {
"ip": request.remote_addr,
'nick': request.json.get('nick', ''),
"port": request.json["port"],
"state": jdb.AGENT_STATE_AVAIL,
"seen": get_ts(),
"labels": ",".join(request.json["labels"])
}
with g.db.pipeline() as pipe:
pipe.hmset(jdb.KEY_AGENT % agent_id, info)
pipe.sadd(jdb.KEY_ALL, agent_id)
for label in request.json["labels"]:
pipe.sadd(jdb.KEY_LABEL % label, agent_id)
pipe.execute()
r = ResQ()
r.enqueue(AgentAvailable, agent_id)
return jsonify()
def pyres_web():
usage = "usage: %prog [options]"
parser = OptionParser(usage)
parser.add_option("--host",
dest="host",
default="localhost",
metavar="HOST")
parser.add_option("--port", dest="port", type="int", default=8080)
parser.add_option("--dsn", dest="dsn", help="redis server to display")
(options, args) = parser.parse_args()
if options.dsn:
from pyres import ResQ
server.HOST = ResQ(options.dsn)
run_itty(host=options.host, port=options.port)
def save(self, resq=None):
if not resq:
resq = ResQ()
data = {
"failed_at": str(datetime.datetime.now()),
"payload": self._payload,
"error": self._parse_message(self._exception),
"backtrace": self._parse_traceback(self._traceback),
"queue": self._queue,
}
if self._worker:
data["worker"] = self._worker
data = ResQ.encode(data)
resq.redis.push("resque:failed", data)
def retry(self, payload_class, args):
retry_every = getattr(payload_class, 'retry_every', None)
retry_timeout = getattr(payload_class, 'retry_timeout', 0)
if retry_every:
now = ResQ._current_time()
first_attempt = self._payload.get("first_attempt", now)
retry_until = first_attempt + timedelta(seconds=retry_timeout)
retry_at = now + timedelta(seconds=retry_every)
if retry_at < retry_until:
self.resq.enqueue_at(retry_at, payload_class, *args,
**{'first_attempt': first_attempt})
return True
return False
def main(args=None):
if args is None:
args = sys.argv[1:]
parser = argparse.ArgumentParser()
parser.add_argument('--conf', '-c', help="Path to configuration file.")
parser.add_argument('--verbose', '-v', action='count', default=0, help='Log level: v=warning, vv=info, vvv=debug.')
options = parser.parse_args(args)
log_level = LOGS[options.verbose].upper()
logging.basicConfig(
level=getattr(logging, log_level),
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
if options.conf:
cfg = Config.load(abspath(expanduser(options.conf)))
else:
cfg = Config()
conn = ResQ(server="%s:%s" % (cfg.REDIS_HOST, cfg.REDIS_PORT), password=cfg.REDIS_PASSWORD)
conn.config = cfg
connect(
cfg.MONGO_DB,
host=cfg.MONGO_HOST,
port=cfg.MONGO_PORT,
username=cfg.MONGO_USER,
password=cfg.MONGO_PASS
)
print
print("--- Wight worker started ---")
print
Worker.run([WorkerJob.queue], conn)
print
print "--- Wight worker killed ---"
print
def __init__(self, queues=(), server="localhost:6379", password=None, timeout=None):
self.queues = queues
self.validate_queues()
self._shutdown = False
self.child = None
self.pid = os.getpid()
self.hostname = os.uname()[1]
self.timeout = timeout
if isinstance(server, string_types):
self.resq = ResQ(server=server, password=password)
elif isinstance(server, ResQ):
self.resq = server
else:
raise Exception("Bad server argument")
def save(self, resq=None):
"""Saves the failed Job into a "failed" Redis queue preserving all its original enqueud info."""
if not resq:
resq = ResQ()
data = {
"failed_at": int(time.mktime(datetime.datetime.now().timetuple())),
"payload": self._payload,
"error": self._parse_message(self._exception),
"backtrace": self._parse_traceback(self._traceback),
"queue": self._queue,
}
if self._worker:
data["worker"] = self._worker
data = ResQ.encode(data)
resq.redis.rpush("resque:failed", data)
def save(self, resq=None):
"""Saves the failed Job into a "failed" Redis queue preserving all its original enqueud info."""
if not resq:
resq = ResQ()
data = {
'failed_at' : int(time.mktime(datetime.datetime.now().timetuple())),
'payload' : self._payload,
'error' : self._parse_message(self._exception),
'backtrace' : self._parse_traceback(self._traceback),
'queue' : self._queue
}
if self._worker:
data['worker'] = self._worker
data = ResQ.encode(data)
resq.redis.rpush('resque:failed', data)
def perform(text, url):
'''
Delayed task.
'''
if not settings.PUBLISHING_VKONTAKTE_USER:
return
if not url is None:
text = '%s %s' % (text, url)
method = 'wall.post'
owner_id, token, extra_params = settings.PUBLISHING_VKONTAKTE_USER
params = {'owner_id': owner_id,
'access_token': token,
'message': text}
params.update(extra_params)
resp = vk(method, params)
post_id = resp.get("response", {}).get("post_id")
if not post_id:
raise ValueError('Post ID not provided.\nMethod %s\nParams %r'\
'\n Response %r' % (method, params, resp))
r = ResQ()
for user, data in settings.PUBLISHING_VKONTAKTE_REPOST_USERS.iteritems():
r.enqueue(VkontakteLike, user, owner_id, post_id)
def working(cls, host):
if isinstance(host, string_types):
resq = ResQ(host)
elif isinstance(host, ResQ):
resq = host
total = []
for key in Worker.all(host):
total.append('resque:worker:%s' % key)
names = []
for key in total:
value = resq.redis.get(key)
if value:
w = Worker.find(key[14:], resq) #resque:worker:
names.append(w)
return names
def save(self, resq=None):
"""Saves the failed Job into a "failed" Redis queue preserving all its original enqueud info."""
if not resq:
resq = ResQ()
data = {
"failed_at": datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S"),
"payload": self._payload,
"exception": self._exception.__class__.__name__,
"error": self._parse_message(self._exception),
"backtrace": self._parse_traceback(self._traceback),
"queue": self._queue,
}
if self._worker:
data["worker"] = self._worker
data = ResQ.encode(data)
resq.redis.rpush("resque:failed", data)
def save(self, resq=None):
"""Saves the failed Job into a "failed" Redis queue preserving all its original enqueud info."""
if not resq:
resq = ResQ()
data = {
'failed_at' : datetime.datetime.now().strftime('%Y/%m/%d %H:%M:%S'),
'payload' : self._payload,
'exception' : self._exception.__class__.__name__,
'error' : self._parse_message(self._exception),
'backtrace' : self._parse_traceback(self._traceback),
'queue' : self._queue
}
if self._worker:
data['worker'] = self._worker
data = ResQ.encode(data)
resq.redis.rpush('resque:failed', data)
def perform(args):
try:
sc2reader_to_esdb = SC2ReaderToEsdb()
filename = args['hash'] + '.s2gs'
gateway = args['gateway']
if gateway == 'sea':
gateway = 'sg'
# retrieve it from battlenet
depoturl = 'http://{0}.depot.battle.net:1119/{1}'.format(
gateway, filename)
try:
s2gsfile = urllib2.urlopen(depoturl).read()
except:
logging.getLogger("jobs").info(
"couldnt retrieve {} s2gs hash {}. maybe its bad.".format(
gateway, args['hash']))
return None
# save it in S3 because we are pack rats
bucket = boto.connect_s3(settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY)\
.get_bucket(settings.S2GS_BUCKET_NAME)
k = Key(bucket)
k.key = filename
k.set_contents_from_string(s2gsfile)
# parse it and write stuff to DB
summaryDB = sc2reader_to_esdb.processSummary(
StringIO(s2gsfile), args['hash'])
except Exception as e:
tb = traceback.format_exc()
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logging.getLogger("jobs").info(
"parsing failed for s2gs {}. oh well. exception={}. {} {} {} {}"
.format(args['hash'], e, exc_type, fname, exc_tb.tb_lineno,
tb))
pass
finally:
# Enqueue ruby PostParse job, always!
ResQ(server=settings.REDIS_SERVER).enqueue_from_string(
'ESDB::Jobs::Sc2::Summary::PostParse', 'summaries-high',
{'hash': args['hash']})
def retry(self, payload_class, args):
"""This method provides a way to retry a job after a failure.
If the jobclass defined by the payload containes a ``retry_every`` attribute then pyres
will attempt to retry the job until successful or until timeout defined by ``retry_timeout`` on the payload class.
"""
retry_every = getattr(payload_class, "retry_every", None)
retry_timeout = getattr(payload_class, "retry_timeout", 0)
if retry_every:
now = ResQ._current_time()
first_attempt = self._payload.get("first_attempt", now)
retry_until = first_attempt + timedelta(seconds=retry_timeout)
retry_at = now + timedelta(seconds=retry_every)
if retry_at < retry_until:
self.resq.enqueue_at(retry_at, payload_class, *args, **{"first_attempt": first_attempt})
return True
return False
def pyres_web():
usage = "usage: %prog [options]"
parser = OptionParser(usage)
parser.add_option("--host",
dest="host",
default="localhost",
metavar="HOST")
parser.add_option("--port", dest="port", type="int", default=8080)
parser.add_option("--dsn", dest="dsn", help="Redis server to display")
parser.add_option("--server",
dest="server",
help="Server for itty to run under.",
default='wsgiref')
(options, args) = parser.parse_args()
if options.dsn:
from pyres import ResQ
resweb_server.HOST = ResQ(options.dsn)
run_itty(host=options.host, port=options.port, server=options.server)
def retry(self, payload_class, args):
"""This method provides a way to retry a job after a failure.
If the jobclass defined by the payload containes a ``retry_every`` attribute then pyres
will attempt to retry the job until successful or until timeout defined by ``retry_timeout`` on the payload class.
"""
retry_every = getattr(payload_class, 'retry_every', None)
retry_timeout = getattr(payload_class, 'retry_timeout', 0)
if retry_every:
now = ResQ._current_time()
first_attempt = self._payload.get("first_attempt", now)
retry_until = first_attempt + timedelta(seconds=retry_timeout)
retry_at = now + timedelta(seconds=retry_every)
if retry_at < retry_until:
self.resq.enqueue_at(retry_at, payload_class, *args,
**{'first_attempt': first_attempt})
return True
return False
def main(params=None):
if params is None:
params = sys.argv[1:]
parser = argparse.ArgumentParser(description='Runs pyres web console.')
conn_group = parser.add_argument_group('Connection arguments')
conn_group.add_argument('--host', default='localhost', help='Binding host')
conn_group.add_argument('--port',
default=8080,
type=int,
help='Binding port')
conn_group = parser.add_argument_group('Redis arguments')
conn_group.add_argument('--redis-host',
default='localhost',
help='Redis host')
conn_group.add_argument('--redis-port',
default=6379,
type=int,
help='Redis port')
conn_group.add_argument('--redis-database',
default=0,
type=int,
help='Redis database')
conn_group.add_argument('--redis-password',
default=None,
help='Redis password')
other_group = parser.add_argument_group('Other arguments')
other_group.add_argument('-l',
'--level',
default='debug',
help='Logging level')
arguments = parser.parse_args(params)
logging.basicConfig(level=getattr(logging, arguments.level.upper()))
redis = Redis(host=arguments.redis_host,
port=arguments.redis_port,
db=arguments.redis_database,
password=arguments.redis_password)
resweb_server.HOST = ResQ(redis)
run_itty(host=arguments.host, port=arguments.port, server='wsgiref')
def run(self):
setproctitle('pyres_minion:%s: Starting' % (os.getppid(), ))
if self.log_path:
if special_log_file(self.log_path):
self.log_file = self.log_path
else:
self.log_file = os.path.join(self.log_path,
'minion-%s.log' % self.pid)
namespace = 'minion:%s' % self.pid
self.logger = setup_logging('minion', namespace, self.log_level,
self.log_file)
#self.clear_logger()
if isinstance(self.server, string_types):
self.resq = ResQ(server=self.server, password=self.password)
elif isinstance(self.server, ResQ):
self.resq = self.server
else:
raise Exception("Bad server argument")
self.work()
def all(cls, resq, start=0, count=1):
"""Get a list of the items in the failure queue.
Redis' documentation: `LLEN <http://redis.io/commands/LLEN>`_
:param resq: The redis queue instance to check
:type resq: :class:`ResQ`
:param start: The location in the queue to start checking at.
:type start: int
:param count: The number of items to retrieve
:type count: int
:returns: A list of items in the queue
:rtype: `list` of `dict`
"""
items = resq.redis.lrange('resque:failed', start, count) or []
ret_list = []
for i in items:
failure = ResQ.decode(i)
failure['redis_value'] = b64encode(i)
ret_list.append(failure)
return ret_list
def perform(fail_until):
if ResQ._current_time() < fail_until:
raise Exception("Don't blame me! I'm supposed to fail!")
else:
return True
def job(self):
data = self.resq.redis.get("resque:worker:%s" % self)
if data:
return ResQ.decode(data)
return {}
