class AnmadQueues:
"""Queues used by anmad."""
def __init__(self, prequeue, queue, info):
self.prequeue = HotQueue(prequeue)
self.queue = HotQueue(queue)
self.info = HotQueue(info)
self.update_job_lists()
def update_job_lists(self):
"""Reset queue_message vars."""
self.prequeue_list = []
self.prequeue_list = read_queue(self.prequeue)
self.queue_list = []
self.queue_list = read_queue(self.queue)
self.info_list = []
trim_queue(self.info, 100)
self.info_list = read_queue(self.info)
self.info_list.reverse()
def prequeue_job(self, job):
"""Adds an item to the pre-run queue."""
self.prequeue.put([job])
def queue_job(self, job):
"""Adds an item to the run queue."""
self.queue.put(job)
def clear(self):
"""Clears all job queues."""
self.queue.clear()
self.prequeue.clear()
def clearinfo(self):
"""Clears info queue."""
self.info.clear()
def test_arguments(self):
"""Test that HotQueue.__init__ accepts arguments correctly, and that
the Redis key is correctly formed.
"""
kwargs = {
'name': "testqueue",
'serializer': DummySerializer,
'host': "localhost",
'port': 6379,
'db': 0}
# Instantiate the HotQueue instance:
self.queue = HotQueue(**kwargs)
# Ensure that the properties of the instance are as expected:
self.assertEqual(self.queue.name, kwargs['name'])
self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs['name'])
self.assertEqual(self.queue.serializer, kwargs['serializer'])
self.assertEqual(self.queue._HotQueue__redis.host, kwargs['host'])
self.assertEqual(self.queue._HotQueue__redis.host, kwargs['host'])
self.assertEqual(self.queue._HotQueue__redis.port, kwargs['port'])
self.assertEqual(self.queue._HotQueue__redis.db, kwargs['db'])
# Instantiate a HotQueue instance with only the required args:
self.queue = HotQueue(kwargs['name'])
# Ensure that the properties of the instance are as expected:
self.assertEqual(self.queue.name, kwargs['name'])
self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs['name'])
self.assertTrue(self.queue.serializer is pickle) # Defaults to cPickle or
def add_to_queue():
"""
we use HotQueue to store input info
:return:
"""
with open(r"C:\Users\29625\Desktop\Trans_xtoy_1.csv","r") as f:
reader = csv.reader(f)
redis_config = config["redis"]
queue = HotQueue(name="amap_busline_route", host=redis_config["host"],
port=redis_config["port"], db=redis_config["db"])
print "start adding to queue"
for row in reader:
num = row[0]
origin_lng = row[1]
origin_lng = origin_lng.strip(" ")
origin_lat = row[2]
origin_lat = origin_lat.strip(" ")
destination_lng = row[3]
destination_lng = destination_lng.strip(" ")
destination_lat = row[4]
destination_lat = destination_lat.strip(" ")
origin = origin_lng + ","+origin_lat
destination = destination_lng + ","+destination_lat
print origin + " " + destination
queue.put((num,origin,destination))
print "finished adding to queue"
def make(name, path, sentinals):
q = HotQueue(name)
with open(path) as f:
for line in f.readlines():
q.put(line.strip().split('\t'))
for n in xrange(sentinals):
q.put(None)
def unsubscribe_uuid(cls, uuid):
if True: #TODO: check uuid
queue = HotQueue("connection_thread_id_queue=" + str(g.connection_thread_id))
msg = {'cmd': 'unsubscribe', 'params': str(uuid)}
queue.put(json.dumps(msg))
return msg
else:
return None
def unsubscribe_uuid(cls, uuid):
if True: #TODO: check uuid
queue = HotQueue("connection_thread_id_queue=" +
str(g.connection_thread_id))
msg = {'cmd': 'unsubscribe', 'params': str(uuid)}
queue.put(json.dumps(msg))
return msg
else:
return None
def open(self):
print "Opening"
logging.info("opening")
print "Initializing queue"
queue = HotQueue("myqueue", host="localhost", port=6379, db=0)
count = 0
while count < 100:
count = count + 1
print "adding " + str(count) + " to queue"
queue.put(count)
def open(self):
print "Opening"
logging.info("opening")
print "Initializing queue"
queue = HotQueue("myqueue", host="localhost", port=6379, db=0)
count = 0
while count < 100:
count = count + 1
print "adding " + str(count) + " to queue"
queue.put(count)
def subscribe_greenlet(ps, connection_thread_id):
queue = HotQueue("connection_thread_id_queue=" + str(connection_thread_id))
for msg in queue.consume():
try:
cmd = json.loads(msg)
if cmd['cmd'] == 'subscribe':
ps.subscribe('uuid=' + cmd['params'])
elif cmd['cmd'] == 'subscribe':
ps.unsubscribe('uuid=' + cmd['params'])
except:
pass
def output_category(cat,data):
global redis_enabled, appcfg
pp.pprint(data)
print "{cat} Entries: {size}".format(cat=cat,size=len(data))
if appcfg['redis']:
queue = HotQueue(
appcfg['queue_name'], host=appcfg['redis_host'], port=appcfg['redis_port'], db=appcfg['redis_db']
)
for item in data:
queue.put(item)
def put(queuename,message=None,senderName=None):
returncode = 200
if not message:
message = str(request.form['body'])
if message:
queue = HotQueue(queuename, senderName=senderName, originIPAddress=request.remote_addr, host="localhost", port=6379, db=0)
put_status = HQMessage()
put_status = queue.put(message)[0]
else:
returncode = 400
return put_status.to_json(), returncode
class RedisMQ(object):
def __init__(self, app=None):
if app is not None:
self.init_app(app)
else:
self.app = None
self._tasks = {}
def init_app(self, app):
name = app.config.setdefault('REDISMQ_NAME', 'mq')
host = app.config.setdefault('REDISMQ_HOST', 'localhost')
port = app.config.setdefault('REDISMQ_PORT', 6379)
db = app.config.setdefault('REDISMQ_DB', 0)
password = app.config.setdefault('REDISMQ_PASSWORD', None)
app.config.get('REDISMQ_BLOCK', True),
app.config.get('REDISMQ_BLOCK_TIMEOUT', 0)
if not hasattr(app, 'extensions'):
app.extensions = {}
app.extensions['redismq'] = self
self.app = app
self._hq = HotQueue(name, host=host, port=port, db=db,
password=password)
def task(self, func):
func_name = "%s.%s" % (func.__module__, func.__name__)
self._tasks[func_name] = func
@wraps(func)
def _func(*args, **kwargs):
self._hq.put((func_name, args, kwargs))
setattr(func, "async", _func)
return func
def work(self, *args, **kwargs):
kwargs.update({
'block': self.app.config.get('REDISMQ_BLOCK', True),
'timeout': self.app.config.get('REDISMQ_BLOCK_TIMEOUT', 0)
})
@self._hq.worker(*args, **kwargs)
def _worker(msg):
try:
func_name, args, kwargs = msg
self._tasks[func_name](*args, **kwargs)
except Exception, e:
pass
return _worker()
def subscribe_greenlet(ps, connection_thread_id):
queue = HotQueue("connection_thread_id_queue=" +
str(connection_thread_id))
for msg in queue.consume():
try:
cmd = json.loads(msg)
if cmd['cmd'] == 'subscribe':
ps.subscribe('uuid=' + cmd['params'])
elif cmd['cmd'] == 'subscribe':
ps.unsubscribe('uuid=' + cmd['params'])
except:
pass
def get(queuename):
returncode = 200
return_hqmessage = None
reservation_id = str(uuid4())
queue = HotQueue(queuename, host="localhost", port=6379, db=0)
hqmessage = HQMessage()
hqmessage = queue.get()
if hqmessage:
return_hqmessage = hqmessage.to_json()
else:
returncode = 204
return_hqmessage = ''
return return_hqmessage, returncode
def __init__(self, in_cnt, out_cnt):
self.in_cnt = in_cnt
self.out_cnt = out_cnt
time_uniq = int(time.time())
checksum_queue_key_prefix = "checksum-%d-" % time_uniq
redis_host, redis_port = REDIS_INFO.split(":")
# 生产者客户端连接池，根据同时比较的表数量而定
self.in_queues = [HotQueue(checksum_queue_key_prefix + str(i),
host=redis_host, port=redis_port, db=11) for i in range(in_cnt)]
# 消费者取用队列，对应同等个数的后端消费者
self.out_queues = [HotQueue(checksum_queue_key_prefix + str(i),
host=redis_host, port=redis_port, db=11) for i in range(out_cnt)]
def Main():
config = file_read('config.json')
config = json.loads(config)
HOST = config['mongohost']
PORT = config['port']
DB_NAME = config['database_name']
LISTENER_QUEUE = config['listener_queue']
RESPONSE_QUEUE = config['response_queue']
file_dir = config['repo_directory']
client = MongoClient(HOST,PORT)
db = client[DB_NAME]
listen = HotQueue(LISTENER_QUEUE,serializer=json)
response = HotQueue(RESPONSE_QUEUE,serializer=json)
r = redis.StrictRedis(host='localhost', port=6379, db=0)
print "\nPython-Redis-Listener-Started "
for item in listen.consume():
files = []
for _file in item['files']:
files.append(diff_file(_file['name'],_file['path'],_file['isnew'],_file['tag'],db,item['repoid'],file_dir))
commits = {
'changes' : files,
'desc' : item['desc'],
'created' : datetime.datetime.utcnow(),
#'comment' : [],
'repo' : {'id':ObjectId(item['repoid']),'slug':item['reposlug']},
'user' : {'id':ObjectId(item['userid']),'username':item['username']}
}
commitid = db.commits.insert(commits)
db.repos.update({'_id':commits['repo']['id']},{'$push':{'commits':commitid}})
db.users.update({'_id':commits['user']['id']},{'$push':{'commits':commitid}})
responseobj= {'commitid':str(commitid),'userid':str(commits['user']['id'])}
#response.put(responseobj)
r.publish('cy-pullcommits', json.dumps(responseobj))
print commits
class RedisInput(object):
input_name = "redis"
def __init__(self, host, queue, port=6379):
self.host = host
self.port = port
self.queue = queue
def handle_input(self):
try:
self.queue = HotQueue(self.queue, serializer=json, host=self.host, port=self.port, db=0)
for data in self.queue.consume():
for fmt in self.format_modules:
if fmt.bind and self.input_name in fmt.bind:
data = fmt.decode(data)
self.output_threads.write(data)
except Exception as err:
raise EveConnectionError(err)
def run(self, format_modules, output_modules):
# Start output threads
self.format_modules = format_modules
self.output_modules = output_modules
self.output_threads = OutputThreads(self.output_modules, self.format_modules)
while True:
try:
self.handle_input()
except EveConnectionError as err:
logger.error("connection error in input handler %s: %r - retrying in 1 second" % (self.input_name, err))
sleep(1)
def initQueue(self):
# hotqueue队列字典表
self.q_dict = {}
# 抓取队列
for q in self.q_list:
self.q_dict[q] = HotQueue(q, host=self.redis_ip, port=self.redis_port, password=self.redis_passwd, db=self.QUEUE_DB)
def main():
queue = HotQueue(main_config.INDEX_QUEUE,
host=main_config.REDIS_HOST,
port=main_config.REDIS_PORT)
index = get_index(main_config.WHOOSH_INDEX_DIR)
writer = BufferedWriter(index, limit=10)
try:
for doc_id in queue.consume():
print "looking at {}".format(doc_id)
doc = Document.query.get(doc_id)
if doc:
write_doc(doc, writer)
else:
print "no doc with doc_id {}".format(doc_id)
finally:
writer.close()
def get_queue(name):
"""获取一个指定名称的队列"""
cp = ConfigParser()
cp.read('./config.ini')
host = cp.get('redis', 'host')
password = cp.get('redis', 'password')
return HotQueue(name, host=host, password=password)
def test_from_key(self):
"""Test the HotQueue.from_key static method"""
redis = Redis()
queues = {
'_test:tens': range(10, 20),
'_test:twenties': range(20, 30),
'_test:thirties': range(30, 40)
}
for name, values in queues.items():
q = HotQueue(name)
q.put(*values)
for key in redis.keys('hotqueue:_test:*'):
q = HotQueue.from_key(key)
self.assertEqual(
list(q.consume(block=False)),
queues[q.name]
)
def __init__(self, source):
self.url_table = source + "_url_table"
redis_config = config["redis"]
if redis_config.has_key("password"):
self.queue = HotQueue(name=self.url_table,
host=redis_config["host"],
port=redis_config["port"],
password=redis_config["password"],
db=redis_config["db"])
else:
self.queue = HotQueue(name=self.url_table,
host=redis_config["host"],
port=redis_config["port"],
db=redis_config["db"])
self.sql = "select urlmd5, url, type, boundary from " + self.url_table + " where status=0 limit 5000"
self.update_sql_base = "update " + self.url_table + " set status=100 where urlmd5='%s'"
self.run_sign = True
self.start_feed()
def sent_to_broker(self, newLogString, host="localhost", port=6379, db=1):
'''function sent_to_broker
sent message/text string to broker (rabbitmq, etc)
Keyword Arguments:
newLogString {str} -- [description] (default: {"localhost"})
host {str} -- [description] (default: {"localhost"})
port {number} -- [description] (default: {6379})
db {number} -- [description] (default: {1})
'''
if self.read_config('QUEUE', 'host'):
host = self.read_config('QUEUE', 'host')
if self.read_config('QUEUE', 'port'):
port = int(self.read_config('QUEUE', 'port'))
from hotqueue import HotQueue
queue = HotQueue("logqueue", host=host, port=port, db=db)
queue.put(newLogString)
def main():
pid = os.getpid()
print(f"{pid} starting")
for item in HotQueue(sys.argv[1]).consume():
if item is None:
print(f"{pid} terminating")
return
print(f"{item} printed by {pid}")
time.sleep(2)
def client(self, queue):
"""
Given one of the configured queues returns a
:class:`hotqueue.HotQueue` instance with the
:attr:`~hirefire.procs.hotqueue.HotQueueProc.connection_params`.
"""
if isinstance(queue, HotQueue):
return queue
return HotQueue(queue, **self.connection_params)
def __init__(self, jukebox):
cmd.Cmd.__init__(self)
threading.Thread.__init__(self)
self.jukebox = jukebox
self.playlist = None
self.track = None
self.results = False
self.queue_play = HotQueue("wekkie:play")
def create_queue(data):
try:
data['hotqueue'] = HotQueue(data['queue'],
serializer=json,
host=data['host'],
port=data['port'],
db=0)
except:
return False
else:
return True
def handle_input(self):
try:
self.queue = HotQueue(self.queue, serializer=json, host=self.host, port=self.port, db=0)
for data in self.queue.consume():
for fmt in self.format_modules:
if fmt.bind and self.input_name in fmt.bind:
data = fmt.decode(data)
self.output_threads.write(data)
except Exception as err:
raise EveConnectionError(err)
def queue_processor():
queue = HotQueue("postprocessing_queue")
ocr = OCR.Engine()
for uuid in queue.consume():
print str(uuid)
dm = DocumentManager()
doc = dm.getDocument(uuid)
print str(doc.url)
url = urlparse(doc.url)
filename = os.path.join(UPLOAD_FOLDER, url.path)
m = magic.Magic()
print filename + ' ' + str(m.id_filename(filename))
if 'PDF' in str(m.id_filename(filename)):
pdf_text = PDFProcessor.extractContent(str(filename))
cm = ContentDocument()
cm.content = unicode(pdf_text, encoding='utf-8')
cm.save()
#ocr_text = ocr.interpret(str(filename))
print pdf_text
def queue_processor():
queue = HotQueue("postprocessing_queue")
ocr = OCR.Engine()
for uuid in queue.consume():
print str(uuid)
dm = DocumentManager()
doc = dm.getDocument(uuid)
print str(doc.url)
url = urlparse(doc.url)
filename = os.path.join(UPLOAD_FOLDER, url.path)
m = magic.Magic()
print filename + ' ' + str(m.id_filename(filename))
if 'PDF' in str(m.id_filename(filename)):
pdf_text = PDFProcessor.extractContent(str(filename))
cm = ContentDocument()
cm.content = unicode(pdf_text, encoding='utf-8')
cm.save()
#ocr_text = ocr.interpret(str(filename))
print pdf_text
def test_arguments(self):
"""Test that HotQueue.__init__ accepts arguments correctly, and that
the Redis key is correctly formed.
"""
kwargs = {
'name': "testqueue",
'serializer': DummySerializer,
'host': "localhost",
'port': 6379,
'db': 0}
# Instantiate the HotQueue instance:
self.queue = HotQueue(**kwargs)
# Ensure that the properties of the instance are as expected:
self.assertEqual(self.queue.name, kwargs['name'])
self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs['name'])
self.assertEqual(self.queue.serializer, kwargs['serializer'])
# Instantiate a HotQueue instance with only the required args:
self.queue = HotQueue(kwargs['name'])
# Ensure that the properties of the instance are as expected:
self.assertEqual(self.queue.name, kwargs['name'])
self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs['name'])
self.assertTrue(self.queue.serializer is pickle) # Defaults to cPickle
def __init__(self, interval_upp, interval_down, source,need_proxy=True):
Thread.__init__(self)
redis_config = config["redis"]
url_table = source+"_url_table"
if redis_config.has_key("password"):
self.queue = HotQueue(name=url_table, host=redis_config["host"],
port=redis_config["port"],password=redis_config["password"], db=redis_config["db"])
else:
self.queue = HotQueue(name=url_table, host=redis_config["host"],
port=redis_config["port"], db=redis_config["db"])
self.activity = 0.0
self.spider = None
self.check_round = 20
self.interval_upp = interval_upp #
self.interval_down = interval_down #
self.lock = RLock()
self.tuple_from_queue = ()
self.proxy_manger = ProxyMgr()
self.proxy_ip = self.proxy_manger.get_proxy()[1]
self.CHKTHD = 20
self.proxy = {'http': 'http://' + self.proxy_ip}
self.need_proxy = need_proxy
def configure(self, gconfig={}, **options):
if self.running:
raise SchedulerAlreadyRunningError
config = combine_opts(gconfig, 'main.', options)
self._config = config
self.misfire_grace_time = int(config.pop('misfire_grace_time', 1))
self.coalesce = asbool(config.pop('coalesce', True))
self.daemonic = asbool(config.pop('daemonic', True))
self.standalone = asbool(config.pop('standalone', False))
timezone = config.pop('timezone', None)
self.timezone = gettz(timezone) if isinstance(timezone, basestring) else timezone or tzlocal()
# config threadpool
threadpool_opts = combine_opts(config, 'threadpool.')
self._worker_threadpool = ThreadPool(**threadpool_opts)
# config jobstore
jobstore_opts = combine_opts(config, 'jobstore.')
self._job_store = SQLAlchemyJobStore(**jobstore_opts)
# config syncqueue
syncqueue_opts = combine_opts(config, 'syncqueue.')
self._changes_queue = HotQueue(**syncqueue_opts)
# config statstore
statstore_opts = combine_opts(config, 'statstore.')
self._stat_store = JobReporter(**statstore_opts)
# config statqueue
statqueue_opts = combine_opts(config, 'statqueue.')
self._stats_queue = HotQueue(**statqueue_opts)
# configure logger
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.DEBUG)
def acknack(reservation_uuid):
returncode = 200
returnmsg = ''
hq = HotQueue(host="localhost", port=6379, db=0)
msg = None
nack = False
if 'nack' in request.path:
nack = True
if 'PUT' in request.method:
nack = True
if nack:
msg = hq.nack(reservation_uuid)
else:
msg = hq.ack(reservation_uuid)
if msg:
hqmessage = HQMessage()
hqmessage = msg
returnmsg = hqmessage.to_json()
else:
returncode = 400
return returnmsg, returncode
def do_with_error(path):
redis_config = config["redis"]
queue = HotQueue(name="amap_busline_route", host=redis_config["host"],
port=redis_config["port"], db=redis_config["db"])
with open(r"C:\Users\29625\Desktop\Trans_xtoy_1.csv", "r") as f_1:
reader = csv.reader(f_1)
reader = list(reader)
with open(path, 'r') as f:
lines = f.readlines()
for line in lines:
num = line.strip("\n")
row = reader[int(num)-1]
origin_lng = row[1]
origin_lng = origin_lng.strip(" ")
origin_lat = row[2]
origin_lat = origin_lat.strip(" ")
destination_lng = row[3]
destination_lng = destination_lng.strip(" ")
destination_lat = row[4]
destination_lat = destination_lat.strip(" ")
origin = origin_lng + "," + origin_lat
destination = destination_lng + "," + destination_lat
print origin + " " + destination
queue.put((num, origin, destination))
def init_app(self, app):
name = app.config.setdefault('REDISMQ_NAME', 'mq')
host = app.config.setdefault('REDISMQ_HOST', 'localhost')
port = app.config.setdefault('REDISMQ_PORT', 6379)
db = app.config.setdefault('REDISMQ_DB', 0)
password = app.config.setdefault('REDISMQ_PASSWORD', None)
app.config.get('REDISMQ_BLOCK', True),
app.config.get('REDISMQ_BLOCK_TIMEOUT', 0)
if not hasattr(app, 'extensions'):
app.extensions = {}
app.extensions['redismq'] = self
self.app = app
self._hq = HotQueue(name, host=host, port=port, db=db,
password=password)
def init_workers(num):
global scout, worker, parser
# q contains list of threads, scraped by the scout
q = HotQueue(home)
# qf contains list of tuples of len 3
# contains source code of scraped pages and metadata, for parser
# (subforum name, link of post, post source)
qf = HotQueue(home + "_sources")
# boolean variable that describes whether the Parser is training
# (only used for GenericParser)
qt = Value('b', False)
workers = []
scout = multiprocessing.Process(target=scout, args=(q,))
scout.start()
logger.info("Starting scout process %s" % str(scout.pid))
workers.append(scout)
sleep(15)
for i in xrange(num):
tmp = multiprocessing.Process(target=worker, args=(q,qf,qt,delay, delay_range))
tmp.start()
logger.info("Starting worker process %s" % str(tmp.pid))
workers.append(tmp)
for i in xrange(num-1):
tmp = multiprocessing.Process(target=parser, args=(qf,qt))
tmp.start()
logger.info("Starting parser process %s" % str(tmp.pid))
workers.append(tmp)
parser(qf,qt, main=True)
logger.info("Using main process as parser")
for worker in workers:
worker.join()
logger.info("All done.")
q.clear()
qf.clear()
def main():
parser = OptionParser(usage="usage: %prog [options]",
version="%prog 1.0")
parser.add_option("-f", "--filename",
dest="filename",
help="Use prerecorded audio file")
parser.add_option("-s", "--string",
dest="string",
help="Use custom string",)
(options, args) = parser.parse_args()
text = None
if options.string:
text = options.string
elif options.filename:
text = googlevoice.recognize(options.filename)
else:
# Record audio:
var = raw_input("Druk <Enter> om de opname te starten: ")
filename = "tmp/recordings/%s.wav" % base64.b64encode(os.urandom(6)).replace("/", "")
soundrecorder.record(filename)
text = googlevoice.recognize(filename)
if text:
text = text.lower()
time = parsetime.parse(text)
artist_uri = parseartist.parse(text)
# TODO: read Spotify user from properties
default_uri = "spotify:user:bertspaan:starred"
if not artist_uri:
artist_uri = default_uri
print "Alarm:", time, "with", artist_uri
if time:
queue_time = HotQueue("wekkie:time")
queue_time.put(time)
if artist_uri:
queue_spotify_uri = HotQueue("wekkie:uri")
queue_spotify_uri.put(artist_uri)
def main(argv) :
global q
featureName = argv[1]
featureStatus = argv[2]
featureServer = argv[3]
q = HotQueue(('server_'+featureName), host="localhost", port=6379, db=0)
r_server = redis.Redis('localhost')
if featureStatus == "ON":
q.put('http://127.0.0.1:'+featureServer)
if(not r_server.exists(featureName)):
r_server.set(featureName, '1')
else:
r_server.incr(featureName)
print 'set key value is: ' + r_server.get(featureName)
else :
k = list()
while(1) :
x = q.get()
print "Queue get : ", x
if x is None :
break;
else :
if x == ('http://127.0.0.1:'+featureServer) :
print("Removing : ",x)
if(r_server.get(featureName) > 0):
r_server.decr(featureName)
print 'set key value is: ' + r_server.get(featureName)
break
else:
k.append(x)
while len(k) != 0:
x = k.pop(0)
q.put(x)
printQueue()
print "Queue name : ",('server_'+featureName)
DEBUG = config.getboolean('Consumer', 'debug')
TERM_OUT = config.getboolean('Consumer', 'term_out')
mcserver = config.get('Memcache', 'server')
mckey = config.get('Memcache', 'key')
statkey = config.get('Memcache', 'statkey')
# Max number of greenlet workers
MAX_NUM_POOL_WORKERS = 75
# item list of stuff we want immediately updated in stats
fastupdate = [34, 35, 36, 37, 38, 39, 40, 29668]
# use a greenlet pool to cap the number of workers at a reasonable level
greenlet_pool = Pool(size=MAX_NUM_POOL_WORKERS)
queue = HotQueue("emdr-messages", host=redisdb, port=6379, db=0)
statqueue = HotQueue("e43-stats", host=redisdb, port=6379, db=0)
# Handle DBs without password
if not dbpass:
# Connect without password
dbcon = psycopg2.connect("host=" + dbhost + " user="******" dbname=" + dbname + " port=" + dbport)
else:
dbcon = psycopg2.connect("host=" + dbhost + " user="******" password="******" dbname=" + dbname +
" port=" + dbport)
#connect to memcache
mc = pylibmc.Client([mcserver],
binary=True,
from hotqueue import HotQueue
import redis
import os
from uuid import uuid4
redis_ip = os.environ.get('REDIS_IP')
if not redis_ip:
raise Exception()
app = Flask(__name__)
rd = redis.StrictRedis(host=redis_ip, port=6379, db=1)
rd_dataset = redis.StrictRedis(host=redis_ip,
port=6379,
db=0,
decode_responses=True)
q = HotQueue('queue', host=redis_ip, port=6379, db=2)
@app.route('/', methods=['GET'])
def instructions():
return """
Try these routes:
/ informational
/run (GET) job instructions
/run/<stats>/<specific_stat> (POST) submit job
/jobs get list of past jobs
/jobs/<UUID> get job status
/delete (GET) delete instructions
/delete (DELETE) delete job
/download/<UUID> download img from job
cursor = db1.cursor()
sql = 'CREATE DATABASE IF NOT EXISTS ' + database_name
print(sql)
cursor.execute(sql)
db1.commit()
db1.close()
db2 = MySQLdb.connect(
host=dbhost, # your host
user=dbuser, # username
passwd=dbpass, # password
db=database_name) # name of the database
# Create a Cursor object to execute queries.
cur = db2.cursor()
sql = """CREATE TABLE IF NOT EXISTS USERS (
NAME CHAR(20) NOT NULL,
EMAIL CHAR(50) )"""
# Select data from table using SQL query.
cur.execute(sql)
queue = HotQueue("myqueue", host=redis_host, port=6379, db=0)
for item in queue.consume():
print(item)
js = json.loads(item)
sql = "INSERT INTO USERS(NAME,EMAIL) VALUES ('%s', '%s' )" % (
js['username'], js['mail'])
cur.execute(sql)
db2.commit()
db2.close()
class LocalScheduler(object):
_stopped = False
_main_thread = None
#init worker thread pool，reporter thread，updater thread
def __init__(self, gconfig={}, **options):
self._wakeup = Event()
self._job_store = None
self._stat_store = None
self._jobs = {}
self.logger = None
self._stats_queue = None
self._changes_queue = None
self._jobs_locks = {}
self._jobs_lock = Lock()
self._log_queue_lock = Lock()
self._worker_threadpool = None
self._reporter_thread = None
self._main_thread = None
self._updater_thread = None
self._monitor_thread = None
self.configure(gconfig, **options)
def configure(self, gconfig={}, **options):
if self.running:
raise SchedulerAlreadyRunningError
config = combine_opts(gconfig, 'main.', options)
self._config = config
self.misfire_grace_time = int(config.pop('misfire_grace_time', 1))
self.coalesce = asbool(config.pop('coalesce', True))
self.daemonic = asbool(config.pop('daemonic', True))
self.standalone = asbool(config.pop('standalone', False))
timezone = config.pop('timezone', None)
self.timezone = gettz(timezone) if isinstance(timezone, basestring) else timezone or tzlocal()
# config threadpool
threadpool_opts = combine_opts(config, 'threadpool.')
self._worker_threadpool = ThreadPool(**threadpool_opts)
# config jobstore
jobstore_opts = combine_opts(config, 'jobstore.')
self._job_store = SQLAlchemyJobStore(**jobstore_opts)
# config syncqueue
syncqueue_opts = combine_opts(config, 'syncqueue.')
self._changes_queue = HotQueue(**syncqueue_opts)
# config statstore
statstore_opts = combine_opts(config, 'statstore.')
self._stat_store = JobReporter(**statstore_opts)
# config statqueue
statqueue_opts = combine_opts(config, 'statqueue.')
self._stats_queue = HotQueue(**statqueue_opts)
# configure logger
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.DEBUG)
def start(self):
if self.running:
raise SchedulerAlreadyRunningError
self.load_jobs()
self._stopped = False
if self.standalone:
self._main_loop()
else:
self._main_thread = Thread(target = self._main_loop, name = 'main')
self._main_thread.setDaemon(self.daemonic)
self._main_thread.start()
print 'main thread is startted'
self._updater_thread = Thread(target = self._sync_changes, name = 'update')
self._updater_thread.setDaemon(self.daemonic)
self._updater_thread.start()
print 'update thread is started'
self._stater_thread = Thread(target = self._stat_runs, name = 'stat')
self._stater_thread.setDaemon(self.daemonic)
self._stater_thread.start()
print 'stat thread is started'
def shutdown(self, shutdown_threadpool=True, close_jobstore=True):
if not self.running:
return
self._stopped = True
self._wakeup.set()
if shutdown_threadpool:
self._worker_threadpool.shutdown()
if self._main_thread:
self._main_thread.join()
if close_jobstore:
self._job_store.close()
@property
def running(self):
return not self._stopped and self._main_thread and self._main_thread.isAlive()
def now(self):
return datetime.now(self.timezone)
def set_jobs(self, jobs):
now = self.now()
with self._jobs_lock:
for job in jobs:
job.compute_next_run_time(now)
self._jobs[job.id] = job
self._jobs_locks[job.id] = Lock()
# loads jobs pool from db
def load_jobs(self):
jobs = self._job_store.load_jobs()
now = self.now()
with self._jobs_lock:
for job in jobs:
self._add_job(job)
def _add_job(self, job):
try:
now = self.now()
job.compute_next_run_time(now)
if job.next_run_time:
self._jobs[job.id] = job
self._jobs_locks[job.id] = Lock()
except:
logger.exception("add job(id=%d, name=%s) failed" % (job.id, job.name))
return False
return True
def _remove_job(self, job_id):
try:
with self._jobs_locks[job_id]:
del self._jobs[job_id]
del self._jobs_locks[job_id]
except:
logger.exception("remove job(id=%d) failed" % (job_id))
return False
return True
def _main_loop(self):
print "get into the main loop"
self._wakeup.clear()
while not self._stopped:
print 'check again'
now = self.now()
next_wakeup_time = self._process_jobs(now)
print "next_wakeup_time:", next_wakeup_time
if next_wakeup_time is not None:
wait_seconds = time_difference(next_wakeup_time, now)
self._wakeup.wait(wait_seconds)
self._wakeup.clear()
else:
self._wakeup.wait()
self._wakeup.clear()
print "get out the main loop"
def _process_jobs(self, now):
next_wakeup_time = None
print self._jobs
for job in self._jobs.values():
run_time_list = job.get_run_times(now)
if run_time_list:
self._worker_threadpool.submit(self._run_job, job, run_time_list)
with self._jobs_locks[job.id]:
next_run_time = job.compute_next_run_time(now + timedelta(microseconds=1))
if not next_run_time:
self._remove_job(job.id)
print 'job.next_run_time:', job.id, job.next_run_time
if not next_wakeup_time:
next_wakeup_time = job.next_run_time
elif job.next_run_time:
next_wakeup_time = min(next_wakeup_time, job.next_run_time)
return next_wakeup_time
def _run_job(self, job, run_time_list):
for run_time in run_time_list:
now = self.now()
difference = now - run_time
grace_time = timedelta(seconds=self.misfire_grace_time)
if difference > grace_time:
self.logger.warning('Run time of job "%s" was missed by %s', job, difference)
self._put_stat(job.id, 'missed', next_run_time=job.next_run_time)
else:
try:
# maybe add a timeout handle by join thread.
# t = Thread(job.run); t.start(); t.join(timeout)
# refer: http://augustwu.iteye.com/blog/554827
self._put_stat(job.id, 'running', next_run_time=job.next_run_time)
result = job.run()
print 'job runned success'
cost = self.now() - now
self._put_stat(job.id, 'succed', cost=cost)
except:
self.logger.exception('Job "%s" raised an exception', job)
cost = self.now() - now
self._put_stat(job.id, 'failed', cost=cost)
if self.coalesce:
break
def _put_stat(self, job_id, status, next_run_time=None, cost=timedelta(seconds=0)):
msg = {
'time': pickle.dumps(self.now()),
'job_id': job_id,
'status': status,
'next_run_time':pickle.dumps(next_run_time),
'cost': cost.total_seconds() + cost.microseconds / 1000000
}
try:
self._stats_queue.put(msg)
except:
logger.exception('failed to put stat item ' + json.dumps(msg))
def _stat_runs(self):
while not self._stopped:
try:
msg = self._stats_queue.get(block=True, timeout=1)
except:
logger.exception('get stat item failed')
msg = None
if not msg:
continue
try:
msg["time"] = pickle.loads(msg['time'])
msg["next_run_time"] = pickle.loads(msg['next_run_time'])
self._stat_store.report(**msg)
except:
traceback.print_exc()
logger.exception('report job status failed ' + pickle.dumps(msg))
def _sync_changes(self):
count = 0
max_items_once = int(self._config.pop('max_items_once', 0))
while not self._stopped:
try:
msg = self._changes_queue.get(block=True, timeout=1)
except:
logger.exception('get sync item failed')
msg = None
if msg:
opt_type = msg['opt_type']
job_id = msg['job_id']
if job_id > 0 and isinstance(opt_type, basestring):
try:
self._apply_change(opt_type, job_id)
except:
pass
self.logger.info('apply change "%s" for job(%d)', opt_type, job_id)
count += 1
if not msg or (max_items_once > 0 and count > max_items_once):
if count > 0:
self.logger.info('wakeup main thread by sync thread with %d updates' % count)
self._wakeup.set()
count = 0
def _apply_change(self, opt_type, job_id):
if opt_type == 'add' or opt_type == 'update':
try:
job = self._job_store.get_job(job_id)
except Exception as e:
self.logger.exception(e)
if job:
if opt_type == 'add':
if not self._jobs.has_key(job_id):
self._add_job(job)
else:
logger.exception("apply channge '%s job(id=%d, name=%s)' failed" % (opt_type, job.id, job.name))
else:
#!todo check if compute next_run_time again is necessary
now = self.now()
job.compute_next_run_time(now)
with self._jobs_locks[job_id]:
self._jobs[job_id] = job
elif opt_type == 'delete' or opt_type == 'pause':
self._remove_job(job_id)
else:
self.logger.exception('opt %s job(%d) to jobs pool is not supported' % (opt_type, job_id))
def setUp(self):
"""Create the queue instance before the test."""
self.queue = HotQueue('testqueue')
class JukeboxUI(cmd.Cmd, threading.Thread):
prompt = "jukebox> "
def __init__(self, jukebox):
cmd.Cmd.__init__(self)
threading.Thread.__init__(self)
self.jukebox = jukebox
self.playlist = None
self.track = None
self.results = False
self.queue_play = HotQueue("wekkie:play")
def run(self):
container_loaded.wait()
container_loaded.clear()
print "Starting consume loop..."
for uri in self.queue_play.consume():
if uri:
print "Try to play", uri
link = Link.from_string(uri)
if link.type() == Link.LINK_TRACK:
self.jukebox.load_track(link.as_track())
self.jukebox.play()
elif link.type() == Link.LINK_ARTIST:
browser = ArtistBrowser(link.as_artist())
while not browser.is_loaded():
time.sleep(0.1)
toptracks = browser.tophit_tracks()
print toptracks
#random.shuffle(toptracks)
track = choice(toptracks)
self.jukebox.load_track(track)
self.jukebox.play()
elif link.type() == 7: #Link.LINK_PLAYLIST:
playlist = self.jukebox.starred
track = choice(playlist)
while track.availability() != 1:
track = choice(playlist)
self.jukebox.load_track(track)
self.jukebox.play()
else:
self.jukebox.stop()
try:
self.cmdloop()
finally:
self.do_quit(None)
def do_logout(self, line):
self.jukebox.session.logout()
def do_quit(self, line):
self.jukebox.stop()
self.jukebox.disconnect()
print "Goodbye!"
return True
def do_list(self, line):
""" List the playlists, or the contents of a playlist """
if not line:
for i, p in enumerate(self.jukebox.ctr):
if p.is_loaded():
print "%3d %s" % (i, p.name())
else:
print "%3d %s" % (i, "loading...")
print "%3d Starred tracks" % (i + 1,)
else:
try:
p = int(line)
except ValueError:
print "that's not a number!"
return
if p < 0 or p > len(self.jukebox.ctr):
print "That's out of range!"
return
print "Listing playlist #%d" % p
if p < len(self.jukebox.ctr):
playlist = self.jukebox.ctr[p]
else:
playlist = self.jukebox.starred
for i, t in enumerate(playlist):
if t.is_loaded():
print "%3d %s - %s [%s]" % (
i, t.artists()[0].name(), t.name(),
self.pretty_duration(t.duration()))
else:
print "%3d %s" % (i, "loading...")
def pretty_duration(self, milliseconds):
seconds = milliseconds // 1000
minutes = seconds // 60
seconds = seconds % 60
duration = '%02d:%02d' % (minutes, seconds)
return duration
def do_play(self, line):
if not line:
self.jukebox.play()
return
if line.startswith("spotify:"):
# spotify url
l = Link.from_string(line)
if not l.type() == Link.LINK_TRACK:
print "You can only play tracks!"
return
self.jukebox.load_track(l.as_track())
else:
try:
playlist, track = map(int, line.split(' ', 1))
self.jukebox.load(playlist, track)
except ValueError:
try:
playlist = int(line)
self.jukebox.load_playlist(playlist)
except ValueError:
print("Usage: play [track_link] | "
"[playlist] [track] | [playlist]")
return
self.jukebox.play()
def do_browse(self, line):
if not line or not line.startswith("spotify:"):
print "Invalid id provided"
return
l = Link.from_string(line)
if not l.type() in [Link.LINK_ALBUM, Link.LINK_ARTIST]:
print "You can only browse albums and artists"
return
def browse_finished(browser, userdata):
print "Browse finished, %s" % (userdata)
self.jukebox.browse(l, browse_finished)
def print_search_results(self):
print "Artists:"
for a in self.results.artists():
print " ", Link.from_artist(a), a.name()
print "Albums:"
for a in self.results.albums():
print " ", Link.from_album(a), a.name()
print "Tracks:"
for a in self.results.tracks():
print " ", Link.from_track(a, 0), a.name()
print self.results.total_tracks() - len(self.results.tracks()), \
"Tracks not shown"
def do_search(self, line):
if not line:
if self.results is False:
print "No search is in progress"
elif self.results is None:
print "Searching is in progress"
else:
print "Artists:"
for a in self.results.artists():
print " ", Link.from_artist(a), a.name()
print "Albums:"
for a in self.results.albums():
print " ", Link.from_album(a), a.name()
print "Tracks:"
for a in self.results.tracks():
print " ", Link.from_track(a, 0), a.name()
print self.results.total_tracks() - \
len(self.results.tracks()), "Tracks not shown"
else:
line = line.decode('utf-8')
self.results = None
def search_finished(results, userdata):
print "\nSearch results received"
self.results = results
self.print_search_results()
self.jukebox.search(line, search_finished)
def do_queue(self, line):
if not line:
for playlist, track in self.jukebox._queue:
print playlist, track
return
try:
playlist, track = map(int, line.split(' ', 1))
except ValueError:
print "Usage: play playlist track"
return
self.jukebox.queue(playlist, track)
def do_stop(self, line):
self.jukebox.stop()
def do_next(self, line):
self.jukebox.next()
def emptyline(self):
pass
def do_watch(self, line):
if not line:
print """Usage: watch [playlist]
You will be notified when tracks are added, moved or removed from the
playlist."""
else:
try:
p = int(line)
except ValueError:
print "That's not a number!"
return
if p < 0 or p >= len(self.jukebox.ctr):
print "That's out of range!"
return
self.jukebox.watch(self.jukebox.ctr[p])
def do_unwatch(self, line):
if not line:
print "Usage: unwatch [playlist]"
else:
try:
p = int(line)
except ValueError:
print "That's not a number!"
return
if p < 0 or p >= len(self.jukebox.ctr):
print "That's out of range!"
return
self.jukebox.watch(self.jukebox.ctr[p], True)
def do_toplist(self, line):
usage = "Usage: toplist (albums|artists|tracks) (GB|FR|..|all|current)"
if not line:
print usage
else:
args = line.split(' ')
if len(args) != 2:
print usage
else:
self.jukebox.toplist(*args)
def do_shell(self, line):
self.jukebox.shell()
def do_add_new_playlist(self, line):
if not line:
print "Usage: add_new_playlist <name>"
else:
new_playlist = self.jukebox.ctr.add_new_playlist(
line.decode('utf-8'))
def do_add_to_playlist(self, line):
usage = "Usage: add_to_playlist <playlist_index> <insert_point>" + \
" <search_result_indecies>"
if not line:
print usage
return
args = line.split(' ')
if len(args) < 3:
print usage
else:
if not self.results:
print "No search results"
else:
index = int(args.pop(0))
insert = int(args.pop(0))
artists = self.results.artists()
tracks = self.results.tracks()
for i in args:
for a in tracks[int(i)].artists():
print u'{0}. {1} - {2} '.format(
i, a.name(), tracks[int(i)].name())
print u'adding them to {0} '.format(
self.jukebox.ctr[index].name())
self.jukebox.ctr[index].add_tracks(
insert, [tracks[int(i)] for i in args])
do_ls = do_list
do_EOF = do_quit
def clear_queue():
q = HotQueue(home)
qf = HotQueue(home + "_sources")
q.clear()
qf.clear()
os.remove(pfile)
hdir = "./" + home
if save_files and not os.path.isdir(hdir):
os.mkdir(hdir)
pfile = hdir[2:] + ".p"
try:
with open(pfile, "r") as f:
state = cPickle.load(f)
if args.generic:
generic = True
P = GenericParser(name=home,save=True)
except:
# no pickle file; fresh start
state = [0, 0]
q = HotQueue(home)
qf = HotQueue(home + "_sources")
q.clear()
qf.clear()
if args.generic:
generic = True
P = GenericParser(name=home,save=False)
if args.vbulletin:
vbulletin = True
P = vBulletinParser()
init_logger()
temp = urlparse.urljoin("http:////", home)
from hotqueue import HotQueue
from apscheduler.job import Job
from apscheduler.jobstores.sqlalchemy_store import SQLAlchemyJobStore
from apscheduler.triggers import IntervalTrigger
from apscheduler.scripts import HttpScript
if __name__ == '__main__':
script = HttpScript(url='http://baidu.com')
local_tz = gettz('Asia/Chongqing')
defaults = {'timezone': local_tz}
trigger = IntervalTrigger(defaults, seconds=3)
store = SQLAlchemyJobStore(url='sqlite:////tmp/task.db', tablename='tasks')
job = store.get_job(3)
if not job:
job = Job(id=3, name='BaiduCheck', script=script, trigger=trigger)
store.add_job(job)
print job
job.trigger = IntervalTrigger(defaults, seconds=5)
store.update_job(job)
queue = HotQueue('job_changes')
queue.put({'job_id':job.id, 'opt_type':'update'})
def start_server(host='localhost', port=8081, IPv6=False, timeout=60,
handler=ConnectionHandler):
global stop
global session
global sequence
global login
global html_response,role_id,authorized,login_url,logout_url,num_roles
final_sequence=[]
final_session=[]
if IPv6==True:
soc_type=socket.AF_INET6
else:
soc_type=socket.AF_INET
soc = socket.socket(soc_type)
soc.bind((host, port))
print "Serving on %s:%d."%(host, port)#debug
soc.listen(0)
print 'Please browse like an unauthorized user first'
while num_roles!=0:
thread.start_new_thread(handler, soc.accept()+(timeout,))
#print " Real Stop=",str(stop)
stop=False
if stop:
#writedata()
if '' in sequence:
del sequence['']
#print sequence
for x in sequence:
if x in sequence[x]:
if sequence[x]==[x]:
pass
else:
sequence[x].remove(x)
final_session.append(session)
final_sequence.append(sequence)
session={}
sequence={}
time.sleep(2)
ch=raw_input('Press y if you have more roles to explore...,press c to change login url')
if ch=='y' or ch=='c':
stop=False
authorized=True
role_id+=1
if ch=='c':
login_url=raw_input('Please enter the login url:')
logout_url=raw_input('Please enter the logout url:')
print final_session
print final_sequence
print params
print login
f=open("responses.txt",'w')
for i in html_response:
print i
f.write(i)
f.write("\n\n")
f.write(html_response[i])
f.close()
fileObject = open("db",'wb')
pickle.dump([state_set,transition_set,login],fileObject)
queue = HotQueue("myqueue", host="localhost", port=6379, db=0)
queue.put(state_set)
queue.put(transition_set)
queue.put(login)
queue.put(login_url)
queue.put(logout_url)
fileObject.close()
def get(self):
queue = HotQueue("myqueue", host="localhost", port=6379, db=0)
self.write('Hello from tornado' + str(queue.get()))
class HotQueueTestCase(unittest.TestCase):
def setUp(self):
"""Create the queue instance before the test."""
self.queue = HotQueue('testqueue')
def tearDown(self):
"""Clear the queue after the test."""
self.queue.clear()
def test_arguments(self):
"""Test that HotQueue.__init__ accepts arguments correctly, and that
the Redis key is correctly formed.
"""
kwargs = {
'name': "testqueue",
'serializer': DummySerializer,
'host': "localhost",
'port': 6379,
'db': 0}
# Instantiate the HotQueue instance:
self.queue = HotQueue(**kwargs)
# Ensure that the properties of the instance are as expected:
self.assertEqual(self.queue.name, kwargs['name'])
self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs['name'])
self.assertEqual(self.queue.serializer, kwargs['serializer'])
# Instantiate a HotQueue instance with only the required args:
self.queue = HotQueue(kwargs['name'])
# Ensure that the properties of the instance are as expected:
self.assertEqual(self.queue.name, kwargs['name'])
self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs['name'])
self.assertTrue(self.queue.serializer is pickle) # Defaults to cPickle
# or pickle, depending
# on the platform.
def test_consume(self):
"""Test the consume generator method."""
nums = [1, 2, 3, 4, 5, 6, 7, 8]
# Test blocking with timeout:
self.queue.put(*nums)
msgs = []
for msg in self.queue.consume(timeout=1):
msgs.append(msg)
self.assertEqual(msgs, nums)
# Test non-blocking:
self.queue.put(*nums)
msgs = []
for msg in self.queue.consume(block=False):
msgs.append(msg)
self.assertEqual(msgs, nums)
def test_cleared(self):
"""Test for correct behaviour if the Redis list does not exist."""
self.assertEqual(len(self.queue), 0)
self.assertEqual(self.queue.get(), None)
def test_get_order(self):
"""Test that messages are get in the same order they are put."""
alphabet = ['abc', 'def', 'ghi', 'jkl', 'mno']
self.queue.put(alphabet[0], alphabet[1], alphabet[2])
self.queue.put(alphabet[3])
self.queue.put(alphabet[4])
msgs = []
msgs.append(self.queue.get())
msgs.append(self.queue.get())
msgs.append(self.queue.get())
msgs.append(self.queue.get())
msgs.append(self.queue.get())
self.assertEqual(msgs, alphabet)
def test_length(self):
"""Test that the length of a queue is returned correctly."""
self.queue.put('a message')
self.queue.put('another message')
self.assertEqual(len(self.queue), 2)
def test_worker(self):
"""Test the worker decorator."""
colors = ['blue', 'green', 'red', 'pink', 'black']
# Test blocking with timeout:
self.queue.put(*colors)
msgs = []
@self.queue.worker(timeout=1)
def appender(msg):
msgs.append(msg)
appender()
self.assertEqual(msgs, colors)
# Test non-blocking:
self.queue.put(*colors)
msgs = []
@self.queue.worker(block=False)
def appender(msg):
msgs.append(msg)
appender()
self.assertEqual(msgs, colors)
# Test decorating a class method:
self.queue.put(*colors)
msgs = []
class MyClass(object):
@self.queue.worker(block=False)
def appender(self, msg):
msgs.append(msg)
my_instance = MyClass()
my_instance.appender()
self.assertEqual(msgs, colors)
def test_threaded(self):
"""Threaded test of put and consume methods."""
msgs = []
def put():
for num in range(3):
self.queue.put('message %d' % num)
sleep(0.1)
def consume():
for msg in self.queue.consume(timeout=1):
msgs.append(msg)
putter = threading.Thread(target=put)
consumer = threading.Thread(target=consume)
putter.start()
consumer.start()
for thread in [putter, consumer]:
thread.join()
self.assertEqual(msgs, ["message 0", "message 1", "message 2"])
def test_custom_serializer(self):
"""Test the use of a custom serializer and None as serializer."""
msg = "my message"
# Test using None:
self.queue.serializer = None
self.queue.put(msg)
self.assertEqual(self.queue.get().decode(), msg)
self.queue.put({"a": 1})
expected = "{u'a': 1}" if sys.version_info[0] == 2 else "{'a': 1}"
self.assertEqual(self.queue.get().decode(), expected) # Should be a string
# Test using DummySerializer:
self.queue.serializer = DummySerializer
self.queue.put(msg)
self.assertEqual(self.queue.get().decode(), "foo")
redisdb = config.get('Redis', 'redishost')
apiServer = config.get('API', 'host')
mcserver = config.get('Memcache', 'server')
mckey = config.get('Memcache', 'key')
psource = config.get('Pricing', 'source')
ecapi = config.get('Pricing', 'echost')
e43api = config.get('Pricing', 'e43host')
psqlhost = config.get('Pricing', 'psqlhost')
psqlname = config.get('Pricing', 'psqlname')
psqluser = config.get('Pricing', 'psqluser')
psqlpass = config.get('Pricing', 'psqlpass')
psqlport = config.get('Pricing', 'psqlport')
MAX_NUM_POOL_WORKERS = 75
queue = HotQueue("killboard-API", host=redisdb, port=6379, db=0)
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.DEBUG)
# use a greenlet pool to cap the number of workers at a reasonable level
greenlet_pool = Pool(size=MAX_NUM_POOL_WORKERS)
def main():
for message in queue.consume():
greenlet_pool.spawn(worker, message)
def priceCheck(typeID):
typeID = int(typeID)
logging.debug("Updating mineral prices for %i" % (typeID))
mc = pylibmc.Client([mcserver], binary=True, behaviors={"tcp_nodelay": True, "ketama": True})
if mckey + "price" + str(typeID) in mc:
class HotQueueTestCase(unittest.TestCase):
def setUp(self):
"""Create the queue instance before the test."""
self.queue = HotQueue('testqueue')
def tearDown(self):
"""Clear the queue after the test."""
self.queue.clear()
def test_arguments(self):
"""Test that HotQueue.__init__ accepts arguments correctly, and that
the Redis key is correctly formed.
"""
kwargs = {
'name': "testqueue",
'serializer': DummySerializer,
'host': "localhost",
'port': 6379,
'db': 0}
# Instantiate the HotQueue instance:
self.queue = HotQueue(**kwargs)
# Ensure that the properties of the instance are as expected:
self.assertEqual(self.queue.name, kwargs['name'])
self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs['name'])
self.assertEqual(self.queue.serializer, kwargs['serializer'])
self.assertEqual(self.queue._HotQueue__redis.host, kwargs['host'])
self.assertEqual(self.queue._HotQueue__redis.host, kwargs['host'])
self.assertEqual(self.queue._HotQueue__redis.port, kwargs['port'])
self.assertEqual(self.queue._HotQueue__redis.db, kwargs['db'])
# Instantiate a HotQueue instance with only the required args:
self.queue = HotQueue(kwargs['name'])
# Ensure that the properties of the instance are as expected:
self.assertEqual(self.queue.name, kwargs['name'])
self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs['name'])
self.assertTrue(self.queue.serializer is pickle) # Defaults to cPickle or
# pickle, depending on the
# platform.
def test_consume(self):
"""Test the consume generator method."""
nums = [1, 2, 3, 4, 5, 6, 7, 8]
# Test blocking with timeout:
self.queue.put(*nums)
msgs = []
for msg in self.queue.consume(timeout=1):
msgs.append(msg)
self.assertEquals(msgs, nums)
# Test non-blocking:
self.queue.put(*nums)
msgs = []
for msg in self.queue.consume(block=False):
msgs.append(msg)
self.assertEquals(msgs, nums)
def test_cleared(self):
"""Test for correct behaviour if the Redis list does not exist."""
self.assertEquals(len(self.queue), 0)
self.assertEquals(self.queue.get(), None)
def test_get_order(self):
"""Test that messages are get in the same order they are put."""
alphabet = ['abc', 'def', 'ghi', 'jkl', 'mno']
self.queue.put(alphabet[0], alphabet[1], alphabet[2])
self.queue.put(alphabet[3])
self.queue.put(alphabet[4])
msgs = []
msgs.append(self.queue.get())
msgs.append(self.queue.get())
msgs.append(self.queue.get())
msgs.append(self.queue.get())
msgs.append(self.queue.get())
self.assertEquals(msgs, alphabet)
def test_length(self):
"""Test that the length of a queue is returned correctly."""
self.queue.put('a message')
self.queue.put('another message')
self.assertEquals(len(self.queue), 2)
def test_worker(self):
"""Test the worker decorator."""
colors = ['blue', 'green', 'red', 'pink', 'black']
# Test blocking with timeout:
self.queue.put(*colors)
msgs = []
@self.queue.worker(timeout=1)
def appender(msg):
msgs.append(msg)
appender()
self.assertEqual(msgs, colors)
# Test non-blocking:
self.queue.put(*colors)
msgs = []
@self.queue.worker(block=False)
def appender(msg):
msgs.append(msg)
appender()
self.assertEqual(msgs, colors)
def test_threaded(self):
"""Threaded test of put and consume methods."""
msgs = []
def put():
for num in range(3):
self.queue.put('message %d' % num)
sleep(0.1)
def consume():
for msg in self.queue.consume(timeout=1):
msgs.append(msg)
putter = threading.Thread(target=put)
consumer = threading.Thread(target=consume)
putter.start()
consumer.start()
for thread in [putter, consumer]:
thread.join()
self.assertEqual(msgs, ["message 0", "message 1", "message 2"])
def test_custom_serializer(self):
"""Test the use of a custom serializer."""
self.queue.serializer = DummySerializer
phrase = "my message"
self.queue.put(phrase)
msg = self.queue.get()
self.assertEqual(msg, phrase)
def setUp(self):
"""Create the queue instance before the test."""
self.queue = HotQueue('testqueue')
class HotQueueTestCase(unittest.TestCase):
def setUp(self):
"""Create the queue instance before the test."""
self.queue = HotQueue('testqueue')
def tearDown(self):
"""Clear the queue after the test."""
self.queue.clear()
def test_arguments(self):
"""Test that HotQueue.__init__ accepts arguments correctly, and that
the Redis key is correctly formed.
"""
kwargs = {
'name': "testqueue",
'serializer': DummySerializer,
'host': "localhost",
'port': 6379,
'db': 0}
# Instantiate the HotQueue instance:
self.queue = HotQueue(**kwargs)
# Ensure that the properties of the instance are as expected:
self.assertEqual(self.queue.name, kwargs['name'])
self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs['name'])
self.assertEqual(self.queue.serializer, kwargs['serializer'])
# Instantiate a HotQueue instance with only the required args:
self.queue = HotQueue(kwargs['name'])
# Ensure that the properties of the instance are as expected:
self.assertEqual(self.queue.name, kwargs['name'])
self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs['name'])
self.assertTrue(self.queue.serializer is pickle) # Defaults to cPickle
# or pickle, depending
# on the platform.
def test_consume(self):
"""Test the consume generator method."""
nums = [1, 2, 3, 4, 5, 6, 7, 8]
# Test blocking with timeout:
nums_added = self.queue.put(*nums)
msgs = []
for msg in self.queue.consume(timeout=1):
msgs.append(msg)
self.assertEqual(msgs, nums_added)
for msg in msgs:
self.queue.ack(msg.get_reservationId())
# Test non-blocking:
nums_added = self.queue.put(*nums)
msgs = []
for msg in self.queue.consume(block=False):
msgs.append(msg)
self.assertEqual(msgs, nums_added)
for msg in msgs:
self.queue.ack(msg.get_reservationId())
def test_nack(self):
"""Test the consume generator method."""
nums_added = self.queue.put("1")
msg = self.queue.get() #1
self.assertEqual(msg, nums_added[0])
self.queue.nack(msg.get_reservationId())
msg = self.queue.get() #2
self.assertEqual(msg, nums_added[0])
self.queue.nack(msg.get_reservationId())
msg = self.queue.get() #3
if msg:
self.assertEqual(msg, nums_added[0])
self.queue.ack(msg.get_reservationId())
self.assertEqual(msg, nums_added[0])
self.assertEqual(len(self.queue), 0)
self.assertEqual(msg.get_deliveryCount(),3) #3
def test_cleared(self):
"""Test for correct behaviour if the Redis list does not exist."""
self.assertEqual(len(self.queue), 0)
self.assertEqual(self.queue.get(), None)
def test_get_order(self):
"""Test that messages are get in the same order they are put."""
alphabet = ['abc', 'def', 'ghi', 'jkl', 'mno']
msg_added = []
msg_added.extend(self.queue.put(alphabet[0], alphabet[1], alphabet[2]))
msg_added.extend(self.queue.put(alphabet[3]))
msg_added.extend(self.queue.put(alphabet[4]))
msgs = []
msgs.append(self.queue.get())
msgs.append(self.queue.get())
msgs.append(self.queue.get())
msgs.append(self.queue.get())
msgs.append(self.queue.get())
self.assertEqual(msgs, msg_added)
for msg in msgs:
self.queue.ack(msg.get_reservationId())
def test_length(self):
"""Test that the length of a queue is returned correctly."""
self.queue.put('a message')
self.queue.put('another message')
self.assertEqual(len(self.queue), 2)
def test_worker(self):
"""Test the worker decorator."""
colors = ['blue', 'green', 'red', 'pink', 'black']
# Test blocking with timeout:
msg_added = self.queue.put(*colors)
msgs = []
@self.queue.worker(timeout=1)
def appender(msg):
msgs.append(msg)
appender()
self.assertEqual(msgs, msg_added)
for msg in msgs:
self.queue.ack(msg.get_reservationId())
# Test non-blocking:
msg_added = self.queue.put(*colors)
msgs = []
@self.queue.worker(block=False)
def appender(msg):
msgs.append(msg)
appender()
self.assertEqual(msgs, msg_added)
for msg in msgs:
self.queue.ack(msg.get_reservationId())
# Test decorating a class method:
msg_added = self.queue.put(*colors)
msgs = []
class MyClass(object):
@self.queue.worker(block=False)
def appender(self, msg):
msgs.append(msg)
my_instance = MyClass()
my_instance.appender()
self.assertEqual(msgs, msg_added)
for msg in msgs:
self.queue.ack(msg.get_reservationId())
def test_threaded(self):
"""Threaded test of put and consume methods."""
msgs = []
msg_added = []
def put():
for num in range(3):
msg_added.extend(self.queue.put('message %d' % num))
sleep(0.1)
def consume():
for msg in self.queue.consume(timeout=1):
msgs.append(msg)
putter = threading.Thread(target=put)
consumer = threading.Thread(target=consume)
putter.start()
consumer.start()
for thread in [putter, consumer]:
thread.join()
self.assertEqual(msgs, msg_added)
for msg in msgs:
self.queue.ack(msg.get_reservationId())
#!/usr/bin/python
#
import re, os
from datetime import datetime, date, time
import redis
import time
import traceback
from hotqueue import HotQueue
#make redis comnnection
r_server = redis.Redis(host='localhost', port=6380, db=0)
r_server_raw = redis.Redis(host='localhost', port=6379, db=0)
queue = HotQueue("logqueue", host="localhost", port=6379, db=1)
lineNo = 0
def matchmyregex(line):
expireParsedLog = 432000 # 5 days
expireIPtoMAC = 259200 # 3 days
expireRAWlogs = 95040 # 1.1 days
if not REGM_oldfw.search(line):
#generate uniq key to combine multiple loglines from same STB (mac + datetime)
macaddr = REGEXmac.findall(line)
datetimeUnix = REGEXdatetimeServer.findall(line)[0]
if macaddr:
macNoDelimtTMP = macaddr[0].replace(':', "")
macNoDelimt = macNoDelimtTMP.replace(" ", "")
import sys
import os
from os.path import dirname, join
sys.path.insert(0, dirname(__file__))
activate_this = join(dirname(__file__), '.env', 'bin', 'activate_this.py')
execfile(activate_this, dict(__file__=activate_this))
from web.config import config, r
from hotqueue import HotQueue
from web.job import read_job, JobObj
from time import time, sleep
# start a thread for listing qlog
qlog = HotQueue(
'joblog',
host=config.get('redis', 'host'),
port=config.getint('redis', 'port'),
password=config.get('redis', 'password'),
db=0)
qlog.put({'cmd': 'purge'})
def loop_handle(log):
if 'cmd' not in log:
return
cmd = log['cmd']
if 'host' in log:
r.set('host:{}:last_alive'.format(log['host']), time())
if cmd in ('available', 'busy'):
r.set('host:{}:status'.format(log['host']), cmd)
scout = multiprocessing.Process(target=scout, args=(q, ))
scout.start()
print "Starting scout process %s" % str(scout.pid)
workers.append(scout)
for i in xrange(num):
tmp = multiprocessing.Process(target=worker, args=(q, ))
tmp.start()
print "Starting worker process %s" % str(tmp.pid)
workers.append(tmp)
for worker in workers:
worker.join()
print "All workers done."
return q.empty()
if len(sys.argv) < 2:
print "Usage: python archives.py link"
q = JoinableQueue()
queue = HotQueue("links")
home = sys.argv[1]
hdir = "./" + re.sub("^http://", "", home)
if not os.path.isdir(hdir):
os.mkdir(hdir)
archive_link = urlparse.urljoin(home, "/archive/index.php")
print archive_link
#atexit.register(save_state)
init_workers(5, q)
from __future__ import absolute_import, division, print_function
from builtins import bytes, str, open, super, range, zip, round, input, int, pow, object
import serial
from hotqueue import HotQueue
from datetime import datetime
ser = serial.Serial("/dev/cu.usbmodem1481", 115200, timeout=1)
queue = HotQueue("serial_message_queue", host="localhost", port=6379, db=0)
# q.put('hello world')
i = 0
# messages = []
message = [datetime.now()]
number = ""
new_message = False
while True:
y = ser.read()
if y == "\r":
new_message = True
queue.put(message)
print(message)
# messages.append(message)
message = [datetime.now()]
elif y == " ":
if number != "":
message.append(float(number))
