import os
# The main Flask app
app = Flask(__name__)
def get_redis_ip():
ip = os.environ.get('REDIS_IP')
if ip is None:
ip = "127.0.0.1"
return ip
try:
rd = redis.StrictRedis(host=get_redis_ip(), port=6379, db=0)
q = HotQueue("queue", host=get_redis_ip(), port=6379, db=1)
print(rd)
q.put('hello', 'queue')
rd.ping()
print('Connected!')
except Exception as ex:
print('Error:', ex)
exit('Failed to connect, terminating.')
# Data from a json file
data = json.load(open('MSFT.json', 'r'))
df = json_normalize(data)
df['Date'] = df['Date'].astype(str)
@app.route('/')
# Dependencies:
# pip install flask
# pip install redis
import hotqueue
import flask
import redis
import time
import json
from flask import Response, stream_with_context
from flask import Flask, render_template,request,json
from hotqueue import HotQueue
redis_host="localhost"
app = Flask(__name__)
app.debug = True
queue = HotQueue("myqueue", host=redis_host, port=6379, db=0)
@app.route('/post',methods=['POST','GET'])
def post():
_name = request.form['inputName']
_email = request.form['inputEmail']
if _name and _email:
queue.put('{"username" : "'+ _name + '","mail" : "'+ _email +'"}')
return json.dumps({'html':'<span>All fields good !!</span>'})
else:
return json.dumps({'html':'<span>Enter the required fields</span>'})
@app.route("/")
def main():
return render_template('index.html')
import uuid
from hotqueue import HotQueue
from redis import StrictRedis
import os
q = HotQueue("queue", host='172.17.0.1', port=6379, db=1)
rd = StrictRedis(host='172.17.0.1', port=6379, db=0)
def _generate_jid():
return str(uuid.uuid4())
def _generate_job_key(jid):
if type(jid) == bytes:
jid = jid.decode('utf-8')
return 'job.{}'.format(jid)
#changes based on job
def _instantiate_job(jid, status, start, end):
if type(jid) == str:
return {'id': jid,
'status': status,
'start': start,
'end': end
}
return {'id': jid.decode('utf-8'),
'status': status.decode('utf-8'),
'start': start.decode('utf-8'),
'end': end.decode('utf-8')
}
def _save_job(job_key, job_dict):
def setUp(self):
"""Create the queue instance before the test."""
self.queue = HotQueue('testqueue')
import uuid
from hotqueue import HotQueue
from redis import StrictRedis
import os
rd_ip = '10.110.179.174'
q = HotQueue("queue", host=rd_ip, port=6379, db=1)
rd = StrictRedis(host=rd_ip, port=6379, db=0)
def _generate_jid():
return str(uuid.uuid4())
def _generate_job_key(jid):
return 'job.{}'.format(jid)
def _instantiate_job(jid, status, start, end):
if type(jid) == str:
return {'id': jid, 'status': status, 'start': start, 'end': end}
return {
'id': jid.decode('utf-8'),
'status': status.decode('utf-8'),
'start': start.decode('utf-8'),
'end': end.decode('utf-8')
}
def _save_job(job_key, job_dict):
"""Save a job object in the Redis database."""
import ConfigParser
import psycopg2
from hotqueue import HotQueue
config = ConfigParser.ConfigParser()
config.read(['api.conf', 'local_api.conf'])
dbhost = config.get('Database', 'dbhost')
dbname = config.get('Database', 'dbname')
dbuser = config.get('Database', 'dbuser')
dbpass = config.get('Database', 'dbpass')
dbport = config.get('Database', 'dbport')
redisdb = config.get('Redis', 'redishost')
queue = HotQueue("killboard-API", host=redisdb, port=6379, db=0)
if not dbpass:
# Connect without password
dbcon = psycopg2.connect("host=" + dbhost + " user="******" dbname=" + dbname + " port=" + dbport)
else:
dbcon = psycopg2.connect("host=" + dbhost + " user="******" password="******" dbname=" + dbname +
" port=" + dbport)
curs = dbcon.cursor()
curs2 = dbcon.cursor()
curs.execute("""select id from killapi where updtime <= now()""")
for api in curs:
sqlid = api[0]
curs2.execute(
"""update killapi set updtime = (now() + interval '15 minutes') where id = %s""",
Worker then begins waiting for a new job_id.
"""
import os
from time import sleep
from hotqueue import HotQueue
import jobs
from get_db_data import get_db_data
REDIS_IP = os.getenv("REDIS_IP")
REDIS_PORT = os.getenv("REDIS_PORT")
DATA_Q_DB_ID = os.getenv("DATA_Q_DB_ID")
# Waits until redis is in operation and queue can be initialized.
while True:
try:
DATA_Q = HotQueue("data_queue", host=REDIS_IP, port=REDIS_PORT, db=DATA_Q_DB_ID)
DATA_Q.get()
break
except:
print("Error: Redis not yet initialized. Reattempting connection in 3 seconds")
sleep(3)
@DATA_Q.worker
def data_worker(job_id):
"""
Decorator waits for queue to have a job put in it. Once an item
is put in queue the graph worker takes the job_id, gets the job_dict
updates job status, executes the job, updates job's dictionary 'results'
key to the results of the function, updates status again to complete
and prints out "job_id + complete" to console.
from hotqueue import HotQueue
import json
import os
import redis
import matplotlib.pyplot as plt
import subprocess
redis_ip = os.environ.get('REDIS_IP')
if not redis_ip:
raise Exception()
rd = redis.StrictRedis(host=redis_ip, port=6379, db=1, decode_responses=True)
rd_dataset = redis.StrictRedis(host=redis_ip, port=6379, db=0, decode_responses=True)
q = HotQueue('queue', host=redis_ip, port=6379, db=2)
@q.worker
def run_job(job):
subprocess.run(["sleep 1"], shell=True, check=True)
# change job status in db=1 to 'creating'
rd.hset(job, 'status', 'creating')
# get data from job info in db=1
data_byte= rd.hgetall(job)
this_stats = ''
this_specific_stat = ''
for key, value in data_byte.items():
origin_lat = origin_lat.strip(" ")
destination_lng = row[3]
destination_lng = destination_lng.strip(" ")
destination_lat = row[4]
destination_lat = destination_lat.strip(" ")
origin = origin_lng + "," + origin_lat
destination = destination_lng + "," + destination_lat
print origin + " " + destination
queue.put((num, origin, destination))
if __name__ == "__main__":
# do_with_error("error.txt")
#add_to_queue()
redis_config = config["redis"]
queue = HotQueue(name="amap_busline_route", host=redis_config["host"],
port=redis_config["port"], db=redis_config["db"])
threads = []
for api_key in amap_api_key:
thread = Bus_Route(queue=queue, key=api_key)
threads.append(thread)
for thread in threads:
thread.start()
for thread in threads:
thread.join()
# jobs.py
import uuid
from hotqueue import HotQueue
from redis import StrictRedis
import os
q = HotQueue("queue", host='10.97.128.183', port=6379, db=1)
rd = StrictRedis(host='10.97.128.183', port=6379, db=0)
def _generate_jid():
return str(uuid.uuid4())
def _generate_job_key(jid):
return 'job.{}'.format(jid)
def _instantiate_job(jid, status, start, end):
if type(jid) == str:
return {'id': jid, 'status': status, 'start': start, 'end': end}
return {
'id': jid.decode('utf-8'),
'status': status.decode('utf-8'),
'start': start.decode('utf-8'),
'end': end.decode('utf-8')
}
def _save_job(job_key, job_dict):
"""Save a job object in the Redis database."""
import json
import numpy as np
from polyreg import polyreg
from datetime import datetime
from datetime import timedelta
import sys
redis_ip = os.environ.get('REDIS_IP')
redis_port = 6379
if not redis_ip:
raise Exception()
rd1 = StrictRedis(host=redis_ip, port=redis_port, db=1, decode_responses=True) # transaction jobs
rd2 = StrictRedis(host=redis_ip, port=redis_port, db=2, decode_responses=True) # accounts
rd3 = StrictRedis(host=redis_ip, port=redis_port, db=3, decode_responses=False) # graphing jobs
rd4 = StrictRedis(host=redis_ip, port=redis_port, db=5, decode_responses=True) # for displaying jobs
q1 = HotQueue("queue", host=redis_ip, port=redis_port, db=4) # transaction queue
q2 = HotQueue("queue", host=redis_ip, port=redis_port, db=6) # graph queue
def _generate_bid():
"""Create a unique banking ID (account number)."""
bid = str(uuid.uuid1().int)
bid = bid[:12] # banking ID/account number is 12 digits long
return bid
def _generate_jid():
"""Create a unique job ID."""
return str(uuid.uuid4())
def _save_account(bid, account_dict):
from __future__ import absolute_import, division, print_function
from builtins import (bytes, str, open, super, range, zip, round, input, int,
pow, object)
import serial
from hotqueue import HotQueue
from datetime import datetime
ser = serial.Serial('/dev/cu.usbmodem1481', 115200, timeout=1)
queue = HotQueue("serial_message_queue", host="localhost", port=6379, db=0)
#q.put('hello world')
i = 0
#messages = []
message = [
datetime.now(),
]
number = ''
new_message = False
while True:
y = ser.read()
if y == '\r':
new_message = True
queue.put(message)
print(message)
#messages.append(message)
message = [
datetime.now(),
def get_queue(self):
return HotQueue('cheeseshop', serializer=None)
console_logger = logging.StreamHandler()
console_logger.setLevel(logging.DEBUG)
console_logger.setFormatter(formatter)
logging.getLogger(__name__).addHandler(console_logger)
file_logger = logging.FileHandler(filename='errors.log')
file_logger.setLevel(logging.ERROR)
file_logger.setFormatter(formatter)
logging.getLogger(__name__).addHandler(file_logger)
LOG = logging.getLogger(__name__)
LOG.setLevel(logging.DEBUG)
QUEUE = HotQueue("urls",
host=settings.queue.split(':')[0],
port=int(settings.queue.split(':')[1]),
db=int(settings.queue.split(':')[2]))
def async (func):
@wraps(func)
def async_func(*args, **kwargs):
func_hl = Thread(target=func, args=args, kwargs=kwargs)
func_hl.start()
return func_hl
return async_func
@async
def check(url):
from uuid import uuid4
from hotqueue import HotQueue
from redis import StrictRedis
import os
import datetime
import json
q = HotQueue("queue", host='10.104.140.94', port=6379, db=1)
rd = StrictRedis(host='10.104.140.94', port=6379, db=0)
rd2 = StrictRedis(host='10.104.140.94', port=6379, db=3) #jobs
def _generate_jid():
return str(uuid.uuid4())
def generate_job_key(jid):
if type(jid) == bytes:
jid = jid.decode('utf-8')
return 'job.{}'.format(jid)
def _instantiate_job(jid, status, start, end):
if type(jid) == str:
return {'id': jid, 'status': status, 'start': start, 'end': end}
return {
'id': jid.decode('utf-8'),
'status': status.decode('utf-8'),
'start': start.decode('utf-8'),
'end': end.decode('utf-8')
}
import uuid
from hotqueue import HotQueue
import redis
import os
q = HotQueue("queue", host='10.178.131.244', port=6379, db=1)
rd = redis.StrictRedis(host='10.178.131.244', port=6379, db=0)
def _generate_jid():
return str(uuid.uuid4())
def _generate_job_key(jid):
return 'job.{}'.format(jid)
def _instantiate_job(jid, status, start, end):
if type(jid) == str:
return {'id': jid,
'status': status,
'start': start,
'end': end
}
return {'id': jid.decode('utf-8'),
'status': status.decode('utf-8'),
'start': start.decode('utf-8'),
'end': end.decode('utf-8')
}
def _save_job(job_key, job_dict):
"""Save a job object in the Redis database."""
rd.hmset(job_key, job_dict)
import numpy as np
import matplotlib.pyplot as plt
from jobs import update_job_status, current_time, generate_job_key, save_job
from app import get_job_info
import os
import seaborn as sns
import sys
REDIS_IP = os.environ.get('REDIS_IP')
REDIS_PORT = os.environ.get('REDIS_PORT')
#worker
rd = redis.StrictRedis(host=REDIS_IP, port=REDIS_PORT, db=0)
q = HotQueue("queue", host=REDIS_IP, port=REDIS_PORT, db=1)
jl = redis.StrictRedis(host=REDIS_IP, port=REDIS_PORT, db=2)
plots = redis.StrictRedis(host=REDIS_IP, port=REDIS_PORT, db=3)
daily_spots = pd.read_csv('sunspots.csv')
daily_spots.columns = ['Year', 'Mean Daily Spots']
daily_spots = daily_spots.set_index('Year')
@q.worker
def execute_job(jid):
# add stuff here
jobs.update_job_status(jid,"running")
job = app.get_job_info(jid)
command=job["command"]
Please look at previous iterations of this for "original" source code.
'''
# Environment setup
REDIS_IP = os.environ.get('REDIS_IP', '172.17.0.1')
try:
REDIS_PORT = int(os.environ.get('REDIS_PORT'))
except:
REDIS_PORT = 6379
# Redis DBs
DATA_DB = 0
QUEUE_DB = 1
rd = StrictRedis(host=REDIS_IP, port=REDIS_PORT, db=DATA_DB)
q = HotQueue("queue", host=REDIS_IP, port=REDIS_PORT, db=QUEUE_DB)
#rd = StrictRedis(host='172.17.0.1', port=6379, db=0)
#q = HotQueue("queue", host='172.17.0.1', port=6379, db=0)
# creates a unique id for job
def _create_jid():
return str(uuid4())
# adds a 'job.' in front of jid to use as a key
def _create_job_key(jid):
return 'job.{}'.format(jid)
def __init__(self):
super().__init__(self)
self.queue = HotQueue("text-updates")
self.msgsPerFrameLimit = 10
self.setBackgroundColor(0, 0, 0)
self.setFrameRateMeter("fps" in sys.argv)
self.setSceneGraphAnalyzerMeter("debug" in sys.argv)
debugNode = BulletDebugNode('Debug')
debugNode.showWireframe(True)
debugNode.showConstraints(True)
debugNode.showBoundingBoxes(False)
debugNode.showNormals(True)
debugNP = self.render.attachNewNode(debugNode)
if "bullet" in sys.argv: debugNP.show()
self.font = self.loader.loadFont(
'/System/Library/Fonts/Supplemental/Arial Bold.ttf')
self.font.setPixelsPerUnit(60)
self.font.setPageSize(512, 512)
self.font.setRenderMode(TextFont.RMSolid)
self.camera.setPos(0, -90, 0)
self.disableMouse()
self.paused = "pause" in sys.argv
self.world = BulletWorld()
if "gravity" in sys.argv:
self.world.setGravity(Vec3(0, 0, -3))
self.world.setDebugNode(debugNP.node())
self.taskMgr.add(self.update, 'update')
self.boundsNp = self.render.attachNewNode("BoundaryBox")
makeBoundaryBox(self.boundsNp, self.world)
self.textNp = self.render.attachNewNode("TextNodes")
self.cameraCard = CameraCard(self.render)
self.cameraCard.setScale(Vec3(-16, 1, 9) * 4)
self.cameraCard.setTwoSided(True)
self.cameraCard.setPos((8 * 4, 22.9, -4.5 * 4))
if "nocamera" in sys.argv:
self.toggleCameraBg()
light = makeLight(1)
lightNp = render.attachNewNode(light)
lightNp.setPos(24, -30, 12)
self.textNp.setLight(lightNp)
light = makeLight(1)
lightNp = render.attachNewNode(light)
lightNp.setPos(-24, -30, -12)
self.textNp.setLight(lightNp)
ambient = PointLight("Ambient")
ambient.setColor((.5, .5, .5, 1))
ambientNp = render.attachNewNode(ambient)
self.boundsNp.setLight(ambientNp)
self.render.setShaderAuto()
self.render.setAntialias(AntialiasAttrib.MAuto)
if "sample" in sys.argv:
self.sampleTexts = []
for i in range(5):
self.sampleTexts.append(self.addText("Sample %s" % i,
(random() * 10 - 5,
random() * 10 - 5,
random() * 10 - 5)))
self.accept('l', self.sampleLaunch)
self.launchers = defaultdict(
lambda: LaunchableText(self.textNp, self.world, self.font))
self.floaters = []
self.floatersByRb = {}
self.sinks = []
self.sinks.append(self.createSink("A", 0, 22, 0))
#self.createSink("B", 0, -23, 0)
self.accept('1', self.debugNodes)
self.accept('k', self.disableKinematic)
self.accept('c', self.oobe)
self.accept('g', self.toggleGravity)
self.accept('p', self.pause)
self.accept('b', self.toggleCameraBg)
import uuid
from hotqueue import HotQueue
import redis
import os
import requests
import json
import pandas as pd
from datetime import datetime
q = HotQueue("queue", host='10.97.252.127', port=6379, db=1)
rd = redis.StrictRedis(host='10.97.252.127', port=6379, db=0)
def _generate_jid():
return str(uuid.uuid4())
def _generate_job_key(jobid):
if type(jobid) == bytes:
jobid = jobid.decode('utf-8')
return 'job.{}'.format(jobid)
def _instantiate_job(jobid, casenumber, typeofbusiness, typeofcomplaint,
opendate, closedate, outcome, latitude, longitude,
status):
if type(jobid) == str:
return {
'JOBID': jobid,
'CASENUMBER': casenumber,
'TYPEOFBUSINESS': typeofbusiness,
import site
try:
import i_am_kidding_pycharm # For suppress pycharm's import warning
except ImportError:
i_am_kidding_pycharm = None
finally:
site.addsitedir("../") # For absolute import
from hotqueue import HotQueue
from slowworld.common.cache import HOTQUEUE_KEY
from slowworld.conf import worker_type
from slowworld.config import config
queue = HotQueue(HOTQUEUE_KEY, host=config.REDIS_HOST, port=config.REDIS_PORT)
@queue.worker
def job_worker(data):
print(data.get("type"))
if not isinstance(data, dict):
return
if not data.get("type"):
return
result = True
lines = 0
if __name__ == '__main__':
import os
from time import sleep
import statistics
from hotqueue import HotQueue
import jobs
from get_db_data import get_db_data
REDIS_IP = os.getenv("REDIS_IP")
REDIS_PORT = os.getenv("REDIS_PORT")
STAT_Q_DB_ID = os.getenv("STAT_Q_DB_ID")
# Waits until redis is in operation and queue can be initialized.
while True:
try:
STATS_Q = HotQueue("stats_queue",
host=REDIS_IP,
port=REDIS_PORT,
db=STAT_Q_DB_ID)
STATS_Q.get()
break
except:
print(
"Error: Redis not yet initialized. Reattempting connection in 3 seconds"
)
sleep(3)
@STATS_Q.worker
def stats_worker(job_id):
"""
Decorator waits for queue to have a job put in it. Once an item
is put in queue the graph worker takes the job_id, gets the job_dict