def get_stations():
"""Setup function to populate stations table if no data exists. Should only
be run once."""
resp = requests.get(bike_data_url).text
hashed = hashlib.sha256(resp).hexdigest()
redis.set('station_data', hashed)
redis.set('bike_data', hashed)
stations_data = json.loads(resp)
for s in stations_data:
latitude, longitude = s['location']
station = Station(id=s['id'],
station_status=s['station_stocking_status'],
name=s['name'],
description=s['description'],
has_kiosk=s['has_kiosk'],
has_ccreader=s['has_ccreader'],
station_type=s['type'],
latitude=latitude,
longitude=longitude)
db.session.add(station)
soup = BeautifulSoup(s['popup'], 'html.parser')
bikes = int(soup.select('span.station-bikes b')[0].string)
docks = int(soup.select('span.station-docks b')[0].string)
redis.hmset(s['name'], {'bikes': bikes, 'docks': docks})
redis.geoadd('stations', latitude, longitude, s['name'])
db.session.commit()
def dss_isa_callback(id):
''' This is the call back end point that other USSes in the DSS network call once a subscription is updated '''
if requires_scope('dss.write.identification_service_areas'):
new_flights_url = request.args.get('flights_url', 0)
try:
assert new_flights_url != 0
redis = redis.Redis(host=app.config['REDIS_HOST'],
port=app.config['REDIS_PORT'])
# Get the flights URL from the DSS and put it in
flights_dict = redis.hgetall("all_uss_flights")
all_flights_url = flights_dict['all_flights_url']
all_flights_url = all_flights_url.append(new_flights_url)
flights_dict["all_uss_flights"] = all_flights_url
redis.hmset("all_uss_flights", flights_dict)
except AssertionError as ae:
return Response("Incorrect data in the POST URL",
status=400,
mimetype='application/json')
else:
# All OK return a empty response
return Response("", status=204, mimetype='application/json')
raise AuthError(
{
"code": "Unauthorized",
"description": "You don't have access to this resource"
}, 403)
def route_post_ad(slot):
if not advertiser_id():
return '', 404
asset = request.files['asset']
id = next_ad_id()
key = ad_key(slot, id)
type = fetch(request.form, 'type')
if not type:
type = asset.mimetype
if not type:
type = 'video/mp4'
redis = get_redis()
redis.hmset(
key, {
'slot': slot,
'id': id,
'title': fetch(request.form, 'title'),
'type': type,
'advertiser': advertiser_id(),
'destination': fetch(request.form, 'destination'),
'impressions': 0
})
redis.set(asset_key(slot, id), asset.read())
redis.rpush(slot_key(slot), id)
redis.sadd(advertiser_key(advertiser_id()), key)
return jsonify(get_ad(slot, id))
async def get_card_price(client, card, redis):
price = card.get('price') or None
timestamp = card.get('timestamp') or (time.time() - 60 * 60 * 24)
if not price or float(timestamp) < (time.time() - 60 * 60 * 12):
url = 'http://pokeprices.doeiqts.com/api/getcard'
params = {'cardset': card['set'].lower(), 'cardnumber': card['number']}
price = 'N/A'
async with client.get(url, params=params) as response:
result = await response.text()
try:
result = json.loads(result)
except:
result = {}
if result.get('status', '').lower() == 'success':
try:
price = float(result['cards'][0]['price'])
except (ValueError, IndexError, KeyError):
price = 'N/A'
timestamp = time.time()
if price != 'N/A':
redis.hmset(card['id'], {'price': price, 'timestamp': timestamp})
return {
'price':
float(price) if price != 'N/A' else price,
'available':
60,
'timestamp':
datetime.datetime.fromtimestamp(
float(timestamp)).strftime('%I:%M %p EST'),
}
def main():
config = ConfigParser.ConfigParser()
if len(config.read(['/home/pi/lol/conf.ini'])):
api_key = config.get("user", "api_key")
redis_host = config.get("database", "redis_host")
redis_port = config.get("database", "redis_port")
redis = setRedisConn(host=redis_host, port=redis_port)
last_match = redis.lindex("last100", 0)
if last_match is None: # for brand-new database
match_id = getLastMatch(region, seed_pid="4460427", api_key=api_key)
else:
match_id = increaseId(last_match)
region = "kr"
while True:
# crawl the match data
match_details = getMatchDetails(region=region, match_id=match_id, api_key=api_key)
if match_details is not None:
# insert match data into redis
mid = match_details.pop("matchId")
redis.hmset(mid, match_details)
# update list of last-inserted
redis.lpush("last100", mid)
redis.ltrim("last100", 0, 99)
match_id = increaseId(match_id)
time.sleep(1) # to avoid api overshooting (max 500 queries per 10 min)
else:
print "File conf.ini not found. Program aborted."
exit(1)
def save_all(redis, iterable):
try:
delete_all(redis, REDIS_PREFIX)
for dict in iterable:
redis.hmset(REDIS_PREFIX + '@'.join(dict.values()), dict)
except redis.exceptions.ResponseError:
print('Redis server is not ready. [Skip]')
def setRedis(keyname,data,sessionID):
inData = pa.serialize(data).to_buffer()
compressLength = len(inData)
inDataCompress= pa.compress(inData,asbytes=True)
inDataDict = {'compressLength':compressLength,'inDataCompress':inDataCompress}
keyDict = {'key':f"{keyname}Cache{sessionID}"}
redis.hmset(keyDict['key'],inDataDict)
def save_all(redis, iterable):
try:
delete_all(redis, REDIS_PREFIX)
for dict in iterable:
redis.hmset(REDIS_PREFIX + '@'.join(dict.values()) ,dict)
except redis.exceptions.ResponseError:
print('Redis server is not ready. [Skip]')
def hashes_redis():
import redis
redis = redis.Redis(host='127.0.0.1',
port=6379,
db=0,
charset="utf-8",
decode_responses=True)
print("-------------")
print("HASH")
print("-------------")
#hmset(name, mapping) hget(name, key) hgetall(name)
redis.hmset("user.1", {"name": "peter", "email": "*****@*****.**"})
print("map.1: ", redis.hgetall("user.1"))
print("name.1:", redis.hget("user.1", "name"))
print("email.1:", redis.hget("user.1", "email"))
print("-------------")
#hset(key, field, value) hget()
redis.hset("user.2", "name1", "peter")
print("map.2: ", redis.hgetall("user.2"))
print("type map.2: ", redis.type("user.2"))
print("name.2:", redis.hget("user.2", "name1"))
print("-------------")
#delete all keys
for key in redis.scan_iter("prefix:*"):
redis.delete(key)
def save(self, openid):
mapping = {
'identity_url': openid.identity_url,
'fullname': openid.fullname,
'nickname': openid.nickname,
'email': openid.email
}
redis.hmset(self.key, mapping)
def save(self, openid):
mapping = {
'identity_url': openid.identity_url,
'fullname': openid.fullname,
'nickname': openid.nickname,
'email': openid.email
}
redis.hmset(self.key, mapping)
def tags():
if request.method == 'POST':
redis.hmset(HASH,
{k: v
for (k, v) in request.form.items() if k in KEYS})
return "success"
elif request.method == 'GET':
return repr(redis.hgetall(HASH))
else:
return "error"
def get_file(self, fid, stop_event, driver=None, restart=False):
"""Transfers a file from a Driver to another.
"""
redis = self.plug.redis
metadata = Metadata.get_by_id(self.plug, fid)
filename = metadata.filename
transfer_key = 'drivers:{}:transfers:{}'.format(self.plug.name, fid)
if driver:
redis.sadd('drivers:{}:transfers'.format(self.plug.name), fid)
redis.hmset(transfer_key, {'from': driver, 'offset': 0})
offset = 0
self.logger.info("Starting to get '{}' from {}", filename, driver)
else:
transfer = redis.hgetall(transfer_key)
driver = transfer['from']
offset = int(transfer['offset'])
self.logger.info("Restarting transfer of '{}' from {}", filename,
driver)
dealer = self.context.socket(zmq.DEALER)
port = redis.get('drivers:{}:router'.format(driver))
dealer.connect('tcp://localhost:{}'.format(port))
end = metadata.size
chunk_size = self.plug.options.get('chunk_size', 1 * 1024 * 1024)
if not restart:
self._call('start_upload', metadata)
while offset < end:
if stop_event.is_set():
# another transaction for the same file has
# probably started
self.logger.info("Aborting transfer of '{}' from {}", filename,
driver)
return
dealer.send_multipart((filename, str(offset), str(chunk_size)))
chunk = dealer.recv()
self.logger.debug("Received chunk of size {} from {} for '{}'",
len(chunk), driver, filename)
self._call('upload_chunk', filename, offset, chunk)
offset = redis.hincrby(transfer_key, 'offset', len(chunk))
self._call('end_upload', metadata)
redis.delete(transfer_key)
redis.srem('drivers:{}:transfers'.format(self.plug.name), fid)
self.logger.info("Transfer of '{}' from {} successful", filename,
driver)
def new_speaker():
check_post_param('name')
new_name = request.form['name']
rand_bits = random.getrandbits(SPEAKER_HASH_LENGTH)
rand_hash = base64.b64encode(bytes(str(rand_bits), 'ascii'))
prefix = new_name[:min(4, len(new_name))]
new_id = "{}:{}".format(prefix, rand_hash.decode('utf-8'))
redis.sadd(USER_IDS_SET, new_id)
redis.hmset(hm_data(new_id), {USER_NAME: new_name})
return new_id
def new_speaker():
check_post_param('name')
new_name = request.form['name']
rand_bits = random.getrandbits(SPEAKER_HASH_LENGTH)
rand_hash = base64.b64encode(bytes(str(rand_bits), 'ascii'))
prefix = new_name[:min(4, len(new_name))]
new_id = "{}:{}".format(prefix, rand_hash.decode('utf-8'))
redis.sadd(USER_IDS_SET, new_id)
redis.hmset(hm_data(new_id), {USER_NAME: new_name})
return new_id
def test_get_by_category(redis):
def get_by_category(client, category):
ids = client.lrange('category:' + category, 0, -1)
items = []
for id in ids:
items.append(client.hget(id, 'name'))
return items
redis.hmset('item:1', {'name': 'Foo', 'category': 'Bar'})
redis.lpush('category:Bar', 'item:1')
assert get_by_category(redis, 'Bar') == ['Foo']
def store_mail(mail):
try:
for line in mail:
mail_parser.feed(line)
email = mail_parser.close()
redis.incr('mail:id')
id = redis.get('mail:id')
redis.hmset(id, {'To:':email['To'],'From:':email['From'], 'Date:':email['Date'], 'Message:':email.get_payload()})
except:
print 'Unexpected error:', sys.exc_info()[0]
exit(1)
return 'Message was stored successfully'
def command_sequence(script_name, command, redis):
command = command.encode('utf-8')
#retrieving existing data
s_dict = redis.hgetall(script_name)
if s_dict is not None:
s_dict['msg'] = command
else:
s_dict = {}
s_dict['msg'] = command
#set new message for script queue
redis.hmset(script_name, s_dict)
def reply_text_and_get_user_profile(event):
# 取出消息內User的資料
################### 須作例外處理
user_profile = line_bot_api.get_profile(event.source.user_id)
# 將用戶資訊做成合適Json
user_info = {
"user_open_id": user_profile.user_id,
"user_nick_name": user_profile.display_name,
"user_status": "",
"user_img": user_profile.picture_url,
"user_register_menu": menu_id
}
#將json傳回API Server
Endpoint = 'http://%s:5001/users' % (ip_location)
#Endpoint='http://%s:5001/users' % (ip_location)
Header = {'Content-Type': 'application/json'}
Response = requests.post(Endpoint,
headers=Header,
data=json.dumps(user_info))
# 先暫時寫的，用來檢查錯誤情形，方便除錯
print(Response)
print(Response.text)
# 將菜單綁定在用戶身上
linkMenuEndpoint = 'https://api.line.me/v2/bot/user/%s/richmenu/%s' % (
user_profile.user_id, menu_id)
linkMenuRequestHeader = {
'Content-Type': 'image/jpeg',
'Authorization': 'Bearer %s' % secretFile["channel_access_token"]
}
lineLinkMenuResponse = requests.post(linkMenuEndpoint,
headers=linkMenuRequestHeader)
print(lineLinkMenuResponse)
print(lineLinkMenuResponse.text)
# 回覆文字消息與圖片消息
line_bot_api.reply_message(
event.reply_token, TextSendMessage(text="請使用下方功能選單\n或是輸入下方字串\ndetail"))
#再跟老師討論存在redis的值有沒有需要進去mysql
redis.hmset(user_profile.user_id, {
'result': 0,
"total": 0,
"sa_qid": 0,
"dev_qid": 0,
"sys_qid": 0
})
def do_report(pipeline, redis):
process_report = {
'id': pipeline_id,
'hostname': hostname,
'fqdn': fqdn,
'pid': pid,
'version': VERSION,
'mem_usage': psutil.virtual_memory().percent,
'disk_usage': psutil.disk_usage(pipeline.data_dir).percent
}
redis.hmset(pipeline_id, process_report)
redis.sadd('pipelines', pipeline_id)
def retry(gist_id, error=None):
gist_key = "gist:#"+str(gist_id)
last_status = redis.hmget(gist_key, 'status')[0]
retry_count = redis.hmget(gist_key, 'retry_count')[0]
if retry_count is None or int(retry_count) < 10:
redis.hmset(gist_key, dict(status='ERR', message=error))
redis.hincrby(gist_key, 'retry_count')
return 'ERR'
else:
redis.hmset(gist_key, dict(status='DEAD'))
return 'DEAD'
def callback(ch, method, properties, body):
print("Receiver_three got the message!")
print(body.decode('utf-8'))
index, text, mes_time = get_values(body.decode('utf-8'))
time.sleep(int(mes_time))
redis.hmset(f"MESG_{index}",
{"TEXT": text,
"TIME": mes_time,
"I": index})
ch.basic_ack(delivery_tag=method.delivery_tag)
def learn_speaker():
check_post_param('id')
user_id = request.form['id']
filename = user_id + next(tempfile._get_candidate_names()) + '.wav'
request.files['wav_sample'].save(filename)
user_training = None
if redis.hexists(hm_data(user_id), USER_TRAINING):
user_training = pickle.loads(redis.hget(hm_data(user_id), USER_TRAINING))
gmm, mfccs = predict.learn(filename, user_training)
redis.hmset(hm_data(user_id), {USER_TRAINING: pickle.dumps(mfccs), USER_MODEL: pickle.dumps(gmm)})
os.remove(filename)
return str(len(mfccs))
def add_update():
if (request.method == 'POST'):
userData = {}
data = request.get_json()
spotlights = data['spotlights']
data['spotlights'] = str(spotlights).strip('[]').replace("'",
"").replace(
" ", "")
for value in ('password', 'username', 'spotlights'):
userData[value] = data[
value] if value != 'password' else sha256.hash(data[value])
redis.hmset(data['username'], userData)
return jsonify({'message': 'Saved!'})
def update_topic_partition(redis, tpos, key_path):
"""
Updates last read topicpartition offsets in redis
"""
from itertools import groupby
key_func = lambda x: x['topic']
for topic, group in groupby(tpos, key_func):
po = {}
for el in group:
po[str(el['partition'])] = int(el['offset'] + 1)
po_str = json.dumps(po)
redis.hmset(key_path, {topic: po_str})
def save_cookie_to_redis(self, redis, backend, hname):
"""
save the cookie to redis
Args:
redis - the redis.Redis object
backend - the backend name
hname - the name of hash
"""
servers = self.get_backend_servers(backend)
cookies = {}
for server in servers:
if len(server['cookie']) > 0:
cookies[server['addr']] = server['cookie']
if len(cookies) > 0: redis.hmset(hname, cookies)
def setup_redis():
if not redis.exists(switch_key):
ret = redis.hmset(switch_key, {oj: 1 for oj in SUPPORT_OJ})
if ret:
logger.info('setup switch key success')
else:
log_spider_status()
def learn_speaker():
check_post_param('id')
user_id = request.form['id']
filename = user_id + next(tempfile._get_candidate_names()) + '.wav'
request.files['wav_sample'].save(filename)
user_training = None
if redis.hexists(hm_data(user_id), USER_TRAINING):
user_training = pickle.loads(
redis.hget(hm_data(user_id), USER_TRAINING))
gmm, mfccs = predict.learn(filename, user_training)
redis.hmset(hm_data(user_id), {
USER_TRAINING: pickle.dumps(mfccs),
USER_MODEL: pickle.dumps(gmm)
})
os.remove(filename)
return str(len(mfccs))
def clear_sign(redis):
""" 清除每日领取救济金 """
now_day = convert_util.to_week_day(datetime.now())
print now_day
if now_day == 1: #周一重置签到
now_sign_id = redis.get(FISH_TABLE_NOW_RUN)
sign_table = redis.hgetall(FISH_SIGN_TABLE % (now_sign_id))
for day, val in sign_table.items():
print day, val
if day == 'title':
continue
val = eval(val)
val['taked'] = []
sign_table[day] = val
#重置签到
redis.hmset(FISH_SIGN_TABLE % (now_sign_id), sign_table)
def update_bike_counts():
"""Job function to update bike and dock counts for all stations on a
schedule."""
log.info('Updating bike counts.')
resp = requests.get(bike_data_url).text
hashed = hashlib.sha256(resp).hexdigest()
if hashed == redis.get('bike_data'):
log.info('Dataset has not changed.')
return
else:
redis.set('bike_data', hashed)
stations_data = json.loads(resp)
for s in stations_data:
soup = BeautifulSoup(s['popup'], 'html.parser')
bikes = int(soup.select('span.station-bikes b')[0].string)
docks = int(soup.select('span.station-docks b')[0].string)
redis.hmset(s['name'], {'bikes': bikes, 'docks': docks})
def repet_change():
a = 0
b = 0
for key in redis.scan_iter("jobs_*"):
item = redis.hgetall(key)
string = item["job_name"]+item["company_name"]+item["office_address"]+item["description"]
sha256 = hashlib.sha256(string).hexdigest()
ret = redis.sadd("check_repetition", sha256)
if ret == 0:
b += 1
print("find one repetition")
item["repetition"] = True
redis.hmset(key, item)
a += 1
if a in range(0, 1000000, 1000):
print(a)
print(a)
print(b)
def enqueue():
data = json.loads(request.data.decode())
if 'input_url' not in data:
response = {
'error':
"The Youtube URL to download must be provided as 'input_url'",
}
logger.warn("Rejecting /api/enqueue request missing 'input_url'")
return json.dumps(response), 400 # bad request
clean_url = util.validate_url(data['input_url'])
if clean_url is None:
response = {
'error':
"I'm sorry, that doesn't really look like a Youtube URL. :-(",
'info':
"Please try again using a link starting with 'https://www.youtube.com'.",
}
logger.warn("Rejecting /api/enqueue request for %s" %
data['input_url'])
return json.dumps(response), 403 # forbidden
logger.info("Accepting /api/enqueue request for %s" % clean_url)
job = rqueue.enqueue_call(
func=util.download,
args=(clean_url, ),
result_ttl=900 # 15 minutes
)
job_id = job.get_id()
redis.lpush(joblist, job_id)
redis.ltrim(joblist, 0, 9)
job_details = {
'job_id': job_id,
'request_url': clean_url,
'submitted': time.time(),
'page_title': '...', # just a placeholder to keep it pretty
}
redis.hmset(jobkey(job_id), job_details)
redis.expire(jobkey(job_id), 86400) # 24 hours
response = {
'job_id': job_id,
}
return json.dumps(response), 201 # created
def reply_text_and_get_user_profile(event):
# 取出消息內User的資料
################### 須作例外處理
user_profile = line_bot_api.get_profile(event.source.user_id)
# 將用戶資訊做成合適Json
user_info = {
"user_open_id":user_profile.user_id,
"user_nick_name":user_profile.display_name,
"user_status" : "",
"user_img" : user_profile.picture_url,
"user_register_menu" : menu_id
}
#將json傳回API Server
Endpoint='http://%s:5001/users' % (ip_location)
#Endpoint='http://%s:5001/users' % (ip_location)
Header={'Content-Type':'application/json'}
Response=requests.post(Endpoint,headers=Header,data=json.dumps(user_info))
# 先暫時寫的，用來檢查錯誤情形，方便除錯
print(Response)
print(Response.text)
# 將菜單綁定在用戶身上
linkMenuEndpoint='https://api.line.me/v2/bot/user/%s/richmenu/%s' % (user_profile.user_id, menu_id)
linkMenuRequestHeader={'Content-Type':'image/jpeg','Authorization':'Bearer %s' % secretFile["channel_access_token"]}
lineLinkMenuResponse=requests.post(linkMenuEndpoint,headers=linkMenuRequestHeader)
print(lineLinkMenuResponse)
print(lineLinkMenuResponse.text)
# 回覆文字消息與圖片消息
line_bot_api.reply_message(
event.reply_token,
TextSendMessage(text="請使用下方功能選單\n或是輸入下方字串\ndetail")
)
#再跟老師討論存在redis的值有沒有需要進去mysql
redis.hmset(user_profile.user_id, {'result': 0,"total" : 0,"sa_qid" : 0,"dev_qid" : 0,"sys_qid" : 0})
def update_search_stack(api_session, tweet_stack, keyword):
"""Searches for a specific term on twitter public timeline"""
# Storing last fetched id in order to make fewer requests
since_id = get_since_id(redis, "%s:%s" % (LOLCOIFFEURS_LIST, keyword))
search_tweet = api_session.GetSearch(term=keyword, since_id=since_id)
for t in search_tweet:
computed_tweet = {
"keyword": keyword,
"username": t.user.screen_name,
"created_at": t.created_at,
"text": t.text,
}
sys.stdout.write("adding tweet with id %s by user %s to database\n" % (str(t.id), str(t.user.screen_name)))
if (computed_tweet["username"] not in BLACKLISTED_USERS):
redis.rpush((LOLCOIFFEURS_LIST + ":%s" % (keyword)), t.id)
redis.hmset("%s:tweet:%s" % (LOLCOIFFEURS_LIST, t.id), computed_tweet)
print "Last since id en vigueur pour ce mot cle : %s" % since_id
return
def spotlight_route(room):
if (request.method == 'POST'):
data = request.get_json()
spotlightData = {}
params = ('living_room', 'kitchen', 'bathroom', 'bedroom')
idents = ('1', '2', '3', '4')
for x in range(0, len(params)):
if room == params[x]:
room = idents[x]
if data["status"] == "on":
spotlightData['status'] = 'on'
spotlightData['hourStart'] = '{}'.format(dt.now())
redis.hmset(room, spotlightData)
instruction = "{}{}".format(room, data["status"])
arduino.write(instruction.encode('utf-8'))
elif data["status"] == "off":
spotlightData['status'] = 'off'
spotlightData['hourEnd'] = '{}'.format(dt.now())
redis.hmset(room, spotlightData)
spotlightData['timeElapsed'] = getTotalTimeSwitchedOn(room)
redis.hmset(room, spotlightData)
instruction = "{}{}".format(room, data["status"])
arduino.write(instruction.encode('utf-8'))
objectToSend = {
"status":
data['status'],
"timeElapsed":
redis.hget(room, 'timeElapsed').decode('utf-8')
if data["status"] == "off" else ""
}
return jsonify(objectToSend)
def enqueue():
data = json.loads(request.data.decode())
if 'input_url' not in data:
response = {
'error': "The Youtube URL to download must be provided as 'input_url'",
}
logger.warn("Rejecting /api/enqueue request missing 'input_url'")
return json.dumps(response), 400 # bad request
clean_url = util.validate_url(data['input_url'])
if clean_url is None:
response = {
'error': "I'm sorry, that doesn't really look like a Youtube URL. :-(",
'info': "Please try again using a link starting with 'https://www.youtube.com'.",
}
logger.warn("Rejecting /api/enqueue request for %s" % data['input_url'])
return json.dumps(response), 403 # forbidden
logger.info("Accepting /api/enqueue request for %s" % clean_url)
job = rqueue.enqueue_call(
func=util.download,
args=(clean_url,),
result_ttl=900 # 15 minutes
)
job_id = job.get_id()
redis.lpush(joblist, job_id)
redis.ltrim(joblist, 0, 9)
job_details = {
'job_id': job_id,
'request_url': clean_url,
'submitted': time.time(),
'page_title': '...', # just a placeholder to keep it pretty
}
redis.hmset(jobkey(job_id), job_details)
redis.expire(jobkey(job_id), 86400) # 24 hours
response = {
'job_id': job_id,
}
return json.dumps(response), 201 # created
def lookup_user_cached(u: list, parameter: str):
if parameter not in ('login', 'id'):
raise ValueError('parameter should be login or id.')
if parameter == 'login':
#store = users
pass
else:
#store = ids
pass
c = chunks([i for i in u if not redis.exists(str(i))], MAX_USER_COUNT)
for n, i in enumerate(c):
if n > 0:
sleep(0.5) # a crude way to mitigate Twitch API rate limit
try:
result = lookup_user(i, parameter)
for i in result:
i['last_cached_time'] = time()
redis.hmset(str(i['id']), i)
redis.hmset(str(i['login']), i)
except RateLimitException:
logger.error('Twitch API rate limit exceeded. Will try later.')
spawn_later(10, lookup_user_cached, i, parameter)
def put(self, request, id):
"""
This function update the existing label
@param request: user request
@param id: user id
@return: update existing label
"""
user = request.user
try:
data = request.data
label = request.data['label']
user_id = self.request.user.id
serializer = self.serializer_class(user_id, data=data)
if serializer.is_valid():
label_update = serializer.save(user_id=user.id)
logger.info("Label successfully updated")
redis.hmset(str(user.id) + "label", {label_update.id: label})
return Response("Label successfully updated", status=status.HTTP_200_OK)
except:
logger.error("Label not updated, Something went wrong, Please check again!")
return Response("Label not updated, Something went wrong, Please check again!",
status=status.HTTP_400_BAD_REQUEST)
def add_key_f(key):
try:
strategies = {
'string': lambda (key, value): redis.set ( key, value )
, 'hash': lambda (key, value): redis.hmset (key, value)
, 'list': lambda (key, value): [redis.rpush (key, x) for x in value]
}
print (strategies)
print flask.request.json
return json.dumps(definite_strategy (strategies, flask.request.json['kind'], (key, flask.request.json['value']) ), indent = 2)
except Exception as e:
return str(e)
def update_search_stack(api_session, tweet_stack, keyword):
"""Searches for a specific term on twitter public timeline"""
# Storing last fetched id in order to make fewer requests
since_id = get_since_id(redis, "%s:%s" % (LOLCOIFFEURS_LIST, keyword))
search_tweet = api_session.GetSearch(term=keyword, since_id=since_id)
for t in search_tweet:
computed_tweet = {
"keyword": keyword,
"username": t.user.screen_name,
"created_at": t.created_at,
"text": t.text,
}
sys.stdout.write("adding tweet with id %s by user %s to database\n" %
(str(t.id), str(t.user.screen_name)))
if (computed_tweet["username"] not in BLACKLISTED_USERS):
redis.rpush((LOLCOIFFEURS_LIST + ":%s" % (keyword)), t.id)
redis.hmset("%s:tweet:%s" % (LOLCOIFFEURS_LIST, t.id),
computed_tweet)
print "Last since id en vigueur pour ce mot cle : %s" % since_id
return
def callback():
# Handles response from token endpoint
auth0.authorize_access_token()
resp = auth0.get('userinfo')
userinfo = resp.json()
sid = str(uuid.uuid4())
session['current_user'] = sid
redis.set(session['current_user'], userinfo['name'], ex=300)
sids = redis.hgetall("sids")
if sids==None:
tmp={sid:False}
redis.hmset("sids",tmp)
else:
sids[sid]=0
redis.hmset("sids",sids)
# Store the user information in flask session.
session['jwt_payload'] = userinfo
session['profile'] = {
'user_id': userinfo['sub'],
'name': userinfo['name'],
'picture': userinfo['picture']
}
return redirect(url_for('homepage'))
def set_object(self, model):
'''
Set values for each of a model's fields in redis.
'''
# >>> model.table_name
# 'person'
# >>> model.id
# 42
# >>> self.transform(model, Storage.cache)
# {
# 'name': 'Andy Warhol',
# 'telephone': '(212) 387-7555',
# 'location': (40.398, -72.037),
# 'email': '*****@*****.**'
# 'birthday': datetime.date(1928, 8, 6)
# }
# >>> self.construct_kv(self.transform(model, Storage.cache))
# {
# 'name': b'\xabAndy Warhol',
# 'telephone': b'\xae(212) 387-7555',
# 'location': b'\x92...',
# 'email': b'\[email protected]',
# 'birthday': b'...'
# }
# TODO: Figure out how to encode references in msgpack
# 'person:42:friends', ['person:12', 'person:92']
key = self.build_key(model)
mapitems = dict(model.fields) #.items()
del mapitems['id']
mapping = {
key.encode('utf-8'): self.encoder.encode(val)
for key, val in mapitems.items()
}
with self as redis:
response = redis.hmset(key, mapping)
return response
def enrol(self, cell_phone_number, pwd):
# not exist the cell phone, can go on enrol
is_exist = self.cell_phone_number_exist(cell_phone_number)
if is_exist != 0 :
# TODO(JIAS): put the new user object into 'Member:memberId' redis object
redis_structure_name = Constant.MEMBER + cell_phone_number
redis.hmset(redis_structure_name, "CELL_PHONE", cell_phone_number)
redis.hmset(redis_structure_name, "PASSWORD", pwd)
# generate session id
# TODO(JIAS): SessionId应该一到MySQL的DAO中处理，然后存储到Redis中
session_id = utils.SessionGenerator.session_generate()
redis.hmset(redis_structure_name, "SESSION_ID", session_id)
# 把号码放置到Member:cellphone中，标识此号码已经被注册过了
redis.sadd(Constant.MEMBER_CELL_PHONE, cell_phone_number)
def get_file(self, fid, stop_event, driver=None, restart=False):
"""Transfers a file from a Driver to another.
"""
redis = self.plug.redis
metadata = Metadata.get_by_id(self.plug, fid)
filename = metadata.filename
transfer_key = 'drivers:{}:transfers:{}'.format(self.plug.name, fid)
if driver:
redis.sadd(
'drivers:{}:transfers'.format(self.plug.name),
fid
)
redis.hmset(transfer_key, {'from': driver, 'offset': 0})
offset = 0
self.logger.info("Starting to get '{}' from {}", filename, driver)
else:
transfer = redis.hgetall(transfer_key)
driver = transfer['from']
offset = int(transfer['offset'])
self.logger.info(
"Restarting transfer of '{}' from {}", filename, driver
)
dealer = self.context.socket(zmq.DEALER)
port = redis.get('drivers:{}:router'.format(driver))
dealer.connect('tcp://localhost:{}'.format(port))
end = metadata.size
chunk_size = self.plug.options.get('chunk_size', 1 * 1024 * 1024)
if not restart:
self._call('start_upload', metadata)
while offset < end:
if stop_event.is_set():
# another transaction for the same file has
# probably started
self.logger.info(
"Aborting transfer of '{}' from {}", filename, driver
)
return
dealer.send_multipart((filename, str(offset), str(chunk_size)))
chunk = dealer.recv()
self.logger.debug(
"Received chunk of size {} from {} for '{}'",
len(chunk), driver, filename
)
self._call('upload_chunk', filename, offset, chunk)
offset = redis.hincrby(transfer_key, 'offset', len(chunk))
self._call('end_upload', metadata)
redis.delete(transfer_key)
redis.srem(
'drivers:{}:transfers'.format(self.plug.name),
fid
)
self.logger.info(
"Transfer of '{}' from {} successful", filename, driver
)
def add(gist):
gist_key = "gist:#"+str(gist['gist_id'])
redis.hmset(gist_key, dict(payload=json.dumps(gist), status='OK'))
index_add(gist['gist_id'], gist['hn_id'])
return 'OK'
subjectplanlist= Base.classes.eas_exmm_subjectplan
DBsession = sessionmaker(bind=engine)
session = DBsession()
#get all courses list of only one examplancode and one examcategorycode and one segmentcode
# two hmset one is use exampapercode as key and courseid as value and other is exampapercode as key and examsessionunit as value and two hmset 'name is same
tcpcode='080305402010409'
examplancode='20150901'
examcatetorycode='04'
segmentcode='010'
hmset1=examplancode+'.'+examcatetorycode+'.'+segmentcode+'.'+tcpcode+'.courses'
hmset2=examplancode+'.'+examcatetorycode+'.'+segmentcode+'.'+tcpcode+'.sessionunit'
for row in session.query(courseplan).filter_by(tcpcode=tcpcode).\
filter_by(examplancode=examplancode).\
filter_by(examcategorycode=examcatetorycode).\
filter_by(segmentcode=segmentcode):
for row2 in session.query(courseplanlist).filter_by(sn=row.sn):
redis.hmset(hmset1,{ row2.exampapercode:row.courseid})
row3 =session.query(subjectplanlist).filter_by(exampapercode=row2.exampapercode).\
filter_by(examplancode=examplancode).\
filter_by(examcategorycode=examcatetorycode).\
filter_by(segmentcode=segmentcode).first()
if row2 is not None:
redis.hmset(hmset2,{ row3.exampapercode:row3.examsessionunit})
session.close()
b=datetime.datetime.now()
c =b-a
print(divmod(c.days*86400+c.seconds,60))
request_region = requests.get("https://public-crest.eveonline.com/regions/{:d}/".format(region['id']))
logger.debug(request_region)
reg = request_region.json()['marketSellOrders']
logger.debug(result)
r_url = reg['href']
t_url = "?type=https://public-crest.eveonline.com/types/{:d}/".format(result['typeID'])
request_url = r_url + t_url
logger.debug(request_url)
marketdata_req = requests.get(request_url)
logger.debug(marketdata_req)
highest = 0
marketdata = marketdata_req.json()
logger.debug(marketdata)
if marketdata['totalCount'] == 0:
exempt.append(region['id'])
continue
market_items = marketdata['items']
logger.debug(market_items)
for item in market_items:
logger.debug(type(item))
if item['price'] > highest:
highest = item['price']
redis_dict = {region['id_str']:highest}
logger.debug("redis_dict: {dict}".format(dict = redis_dict))
redis.hmset(i_type, redis_dict)
redis.expire(i_type, 600)
logger.info("Finished caching for {i_type} in {region}".format(i_type = i_type, region = region['id']))
logger.info("Finished caching for all regions for the i_type: {:d}".format(i_type))
pipe.execute()
logger.info("Finished Caching")
try:
port = int(request.args.get('port', default=25565))
except Exception, e:
port = 25565
test = (host == 'test.mc.glassmoon.ru')
if test:
return jsonify(players=['doge', 'such', 'wow', 'diamonds'])
result = None
if redis:
key = "%s:%s" % (host, str(port))
result = redis.hgetall(key)
if not result:
try:
result = MinecraftQuery(host, port).get_rules()
except Exception as e:
result = {'message': 'No response from the Minecraft server'}
if redis:
redis.hmset(key, result)
redis.pexpire(key, 30000) # 30s
return jsonify(**result)
if __name__ == '__main__':
app.run(debug=options.debug, port=options.port)
def add_to_dict(key_prefix, id, data_dict):
if data_dict:
key = "%s:%s" % (key_prefix, id)
redis.hmset(key, data_dict)
def save(self, video):
redis.hmset(self.key, video)
self._load_video()
def update_user(self, user):
redis_structure_name = Constant.MEMBER + user.get_member_id()
# update the session id
redis.hmset(redis_structure_name, "SESSION_ID", user.session_id)
else :
print(' %s is not exists'%(batname,))
# --------------------begin --------------------------
redis = redis.Redis(host='10.96.142.109',port=6380,db=2)
#first flush db
redis.flushdb()
xls = pd.ExcelFile('/home/user/python/datamigrate.xlsx')
sheet1 = xls.parse('db')
for index ,row in sheet1.iterrows():
if (row['code']==10)==False:
redis.sadd('segment',row['code'])
if (pd.isnull(row['dbip'])==False):
redis.hmset('dbip',{row['code']:row['dbip']})
redis.hmset('dbuser',{row['code']:row['dbuser']})
redis.hmset('dbpwd',{row['code']:row['dbpwd']})
redis.hmset('dbip',{row['code']:row['dbip']})
redis.hmset('dbname',{row['code']:row['dbname']})
redis.sadd('alldb',row['code'])
sheet2 = xls.parse('expFile')
for index ,row in sheet2.iterrows():
redis.hmset('batdo',{row['batfilename']:row['expfilename']})
redis.hmset('batscope',{row['batfilename']:row['scope']})
redis.hmset('expfilepath',{row['batfilename']:row['filepath']})
# by config add message to MQ
def save(user):
if user.id is None:
user.id = redis.incr('user_count')
redis.hmset('user:%d' % user.id, user.__dict__)
return user
def route_post_ad(slot):
if not advertiser_id():
return '', 404
asset = request.files['asset']
id = next_ad_id()
key = ad_key(slot, id)
type = fetch(request.form, 'type')
if not type:
type = asset.mimetype
if not type:
type = 'video/mp4'
redis = get_redis()
redis.hmset(key, {
'slot': slot,
'id': id,
'title': fetch(request.form, 'title'),
'type': type,
'advertiser': advertiser_id(),
'destination': fetch(request.form, 'destination'),
'impressions': 0
})
redis.set(asset_key(slot, id), asset.read())
redis.rpush(slot_key(slot), id)
redis.sadd(advertiser_key(advertiser_id()), key)
return jsonify(get_ad(slot, id))