def index(request):
if request.method == 'POST':
keyword = request.POST['search']
if not redis.get(keyword):
# Here, we are only getting first 50 results
search_params = {
'key': YOUTUBE_API_KEY,
'part': 'snippet',
'q': keyword,
'maxResults': '50',
'order': 'date',
'type': 'video'
}
search_response = requests.get(YOUTUBE_SEARCH_URL,
params=search_params)
json_data = search_response.json()
next_token = json_data.get("nextPageToken")
if next_token:
tasks.fetch_all(
keyword, next_token
) # This task runs in the background to fetch the remaining 150 results
videos_list = []
video_data = OrderedDict()
for r in search_response.json()['items']:
videos_list.append(
r['id']
['videoId']) # gather all video ids to use for next query
video_data[r['id']['videoId']] = r[
'snippet'] # save video data from search endpoint
videos_params = {
'key': YOUTUBE_API_KEY,
'part': 'statistics',
'id': ','.join(videos_list),
}
videos_response = requests.get(
YOUTUBE_VIDEO_URL,
params=videos_params) # second query to get video views
results = videos_response.json()['items']
videos = []
for r in results:
data = video_data.get(r['id'])
data['views'] = r['statistics'][
'viewCount'] # append views to video data
data[
'url'] = f"https://www.youtube.com/watch?v={r['id']}" # generate url
videos.append(data)
new_cache = json.dumps(videos)
redis.setex(keyword, 600, new_cache) # cache the result
context = {'videos': videos, 'keyword': keyword, 'show_all': False}
return render(request, "makr/result.html", context)
cached = redis.get(keyword)
cached_videos = json.loads(cached)
context = {
'videos': cached_videos,
'keyword': keyword,
'show_all': True
}
return render(request, "makr/result.html", context)
return render(request, "makr/index.html")
def Info(name: str = None,info = None):
x = connect(name)
response = requests.get(x).content
country = response
result = redis.get(str(info))
if not result:
info = info.split(",")
out = "{"
websiteinfo = json.loads(country)[0]
if websiteinfo is None:
return "error in name"
else:
for s in info:
try:
out += s+":"+str(websiteinfo[s])+ ","
except KeyError:
return "wrong info"
out += "}"
redis.setex (str(info),60,str(out))
result = redis.get(str(info))
return str(result)
else:
return result
def search_movie(request):
result = []
credentials = yaml.load(open('credentials.yaml'), Loader=yaml.FullLoader)
simliar_movies = movie_recommedation.movie_recommendation(
request.GET.get('search'))
if not simliar_movies:
return render(
request, 'similar_movie.html', {
'result': result,
'message': 'Sorry couldnt get the result for this movies.'
})
url = 'http://www.omdbapi.com/?s={}&type=movie&apikey={}'
for movie in simliar_movies:
cached_movie = redis.get(movie)
if not cached_movie:
response = requests.get(url.format(movie, credentials['api_key']))
if not 'Error' in response.text:
redis.setex(movie, 3060, str(response.text))
result.append(json.loads(response.text))
else:
json_data = ast.literal_eval(str(cached_movie, 'utf-8'))
result.append(json_data)
return render(request, 'similar_movie.html', {
'result': result,
'search': request.GET.get('search').capitalize()
})
def fetch_github_profile(username):
# Gets the result from Github and caches it
result = requests.get(GITHUB_URL + username)
print(result.json())
redis.setex(username, 10, json.dumps(result.json()))
return {"cached": False, "profile": result.json()}
def country_info(name: str = None):
x = connect(name)
response = requests.get(x).content
if response is None:
return "Error in Connection"
else:
redis.setex (name,60,str(response))
def get_page(url: str) -> str:
"""Function: expiring web cache and tracker"""
response = requests.get(url)
redis.set(f"cached:{url}", incr)
redis.incr(f"count:{url}")
redis.setex(f"cached:{url}", 10, redis.get(f"cached:{url}"))
return response.text
def wrapper(url):
""" Wrapper for decorator guy """
redis.incr(f"count:{url}")
cached_response = redis.get(f"cached:{url}")
if cached_response:
return cached_response.decode('utf-8')
result = fn(url)
redis.setex(f"cached:{url}", 10, result)
return result
def update_images(response):
# poor man's background task
try:
if not redis.get('fresh'):
redis.setex('fresh', 'yes', 60)
fetch_latest_images()
except ConnectionError as e:
if app.debug:
fetch_latest_images(local=True)
return response
def fetch_github_profile(username):
# Get the result from Github and cache it
result = requests.get("https://api.github.com/users/" + username)
redis.setex(username, 10, str(result.json()))
print(
json.dumps({
"cached": False,
"profile": result.json()
})
)
def get(self, name):
dog = redis.get(name)
if dog:
print('-- This is coming from cache --')
return json.loads(dog)
else:
dog = DogModel.get_from_db(name)
if dog:
redis.setex(dog.name, 90, json.dumps(dog.json()))
return dog.json()
return {'message': 'Doggo not found'}, 404
def _f(*args, **kwargs):
# TODO: this could be used as a DoS attack by filling up
# redis. Maybe add global rate limiting?
k = "rl:%s_%s" % (f.__name__, request.remote_addr)
if not redis.exists(k) or not redis.ttl(k):
redis.delete(k)
redis.setex(k, 1, 60)
return f(*args, **kwargs)
if int(redis.get(k)) > per_minute:
return "Too many requests per minute!", 429
redis.incr(k)
return f(*args, **kwargs)
def antiflood(m):
if not is_sudo(m.from_user.id):
_hash = "anti_flood:user:"******"شما به دلیل ارسال پشت سرهم پیغام، 60 ثانیه مسدود شدید 🚫")
redis.delete(_hash)
redis.setex('ban' + str(m.from_user.id), 60, True)
def wrapper(url):
""" Wrapper """
redis.incr(f"count:{url}")
cached = redis.get(f"cached:{url}")
if cached:
return cached.decode('utf-8')
html = method(url)
redis.setex(f"cached:{url}", 10, html)
return html
def index():
"""Returns/Renders the home (only) page."""
# Check cache for most recent status
has_changed = redis.get('has_changed')
# If the cache expired, check if the status has changed and store in cache (timeout=10 mins)
if not has_changed:
print str(datetime.now()), 'cache miss'
has_changed = 'yes' if status_has_changed() else 'no'
redis.setex('has_changed', has_changed, 60*10)
else:
print str(datetime.now()), 'cache hit'
return render_template('index.html', has_it_shipped=has_changed)
def InsertData(shorturl, fullurl, expiration=0):
""" Insert key and full URL into the database for later access.
Value of 0 results in no expiration of the key. """
try:
# Set expiration for key to remove shot URL after x seconds
if expiration == 0:
# Set the key only if it does not exist to prevent overwriting
# exisitng URL data.
redis.set(shorturl, fullurl)
elif expiration > 0:
redis.setex(shorturl, fullurl, expiration)
return True
except:
return False
def cms_assert(boolexp, code, msg):
global redis
global timeout
global task_id
global db
global pid
if boolexp:
redis.setex(task_id, timeout, json.dumps([{
'code': 500,
'errmsg': msg
}]))
if (db is not None) and pid != '':
db.closeConn([pid])
logging.warning('CODE:' + str(code) + ' MSG:' + msg)
# print code,msg
exit(1)
return
def upload_file(sha):
if not redis.get("upload-lock:" + sha):
abort(403)
# check if the post request has the file part
if 'file' not in request.files:
abort(400)
f = request.files['file']
# if user does not select file, browser also
# submit a empty part without filename
if f.filename == '':
abort(400)
if f and f.filename == secure_filename(f.filename):
filename = secure_filename(f.filename)
# Store files in redis with an expiration so we hopefully don't leak resources.
redis.setex("file:" + filename, 60 * 60, f.read())
print(filename, "uploaded")
else:
abort(400)
return jsonify({'msg': 'Ok'})
def set(self, key, value, expire=-1):
redis = self._getRedis(key)
if redis == False:
return False
if expire < 0 and expire != -1:
return False
if expire == -1:
return redis.set(key, value)
else:
return redis.setex(key, value, expire)
def fetch_all(keyword, next_token):
count = 3 # For the remaining 150 videos ( 3 * 50 )
videos = []
while next_token and count is not 0:
search_params = {
'key': YOUTUBE_API_KEY,
'part': 'snippet',
'q': keyword,
'maxResults': '50',
'order': 'date',
'type': 'video',
'pageToken': next_token
}
search_response = requests.get(YOUTUBE_SEARCH_URL, params=search_params)
json_data = search_response.json()
next_token = json_data.get("nextPageToken")
videos_list = []
video_data = OrderedDict()
for r in search_response.json()['items']:
videos_list.append(r['id']['videoId']) # gather all video ids to use for next query
video_data[r['id']['videoId']] = r['snippet'] # save video data from search endpoint
videos_params = {
'key': 'AIzaSyCBa7p6H8fzlULejUu_Dxh4g8gP6bhL2IM',
'part': 'statistics',
'id': ','.join(videos_list),
}
videos_response = requests.get(YOUTUBE_VIDEO_URL, params=videos_params) # second query to get video views
results = videos_response.json()['items']
for result in results:
data = video_data.get(result['id'])
data['views'] = result['statistics']['viewCount'] # append views to video data
data['url'] = f"https://www.youtube.com/watch?v={result['id']}" # generate url
videos.append(data)
count -= 1
cached = redis.get(keyword) # fetch cached videos from redis
cached_videos = json.loads(cached)
cached_videos.extend(videos) # append new videos to cached
all_videos = json.dumps(cached_videos)
redis.setex(keyword, 600, all_videos) # set cache to all videos
def status():
status = redis.get('status')
if status is None:
status = json.dumps(get_status())
redis.setex('status', status, 10)
return Response(status, mimetype='application/json')
def register_server(self, server_id, server_details):
redis_return = redis.setex(server_id, 30, json.dumps(server_details))
app.logger.debug("Redis registration return: {}".format(
str(redis_return)))
def namethatdataset_quiz(request):
# TODO beautify this monstrocity please
context = utils.get_global_context(request)
# Are we mid game?
session = request.COOKIES.get('quiz_session')
# Did the user just land here?
if not session:
session = str(uuid.uuid4())
full_key = lambda x: 'aishack.quiz.%s' % x
# Did redis forget about us?
in_redis = full_key(session) in redis
# Did we finish a game and we returned again?
if in_redis:
session_data = json.loads(redis[full_key(session)])
# Did we cross all the 10 questions?
if session_data['state'] == 13:
# Create a new session
del redis[full_key(session)]
in_redis = False
session_data = {}
if not in_redis:
# No session information
session_data = {}
session_data['state'] = 0
session_data['score'] = 0
session_data['timestamp'] = time.time()
session_data['done'] = []
redis.setex(full_key(session), 900, json.dumps(session_data))
datasets = {
'voc-2012': 'VOC 2012',
'caltech-101': 'Caltech 101',
'caltech-256': 'Caltech 256',
'sun09': 'Sun 09',
'graz-01': 'Graz 01',
'graz-02': 'Graz 02',
'coil-100': 'Coil 100',
None: None
}
state = session_data['state']
ans = request.GET.get('ans', None)
if ans:
ans = int(ans)
prev_answer_success = None
prev_answer_fail = None
if 'expecting' in session_data and ans != None:
if ans == session_data['expecting']:
# Correct answer!
prev_answer_success = session_data['expecting']
session_data['score'] += 1
elif ans != None:
# Failed answer!
prev_answer_fail = session_data['expecting']
session_data['done'].append(session_data['pick'])
del session_data['pick']
session_data['state'] += 1
state += 1
# The image path to display
src_img = None
# The options
opt1 = None
opt2 = None
opt3 = None
opt4 = None
topic_mapping = {
0: 'cat',
1: 'cat',
2: 'dog',
3: 'dog',
4: 'person',
5: 'person',
6: 'cup',
7: 'cup',
8: 'airplane',
9: 'airplane',
10: 'car',
11: 'car'
}
topic = None
if state in topic_mapping:
topic = topic_mapping[state]
else:
topic = 'done'
message = None
score = session_data['score']
if score >= 11:
message = 'Your grace through math leaves people spellbound.'
img = 'quiz-grand-master.png'
elif score >= 9:
message = 'You are almost there and you know it.'
img = 'quiz-jedi-master.png'
elif score >= 7:
message = 'You understand the ways of the force.'
img = 'quiz-jedi-knight.png'
elif score >= 4:
message = 'You are on the right path.'
img = 'quiz-padawan.png'
else:
message = 'Your inner strength needs to be unleashed.'
img = 'quiz-youngling.png'
session_data['state'] += 1
context.update({
'topic': topic,
'score': session_data['score'],
'current_page': 'name-that-dataset',
'message': message,
'avatar': img,
})
response = render(request, 'name-that-dataset-quiz.html', context)
del redis[full_key(session)]
response.delete_cookie('quiz_session')
return response
key = 'question_%d' % state
if key not in session_data:
paths = os.listdir('./name-that-dataset/%s/' % (topic))
while True:
random.shuffle(paths)
pick = '%s/%s' % (topic, paths[0])
if pick not in session_data['done']:
imglist = os.listdir('./name-that-dataset/%s/%s/' %
(topic, paths[0]))
random.shuffle(imglist)
with open(
'./name-that-dataset/%s/%s/%s' %
(topic, paths[0], imglist[0]), 'r') as fp:
src_img = base64.b64encode(fp.read())
correct = paths[0]
random.shuffle(paths)
opt1 = paths[0]
opt2 = paths[1]
if len(paths) > 2:
opt3 = paths[2]
if len(paths) > 3:
opt4 = paths[3]
session_data['expecting'] = paths.index(correct)
session_data['pick'] = pick
session_data[key] = {}
session_data[key]['src_img'] = src_img
session_data[key]['opt1'] = opt1
session_data[key]['opt2'] = opt2
session_data[key]['opt3'] = opt3
session_data[key]['opt4'] = opt4
break
else:
src_img = session_data[key]['src_img']
opt1 = session_data[key]['opt1']
opt2 = session_data[key]['opt2']
opt3 = session_data[key]['opt3']
opt4 = session_data[key]['opt4']
context.update({
'current_page': 'name-that-dataset',
'score': session_data['score'],
'state': session_data['state'] + 1,
'src_img': src_img,
'opt1': datasets[opt1],
'opt2': datasets[opt2],
'opt3': datasets[opt3],
'opt4': datasets[opt4],
'topic': topic,
'prev_answer_fail': prev_answer_fail,
'prev_answer_success': prev_answer_success,
'expecting': session_data['expecting'],
})
redis.setex(full_key(session), 900, json.dumps(session_data))
response = render(request, 'name-that-dataset-quiz.html', context)
response.set_cookie('quiz_session', session)
return response
}
pass
publish_url = publish_url if len(
doc_data[0][len(doc_data[0]) -
2]) == 0 else doc_data[0][len(doc_data[0]) - 2]
print('continue-4', sql, algorithm_type)
logging.info('[Compiler]:continue-4=' + sql)
# db.closeConn([pid])
# 加载插件语言配置模板,未做键值检查
language_config = json.loads(str(redis.get('lang'), encoding='utf-8'))
# 开始进入渲染队列
render_result = []
algor = Algorithm()
task_mail.append({'code': 201, 'errmsg': u'开始执行渲染队列', 'progress': 0})
redis.setex(task_id, timeout, json.dumps(task_mail, ensure_ascii=False))
print('continue-5')
logging.info('[Compiler]:continue-5')
for i in range(0, field_count):
field_id = data[i][0]
field_name = data[i][1]
field_type = data[i][2]
field_algorithm = data[i][5]
# 用于输出渲染的script类型需要field_value，input不需要
if algorithm_type == 'script':
field_value = data[i][6] if doc_data[0][i] == '' else doc_data[0][i]
else:
field_value = ''
def register_worker(self, client_id, client_details):
redis_return = redis.setex(client_id, 30, json.dumps(client_details))
app.logger.debug("Redis registration return: {}".format(
str(redis_return)))
def caching(token, data):
redis.setex(token, 86400, str(data))
def cache(self, expiry=1800):
"""
Serializes and caches the Preview object to Redis
"""
serialized = json.dumps(self.to_dict())
redis.setex(self.url, serialized.encode("utf-8"), expiry)
def cache(self):
redis = get_redis()
redis.setex('session-%s' % self.id, 1, int(self.expires.strftime("%s")))
arguments = parser.parse_args()
redis = redis.Redis()
# for getting values from redis----------------------------------
if arguments.get:
if redis.get(arguments.get):
print(redis.get(arguments.get).decode("utf-8"))
else:
print("No value in DB")
# for setting valye in redis--------------------------------------
if arguments.set:
if arguments.value:
if (arguments.expire):
redis.setex(arguments.set, arguments.expire, arguments.value)
print(arguments.set, "set to", arguments.value, "for",
arguments.expire, "seconds")
else:
redis.set(arguments.set, arguments.value)
print(str(arguments.set), ":", str(arguments.value))
else:
print("value not given for the key :", arguments.set)
# for adding to a orered set ---------------------------------------------
if arguments.zadd:
if arguments.key:
if arguments.score:
if arguments.member:
# redis.zadd("key",{"value":score}) syntax of zadd for newer update of redis-----------------
def namethatdataset_quiz(request):
# TODO beautify this monstrocity please
context = utils.get_global_context(request)
# Are we mid game?
session = request.COOKIES.get('quiz_session')
# Did the user just land here?
if not session:
session = str(uuid.uuid4())
full_key = lambda x: 'aishack.quiz.%s' % x
# Did redis forget about us?
in_redis = full_key(session) in redis
# Did we finish a game and we returned again?
if in_redis:
session_data = json.loads(redis[full_key(session)])
# Did we cross all the 10 questions?
if session_data['state'] == 13:
# Create a new session
del redis[full_key(session)]
in_redis = False
session_data = {}
if not in_redis:
# No session information
session_data = {}
session_data['state'] = 0
session_data['score'] = 0
session_data['timestamp'] = time.time()
session_data['done'] = []
redis.setex(full_key(session), 900, json.dumps(session_data))
datasets = {'voc-2012': 'VOC 2012',
'caltech-101': 'Caltech 101',
'caltech-256': 'Caltech 256',
'sun09': 'Sun 09',
'graz-01': 'Graz 01',
'graz-02': 'Graz 02',
'coil-100': 'Coil 100',
None: None}
state = session_data['state']
ans = request.GET.get('ans', None)
if ans:
ans = int(ans)
prev_answer_success = None
prev_answer_fail = None
if 'expecting' in session_data and ans != None:
if ans == session_data['expecting']:
# Correct answer!
prev_answer_success = session_data['expecting']
session_data['score'] += 1
elif ans != None:
# Failed answer!
prev_answer_fail = session_data['expecting']
session_data['done'].append(session_data['pick'])
del session_data['pick']
session_data['state'] += 1
state += 1
# The image path to display
src_img = None
# The options
opt1 = None
opt2 = None
opt3 = None
opt4 = None
topic_mapping = {0: 'cat', 1: 'cat',
2: 'dog', 3: 'dog',
4: 'person', 5: 'person',
6: 'cup', 7: 'cup',
8: 'airplane', 9: 'airplane',
10: 'car', 11: 'car'}
topic = None
if state in topic_mapping:
topic = topic_mapping[state]
else:
topic = 'done'
message = None
score = session_data['score']
if score >= 11:
message = 'Your grace through math leaves people spellbound.'
img = 'quiz-grand-master.png'
elif score >= 9:
message = 'You are almost there and you know it.'
img = 'quiz-jedi-master.png'
elif score >= 7:
message = 'You understand the ways of the force.'
img = 'quiz-jedi-knight.png'
elif score >= 4:
message = 'You are on the right path.'
img = 'quiz-padawan.png'
else:
message = 'Your inner strength needs to be unleashed.'
img = 'quiz-youngling.png'
session_data['state'] += 1
context.update({'topic': topic,
'score': session_data['score'],
'current_page': 'name-that-dataset',
'message': message,
'avatar': img,
})
response = render(request, 'name-that-dataset-quiz.html', context);
del redis[full_key(session)]
response.delete_cookie('quiz_session')
return response
key = 'question_%d' % state
if key not in session_data:
paths = os.listdir('./name-that-dataset/%s/' % (topic))
while True:
random.shuffle(paths)
pick = '%s/%s' % (topic, paths[0])
if pick not in session_data['done']:
imglist = os.listdir('./name-that-dataset/%s/%s/' % (topic, paths[0]))
random.shuffle(imglist)
with open('./name-that-dataset/%s/%s/%s' % (topic, paths[0], imglist[0]), 'r') as fp:
src_img = base64.b64encode(fp.read())
correct = paths[0]
random.shuffle(paths)
opt1 = paths[0]
opt2 = paths[1]
if len(paths) > 2:
opt3 = paths[2]
if len(paths) > 3:
opt4 = paths[3]
session_data['expecting'] = paths.index(correct)
session_data['pick'] = pick
session_data[key] = {}
session_data[key]['src_img'] = src_img
session_data[key]['opt1'] = opt1
session_data[key]['opt2'] = opt2
session_data[key]['opt3'] = opt3
session_data[key]['opt4'] = opt4
break
else:
src_img = session_data[key]['src_img']
opt1 = session_data[key]['opt1']
opt2 = session_data[key]['opt2']
opt3 = session_data[key]['opt3']
opt4 = session_data[key]['opt4']
context.update({'current_page': 'name-that-dataset',
'score': session_data['score'],
'state': session_data['state'] + 1,
'src_img': src_img,
'opt1': datasets[opt1],
'opt2': datasets[opt2],
'opt3': datasets[opt3],
'opt4': datasets[opt4],
'topic': topic,
'prev_answer_fail': prev_answer_fail,
'prev_answer_success': prev_answer_success,
'expecting': session_data['expecting'],
})
redis.setex(full_key(session), 900, json.dumps(session_data))
response = render(request, 'name-that-dataset-quiz.html', context);
response.set_cookie('quiz_session', session)
return response
def travis():
signature = base64.b64decode(request.headers.get('Signature'))
try:
public_key = _get_travis_public_key()
except requests.Timeout:
print("Timed out when attempting to retrieve Travis CI public key")
abort(500)
except requests.RequestException as e:
print("Failed to retrieve Travis CI public key")
abort(500)
try:
check_authorized(signature, public_key, request.form["payload"])
except SignatureError:
abort(401)
data = json.loads(request.form["payload"])
repo = data["repository"]["owner_name"] + "/" + data["repository"]["name"]
build_number = data["id"]
sha = data["commit"]
if data["type"] == "pull_request":
sha = data["head_commit"]
tag = None
if data["type"] == "push" and data["tag"] != None:
tag = data["tag"]
print(data)
key = sha
if tag is not None:
key = tag
upload_lock = "upload-lock:" + key
if data["state"] in ("started", ):
print("travis started", key)
# Handle pulls differently.
if data["pull_request"]:
load_code.delay(
repo, "pull/" + str(data["pull_request_number"]) + "/head")
else:
load_code.delay(repo, "refs/heads/" + data["branch"])
redis.setex(upload_lock, 20 * 60, "locked")
set_status(repo, sha, "pending", data["build_url"],
"Waiting on Travis to complete.")
elif data["state"] in ("passed", "failed"):
print("travis finished")
key = repo + "/" + key
set_status(repo, sha, "pending",
"https://rosie-ci.ngrok.io/log/" + key,
"Queueing Rosie test.")
redis.delete(upload_lock)
test_commit(repo, sha, tag)
elif data["state"] is ("cancelled", ):
print("travis cancelled")
redis.delete(upload_lock)
set_status(repo, sha, "error", data["build_url"], "Travis cancelled.")
elif data["status"] is None:
set_status(repo, sha, "error", data["build_url"], "Travis error.")
else:
print("unhandled state:", data["state"])
print(data)
return jsonify({'status': 'received'})
def access_domain(domain, port):
redis = _get_redis()
if redis:
redis.lpush('last-domains', '%s:%d' % (domain, port))
redis.ltrim('last-domains', 0, 10)
redis.setex('%s:%d' % (domain, port), '', 60 * 10)
def set(key, value, lifetime=300):
redis.setex(
key,
timedelta(seconds=lifetime),
value=value
)