def task():
while RUN:
bug_beans = zen_tao_client.get_my_bug()
# 过滤结果，并将最新数据放入缓存
new_beans = cache.filter_new("bug", bug_beans)
cache.add("bug", bug_beans)
if len(new_beans) != 0:
tools.print_log(__name__ + "." + sys._getframe().f_code.co_name,
"本次显示的bug:" + str(new_beans))
for bean in new_beans:
msg_title = "当前有Bug指向您"
msg_content = bean.title + "\n\t\n"
msg_content += "级别：" + bean.level + "\n" \
+ "类型：" + bean.error_type + "\n" \
+ "From:" + bean.author
tools.show_notify(msg_title, msg_content)
else:
pass
if len(new_beans) != 0:
sleep_time = len(new_beans) * 13
else:
sleep_time = 1
time.sleep(sleep_time)
def post(self):
tx = self.request.get_range('x', None)
ty = self.request.get_range('y', None)
z = self.request.get_range('z', None)
limit = self.request.get_range('limit', min_value=1, max_value=1000, default=1000)
offset = self.request.get_range('offset', min_value=0, default=0)
name = self.request.get('name', None)
source_name = self.request.get('source', None)
if tx is None or ty is None or z is None or name is None or source_name is None:
self.error(400)
return
key = 'tile-%s-%s-%s-%s-%s' % (z, ty, tx, source_name, name)
png = cache.get(key)
if png is None:
png = get_tile_png(tx, ty, z, name, source_name, limit, offset)
if png is None:
png = PointTile.blank()
cache.add(key, png, dumps=False)
logging.info('TILE BACKEND MEMORY = %s' % runtime.memory_usage().current())
self.response.set_status(200)
self.response.headers['Content-Type'] = 'image/png'
self.response.out.write(png)
def unlocked(path):
folderConfig = os.path.join(path, FOLDERCFG)
lock = getParam('LOCK', folderConfig)
if not lock:
return True
import cache
if cache.exists(path):
return True
md5 = checkPassword(path, lock)
if len(md5) == 0:
return False
if md5 == 'ERROR':
utils.DialogOK(GETTEXT(30080))
return False
periods = [0, 1, 5, 15]
setting = int(ADDON.getSetting('CACHE'))
period = periods[setting]
cache.add(path, period)
return True
def delete_all_notifications(user_email):
notifications = db.Notification.all().filter('recipient_email =',
user_email)
for n in notifications:
n.delete()
cache_key = cache.MC_USER_NOTIFICATION_PREFIX + user_email
cache.add(cache_key, 0)
def get_series_us(self, chat_id, message_id, search_query_id):
search_query = db.get_search_query(search_query_id)
result = avtonomer.validate_us_plate_series(search_query.query_text)
state, series_number = result.groups()
key = f"avtonomer.get_series_us({state}, {series_number})"
state_id, ctype_id = avtonomer.US_STATES_ID[state]
result = cache.get(key)
if not result:
result = avtonomer.get_series_us(state_id, ctype_id, series_number)
if result is None:
logger.warning(f"Not data for query {series_number}")
bot.send_message(
chat_id,
"Нет данных",
reply_to_message_id=message_id,
)
return
cache.add(key, result, timedelta(minutes=5))
url = avtonomer.get_series_us_url(state_id, ctype_id, series_number)
message = (
f"В серии [{series_number}]({url}) штата `{state}` пока нет ни одного номера"
if result == 0 else
f"Количество фотографий в серии [{series_number}]({url}) штата `{state}`: {result}"
)
bot.send_message(
chat_id,
message,
parse_mode="Markdown",
reply_to_message_id=message_id,
)
def unlocked(path):
folderConfig = os.path.join(path, FOLDERCFG)
lock = getParam('LOCK', folderConfig)
if not lock:
return True
import cache
if cache.exists(path):
return True
md5 = checkPassword(path, lock)
if len(md5) == 0:
return False
if md5 == 'ERROR':
utils.DialogOK(GETTEXT(30080))
return False
periods = [0, 1, 5, 15]
setting = int(ADDON.getSetting('CACHE'))
period = periods[setting]
cache.add(path, period)
return True
def get_ru_region(self, chat_id, message_id, search_query_id):
search_query = db.get_search_query(search_query_id)
result = avtonomer.validate_ru_region(search_query.query_text)
region = result.groups()[0]
key = f"avtonomer.get_ru_region({region})"
region_name, region_id = avtonomer.RU_REGIONS_ID[region]
result = cache.get(key)
if not result:
result = avtonomer.search_ru(
ctype=1,
regions=[region],
tags=[avtonomer.TAG_NEW_LETTER_COMBINATION],
)
cache.add(key, result, timedelta(minutes=5))
message = f"Регион *{region}* — {region_name}"
if result is not None:
message += "\n\nСвежие серии:\n"
message += "\n".join([
"• {} /{} — {} {}".format(
car.date,
avtonomer.translate_to_latin(car.license_plate.replace(
" ", "")).upper(),
car.make,
car.model,
) for car in result.cars
])
bot.send_message(
chat_id,
message,
parse_mode="Markdown",
reply_to_message_id=message_id,
)
def post(self):
# Check parameters
name = self.request.get('name', None)
if not name:
self.error(400)
return
source_name = self.request.get('source', None)
# Check cache
key = self.cache_key(name, source_name)
names = cache.get(key)
if names:
self.response.set_status(200)
self.response.headers['Content-Type'] = "application/json"
self.response.out.write(names)
return
source = sources.get(source_name)
if not source:
# TODO: Get names from all sources?
self.error(404)
return
# Make service request for names
names, status_code = self.get_names(source, name)
if status_code != 200:
self.error(status_code)
return
# Update cache and send response
cache.add(key, names)
self.response.set_status(200)
self.response.headers['Content-Type'] = "application/json"
self.response.out.write(simplejson.dumps(names))
def create_user_model(offset_param, user_query_func, href_create_func,
memcache_cursor_key_func):
if offset_param:
offset = int(offset_param)
else:
offset = 0
if offset < 0:
offset = 0
fetch_count = users_page_size
query = user_query_func()
if offset == 0:
all_users = query.run(limit=fetch_count + 1)
else:
cursor = cache.get(memcache_cursor_key_func(offset))
if cursor:
query = query.with_cursor(start_cursor=cursor)
all_users = query.run(limit=fetch_count + 1)
else:
all_users = query.run(limit=fetch_count + 1, offset=offset)
cache.add(memcache_cursor_key_func(offset), query.cursor())
import logging
has_prev_page = (offset > 0)
has_next_page = False
logging.error('offset={0} has_prev_page={1}'.format(offset, has_prev_page))
index = 0
users = []
for u in all_users:
index = index + 1
if index > fetch_count:
has_next_page = True
else:
converted_user = convert.convert_user_profile(u)
users.append(converted_user)
if index == fetch_count:
cache.add(memcache_cursor_key_func(offset + fetch_count),
query.cursor())
next_page_href = href_create_func(offset + fetch_count)
if offset - users_page_size <= 1:
prev_page_href = href_create_func(0)
else:
prev_page_href = href_create_func(offset - users_page_size)
return {
'has_next_page': has_next_page,
'has_prev_page': has_prev_page,
'next_page_href': next_page_href,
'prev_page_href': prev_page_href,
'first_page_href': href_create_func(0),
'users': users
}
def search_license_plate(self, chat_id, message_id, search_query_id, page,
edit):
search_query = db.get_search_query(search_query_id)
lp_num = search_query.query_text
lp_type = search_query.num_type
key = f"avtonomer.search_{lp_type}({lp_num})"
result = cache.get(key)
if not result:
if lp_type == "ru":
result = avtonomer.search_ru(lp_num)
else:
result = avtonomer.search_su(lp_num)
if not result:
bot.send_message(
chat_id,
"По вашему запросу ничего не найдено",
reply_to_message_id=message_id,
)
return
cache.add(key, result, timedelta(minutes=5))
car = result.cars[page]
cars_count = len(result.cars)
key = f"avtonomer.load_photo({car.thumb_url})"
file_id = cache.get(key)
if not file_id:
photo = avtonomer.load_photo(car.thumb_url)
if not photo:
photo = open(PHOTO_NOT_FOUND, "rb")
else:
photo = file_id
caption = get_car_caption(car, lp_num, page, cars_count)
markup = get_car_reply_markup(cars_count, search_query_id, page)
if edit:
message = bot.edit_message_media(
media=telegram.InputMediaPhoto(photo, caption=caption),
reply_markup=markup,
chat_id=chat_id,
message_id=message_id,
)
else:
message = bot.send_photo(
chat_id,
photo,
caption,
reply_to_message_id=message_id,
reply_markup=markup,
)
if not file_id:
cache.add(key, message.photo[-1].file_id, timedelta(minutes=30))
def _do_compile(compile_cmd, source):
try:
compiled = cache.get_compiled_file_path(source)
cmd = fill_placeholders(compile_cmd, source, compiled)
_execute(cmd, check=True)
cache.add(source, compiled)
return compiled
except subprocess.CalledProcessError as error:
raise CompilationError from error
def add_user_profile(profile):
profile_exist = get_user_profile(profile.email)
if profile_exist:
return profile_exist
result = profile.put()
cache.add(cache.MC_USER_PROFILE + profile.email, profile)
cache.delete(cache.MC_MAIN_PAGE_RECENT_IMAGES_KEY)
cache.delete(cache.MC_MAIN_PAGE_RECENT_COMMENTS)
return result
def _update_cache(self, pkg):
if pkg.flags.rcode != protocol.Flags.RCODE_OK:
for record in pkg.questions:
cache = self._cache["error"]
cache.add(record.domain_name, "", time.time() + 30 * 60)
for record in chain(pkg.answers, pkg.access_rights, pkg.extra_records):
key = record.domain_name
qtype = record.type
remove_time = time.time() + record.ttl
rdata = record.rdata
self._cache[qtype].add(key, rdata, remove_time)
def initialize(self, request, response):
user_agent = request.headers.get('User-Agent')
if user_agent:
check_user_agent(user_agent)
webapp.RequestHandler.initialize(self, request, response)
self.user_info = UserInfo(self.request.uri)
self.settings = cache.get(cache.MC_SETTINGS)
if not self.settings:
self.settings = get_settings()
cache.add(cache.MC_SETTINGS, self.settings)
def get_notification_count(user_email):
cache_key = cache.MC_USER_NOTIFICATION_PREFIX + user_email
value = cache.get(cache_key)
if value <> None:
return value
else:
count = db.Notification.all().filter('recipient_email =',
user_email).filter(
'read =', False).count()
cache.add(cache_key, count)
return count
def get_friends(session=None, target=None):
starget = str(target)
if cache.contains(starget):
return cache.get(starget)
else:
try:
friends = session.friends.get(user_id=target)["items"]
except vk_api.exceptions.ApiError:
print("[!] ID " + str(target) + " is private. Skipping")
return -1
cache.add(starget, friends)
return friends
def get_artwork_favorite_count(artwork):
memcache_key = cache.MC_FAVORITE_COUNT + str(artwork.key().id())
result = cache.get(memcache_key)
if result:
return result
else:
fav = db.FavoriteCounter.all().filter('artwork =', artwork).get()
if fav:
result = fav.count
else:
result = 0
cache.add(memcache_key, result)
return result
def handle_result(self, rpc, url, type, X, Y):
result = rpc.get_result()
try:
if result.status_code == 200:
logging.info('%s tile X:%i Y:%i made it' % (type, X, Y))
self.addResult(result.content, type, X, Y)
logging.info('caching url %s' % url)
memcache.add(url, result.content)
cache.add(url, result.content, value_type="blob")
except:
logging.info('%s tile X:%i Y:%i failed' % (type, X, Y))
self.tilesDone = self.tilesDone + 1
def is_follower(leader_email, follower_email):
memcache_key = cache.MC_FAVORITE_BY_USER + leader_email + '_' + follower_email
result = cache.get(memcache_key)
if result == True or result == False:
return result
else:
follow = db.Follow.all().filter('leader_email =', leader_email).filter(
'follower_email =', follower_email).get()
if follow:
cache.add(memcache_key, True)
return True
else:
cache.add(memcache_key, False)
return False
def tag_by_url_name(url_name):
cache_tag = cache.get(cache.MC_TAG + url_name)
if cache_tag:
return cache_tag
else:
tag = db.Tag.all().filter('url_name =', url_name).get()
if tag:
cache.add(cache.MC_TAG + url_name, tag)
else:
tag = db.Tag()
tag.url_name = url_name
tag.title = url_name
tag.title_lower = url_name
return tag
def check_favorite(user_email, artwork_id):
cache_key = FAVORITE_CACHE_PREFIX + user_email
cache_value = cache.get(cache_key)
result = False
if cache_value is None:
result = True
else:
if artwork_id <> cache_value[VALUE_ARTWORK_ID]:
result = True
cache_value = {
VALUE_ARTWORK_ID: artwork_id
}
cache.add(cache_key, cache_value)
return result
def get_user_profile(user_email):
cache_value = cache.get(cache.MC_USER_PROFILE + user_email)
if cache_value:
return cache_value
else:
user_profile = db.UserProfile.all().filter('email =', user_email).get()
if user_profile:
cache.add(cache.MC_USER_PROFILE + user_email, user_profile)
return user_profile
else:
user_profile = db.UserProfile.all().filter('alternative_emails =',
user_email).get()
if user_profile:
cache.add(cache.MC_USER_PROFILE + user_email, user_profile)
return user_profile
else:
return None
def is_artwork_favorite_by_user(artwork, user_email):
if not user_email:
return False
memcache_key = cache.MC_FAVORITE_BY_USER + str(
artwork.key().id()) + '_' + user_email
result = cache.get(memcache_key)
if result == True or result == False:
return result
else:
fav_user = db.Favorite.all().filter('artwork =', artwork).filter(
'user_email =', user_email).get()
if fav_user:
cache.add(memcache_key, True)
return True
else:
cache.add(memcache_key, False)
return False
def get(self):
articles = Node.get_all_articles()
self.values['articles'] = articles
self.values['urls'] = Node.get_urls()
site_domain = self.values.get('site_domain', None)
site_domain_sync = Datum.get('site_domain_sync', None)
mentions_web = cache.get('mentions_web')
if mentions_web == None and site_domain:
link = ''.join([
'http://blogsearch.google.com/blogsearch_feeds?hl=en&q=',
urllib.quote('link:' + site_domain),
'&ie=utf-8&num=10&output=atom'
])
mentions_web = feedparser.parse(link)
cache.set('mentions_web', mentions_web, 600)
if mentions_web:
self.values['mentions_web'] = mentions_web.entries
else:
self.values['mentions_web'] = None
mentions_twitter = cache.get('mentions_twitter')
if mentions_twitter is None:
q = None
if site_domain_sync is None:
q = site_domain
else:
q = ' OR '.join([site_domain, site_domain_sync])
q = urllib.quote(q)
try:
result = urlfetch.fetch(
'http://search.twitter.com/search.json?q=' + q)
if result.status_code == 200:
mentions_twitter = simplejson.loads(result.content)
cache.add('mentions_twitter', mentions_twitter, 600)
except:
mentions_twitter = None
if mentions_twitter and len(mentions_twitter['results']) > 0:
self.values['mentions_twitter'] = mentions_twitter['results']
else:
self.values['mentions_twitter'] = None
self.render('backstage/overview.html', **self.values)
def check_comment(user_email, artwork_id, comment_text):
cache_key = COMMENT_CACHE_PREFIX + user_email
cache_value = cache.get(cache_key)
result = False
if cache_value is None:
result = True
else:
if datetime.now() - cache_value[VALUE_DATE] > COMMENT_MIN_INTERVAL:
if artwork_id <> cache_value[VALUE_ARTWORK_ID]:
result = True
else:
if comment_text <> cache_value[VALUE_COMMENT]:
result = True
cache_value = {
VALUE_DATE: datetime.now(),
VALUE_ARTWORK_ID: artwork_id,
VALUE_COMMENT: comment_text
}
cache.add(cache_key, cache_value)
return result
def get (self):
articles = Node.get_all_articles ()
self.values['articles'] = articles
self.values['urls'] = Node.get_urls ()
site_domain = self.values.get ('site_domain', None)
site_domain_sync = Datum.get ('site_domain_sync', None)
mentions_web = cache.get ('mentions_web')
if mentions_web == None and site_domain:
link = ''.join (['http://blogsearch.google.com/blogsearch_feeds?hl=en&q=',
urllib.quote('link:' + site_domain),
'&ie=utf-8&num=10&output=atom'])
mentions_web = feedparser.parse (link)
cache.set ('mentions_web', mentions_web, 600)
if mentions_web :
self.values ['mentions_web'] = mentions_web.entries
else:
self.values ['mentions_web'] = None
mentions_twitter = cache.get('mentions_twitter')
if mentions_twitter is None:
q = None
if site_domain_sync is None:
q = site_domain
else:
q = ' OR '.join ([site_domain, site_domain_sync])
q = urllib.quote (q)
try:
result = urlfetch.fetch('http://search.twitter.com/search.json?q=' + q)
if result.status_code == 200:
mentions_twitter = simplejson.loads(result.content)
cache.add('mentions_twitter', mentions_twitter, 600)
except:
mentions_twitter = None
if mentions_twitter and len(mentions_twitter['results']) > 0:
self.values['mentions_twitter'] = mentions_twitter['results']
else:
self.values['mentions_twitter'] = None
self.render ('backstage/overview.html', **self.values)
def get_series_ru(self, chat_id, message_id, search_query_id):
search_query = db.get_search_query(search_query_id)
series_number = search_query.query_text
key = f"avtonomer.get_series_ru({series_number})"
result = cache.get(key)
if not result:
result = avtonomer.search_ru(fastsearch="{}*{}".format(
series_number[:1],
series_number[1:],
), )
if result is not None:
result = result.total_results
if result is None:
logger.warning(f"No data for query {series_number}")
bot.send_message(
chat_id,
"Нет данных",
reply_to_message_id=message_id,
)
return
cache.add(key, result, timedelta(minutes=5))
url = avtonomer.get_series_ru_url(series_number)
series_number = avtonomer.translate_to_cyrillic(series_number)
message = (
f"В серии [{series_number}]({url}) пока нет ни одного номера"
if result == 0 else
f"Количество фотографий в серии [{series_number}]({url}): {result}")
bot.send_message(
chat_id,
message,
parse_mode="Markdown",
reply_to_message_id=message_id,
)
def post(self):
# Check parameters
name = self.request.get('name', None)
source_name = self.request.get('source', None)
if not name or not source_name:
logging.info('Unable to harvest without name and source')
self.error(400)
return
# Check cache for harvest job (handles polling)
key = harvest.get_job_cache_key(name, source_name)
job = cache.get(key)
if job:
self.response.set_status(200)
self.response.headers['Content-Type'] = "application/json"
self.response.out.write(simplejson.dumps(job))
return
else:
cache.add(key, harvest.get_job(name, source_name, 'new'))
# Add harvest task that targets harvest backed instance 1
params = dict(name=name, source=source_name)
taskqueue.add(url='/backend/harvest', queue_name='backend-harvest', eta=datetime.datetime.now(), target='harvest', params=params)
self.response.set_status(202) # Accepted
def post(self):
# Parameters checked by frontend
name = self.request.get('name')
source_name = self.request.get('source')
# Get source
source = sources.get(source_name)
if not source:
logging.error('Cannot harvest without a source')
self.error(404)
# Update job status to 'error'
job = get_job(name, source_name, 'error', msg='Unsupported source')
cache.add(key, job)
return
# Check cache for harvest job
key = get_job_cache_key(name, source_name)
job = cache.get(key)
if not job:
self.error(404)
self.response.headers['Content-Type'] = "application/json"
self.response.out.write('{"error":"unknown job %s"}' % key)
return
count = 0
# Update job status to 'working'
cache.add(key, get_job(name, source_name, 'working', msg=count))
# Get points from source and put them into datastore in batches
pcount = 0
for points in self.get_points(name, source):
logging.info('HARVEST BACKEND MEMORY = %s after %s points' % (runtime.memory_usage().current(), count))
entities = []
for p in points:
pkey = Key('Point', '%s-%s-%s' % (source_name, name, pcount))
pcount += 1
entities.append(Point(key=pkey, lat=p[0], lng=p[1]))
entities.append(PointIndex.create(pkey, p[0], p[1], name, source_name))
model.put_multi(entities)
count += len(points)
cache.add(key, get_job(name, source_name, 'working', msg=count))
# Update job status to 'done'
# TODO: Done now or after backend rendering completes?
cache.add(key, get_job(name, source_name, 'done', msg=count))
def cacheImage(self):
memcache.add(self.request.url, self.image)
cache.add(self.request.url, self.image,value_type="blob")
def testExpired(self):
self.assertEqual(cache.add("a",GOOD_RESPONSE,1),True)
time.sleep(1)
self.assertEqual(cache.get('a'),False)
def testClear(self):
cache.add("a",GOOD_RESPONSE,2)
self.assertEqual(cache.get('a'),'result')
self.assertEqual(cache.clear(),True)
self.assertEqual(cache.get('a'),False)
def testRemove(self):
cache.add("a",GOOD_RESPONSE,2)
self.assertEqual(cache.get('a'),'result')
self.assertEqual(cache.remove('a'),True)
self.assertEqual(cache.get('a'),False)
import metro_transit
import cache
from model import BusLocationList
from datetime import datetime
import time
cached_time = 60 # 60 seconds
def get_bus_locations(stop_number):
if cached_bus_locations_list := cache.fetch(stop_number, BusLocationList):
print('RETURN FROM CACHE')
return cached_bus_locations_list
else:
print('RETURN FROM API CALL')
bus_locations = metro_transit.get_bus_locations(stop_number)
bus_locations_list = BusLocationList(bus_locations, stop_number,
now_plus_expiry())
cache.add(bus_locations_list)
return bus_locations_list
def now_plus_expiry():
now = time.time()
return now + cached_time
def get(self):
newsfeed = None
if self.user_info.user:
newsfeed_artworks = db.NewsFeed.all().filter(
'user_email =',
self.user_info.user_email).order('-date').fetch(7)
newsfeed = [
convert.convert_artwork_for_page(a, 200, 150)
for a in newsfeed_artworks
]
if 0 < len(newsfeed) < 3:
for i in range(len(newsfeed), 3):
newsfeed.append({'not_found': True})
elif len(newsfeed) < 7:
newsfeed = newsfeed[0:3]
recent_artworks = cache.get(cache.MC_MAIN_PAGE_RECENT_IMAGES_KEY)
if not recent_artworks:
all_artworks = db.Artwork.all()
all_artworks = all_artworks.order('-date').fetch(3, 0)
recent_artworks = [
convert.convert_artwork_for_page(a, 200, 150)
for a in all_artworks
]
cache.add(cache.MC_MAIN_PAGE_RECENT_IMAGES_KEY, recent_artworks)
editor_choice = cache.get(cache.MC_MAIN_PAGE_RECENT_EDITOR_CHOICE)
if not editor_choice:
choice_artworks = db.Artwork.all().filter('editor_choice =', True)
choice_artworks = choice_artworks.order(
'-editor_choice_date').fetch(3, 0)
editor_choice = [
convert.convert_artwork_for_page(a, 200, 150)
for a in choice_artworks
]
cache.add(cache.MC_MAIN_PAGE_RECENT_EDITOR_CHOICE, editor_choice)
top_favorites = cache.get(cache.MC_MAIN_PAGE_TOP_FAVORITES)
if not top_favorites:
top_favorite_artworks = db.FavoriteCounter.all().order(
'-count').order('-date')
top_favorite_artworks = top_favorite_artworks.fetch(3, 0)
top_favorites = [
convert.convert_artwork_for_page(a, 200, 150)
for a in top_favorite_artworks
]
cache.add(cache.MC_MAIN_PAGE_TOP_FAVORITES, top_favorites)
recent_favorites = cache.get(cache.MC_MAIN_PAGE_RECENT_FAVORITES)
if not recent_favorites:
recent_favorites_artworks = db.Favorite.all().order('-date')
recent_favorites_artworks = recent_favorites_artworks.fetch(3, 0)
recent_favorites = [
convert.convert_artwork_for_page(a, 200, 150)
for a in recent_favorites_artworks
]
cache.add(cache.MC_MAIN_PAGE_RECENT_FAVORITES, recent_favorites)
recent_comments = cache.get(cache.MC_MAIN_PAGE_RECENT_COMMENTS)
if not recent_comments:
comments = db.Comment.all().order('-date').fetch(5, 0)
recent_comments = [
convert.convert_comment_for_page_rich(c) for c in comments
]
cache.add(cache.MC_MAIN_PAGE_RECENT_COMMENTS, recent_comments)
productive_artists = cache.get(cache.MC_MAIN_PAGE_PRODUCTIVE_ARTISTS)
if not productive_artists:
p_artists = db.UserProfile.all().order('-artworks_count').fetch(5)
productive_artists = [
convert.convert_user_profile(a) for a in p_artists
]
cache.add(cache.MC_MAIN_PAGE_PRODUCTIVE_ARTISTS,
productive_artists)
top_rated_artists = cache.get(cache.MC_MAIN_PAGE_TOP_RATED_ARTISTS)
if not top_rated_artists:
r_artists = db.UserProfile.all().order('-favorite_count').fetch(5)
top_rated_artists = [
convert.convert_user_profile(a) for a in r_artists
]
cache.add(cache.MC_MAIN_PAGE_TOP_RATED_ARTISTS, top_rated_artists)
last_week_favorites = cache.get(cache.MC_MAIN_PAGE_LAST_WEEK_FAVORITES)
if not last_week_favorites:
last_week_favorites = db.LastWeekFavoriteCounters.all().order(
'-count')
last_week_favorites = last_week_favorites.fetch(3, 0)
last_week_favorites = [
convert.convert_artwork_for_page(a, 200, 150)
for a in last_week_favorites
]
cache.add(cache.MC_MAIN_PAGE_LAST_WEEK_FAVORITES,
last_week_favorites)
last_month_favorites = cache.get(
cache.MC_MAIN_PAGE_LAST_MONTH_FAVORITES)
if not last_month_favorites:
last_month_favorites = db.LastMonthFavoriteCounters.all().order(
'-count')
last_month_favorites = last_month_favorites.fetch(3, 0)
last_month_favorites = [
convert.convert_artwork_for_page(a, 200, 150)
for a in last_month_favorites
]
cache.add(cache.MC_MAIN_PAGE_LAST_MONTH_FAVORITES,
last_month_favorites)
self.write_template(
'templates/index.html', {
'artworks': recent_artworks,
'editor_choice': editor_choice,
'top_favorites': top_favorites,
'last_week_favorites': last_week_favorites,
'last_month_favorites': last_month_favorites,
'recent_favorites': recent_favorites,
'comments': recent_comments,
'productive_artists': productive_artists,
'top_rated_artists': top_rated_artists,
'newsfeed': newsfeed,
'og_title': 'Grid Paint',
'og_description':
'An online tool for pixel drawing with different shapes of pixels.',
'og_url': 'https://grid-paint.com',
'og_image': 'https://grid-paint.com/img/grid-paint-poster.png'
})
def create_gallery_model(offset_param,
artworks_query_func,
href_create_func,
memcache_cursor_key_func,
additional_values_func=None):
"""
Create a model of artwork list based on request.
offset_param - offset parameter from request
artworks_query_func() - function for create query for artworks
href_create_func(offset) - function for create next and prev page hyperlinks
memcache_cursor_key_func(offset) - function to generate keys for cursors stored in MemCache
additional_values_func(object, values_dict) - function extracts additional values from objects of query result
"""
if offset_param:
offset = int(offset_param)
else:
offset = 0
if offset < 0:
offset = 0
if offset == 0:
fetch_count = page_size + 1
else:
fetch_count = page_size
query = artworks_query_func()
if offset == 0:
all_artworks = query.run(limit=fetch_count + 1)
else:
cursor = cache.get(memcache_cursor_key_func(offset))
if cursor:
query = query.with_cursor(start_cursor=cursor)
all_artworks = query.run(limit=fetch_count + 1)
else:
all_artworks = query.run(limit=fetch_count + 1, offset=offset)
cache.add(memcache_cursor_key_func(offset), query.cursor())
has_prev_page = (offset > 0)
has_next_page = False
index = 0
artworks = []
for a in all_artworks:
index = index + 1
if index > fetch_count:
has_next_page = True
else:
converted_artwork = convert.convert_artwork_for_page(a, 200, 150)
if additional_values_func:
additional_values_func(a, converted_artwork)
artworks.append(converted_artwork)
if index == fetch_count:
cache.add(memcache_cursor_key_func(offset + fetch_count),
query.cursor())
next_page_href = href_create_func(offset + fetch_count)
if offset - page_size <= 1:
prev_page_href = href_create_func(0)
else:
prev_page_href = href_create_func(offset - page_size)
return {
'has_next_page': has_next_page,
'has_prev_page': has_prev_page,
'next_page_href': next_page_href,
'prev_page_href': prev_page_href,
'artworks': artworks
}
def explore (root):
todolist.append( root )
count = 0
while len(todolist) != 0:
log('[debug] todolist has {0} nodes'.format (len(todolist)))
if len(todolist) == MAXTODOLEN:
if (count == MAXTODOLEN):
log ('\n\n\n ******* Stuck in a loop.. hence exiting !! ********\n\n')
print '\n\n\n ******* Stuck in a loop.. hence exiting !! ********\n\n'
return
count += 1
else:
count = 0
n = todolist.pop(0)
blist = []
#log('[debug] n.title : {0}\n does_exist returns : {1}'.format (n.title, graph.does_exist(n.title)) )
if graph.does_exist (n.title) == False:
try:
n.setinfo ()
if n.authorlist == None:
log('\n\n[error] node.setinfo rendered authorlist None')
raise Exception
except:
log('[error] node.setinfo failed for : {0} url[{1}]\n'.format (n.title, n.url) )
cache.add ( n.title, '[failure] '+n.url)
continue
try:
graph.addnode (n)
except:
log("[error] <addnode failed for {0}> {1}".format ( n.title ,str(traceback.print_stack()) ))
continue
for b in n.blist:
graph.addedge (n, b)
blist = n.blist
else:
log('[debug] ** traversing already crawled node **')
blist = graph.get_neighbours(n.title)
cache.add ( n.title, n.url )
print blist
for b in blist:
if len(todolist) == MAXTODOLEN :
log('[debug] ** MAXTODOLEN hit !! **')
continue
if cache.get(b.title) == None:
todolist.append(b)
else:
log('[debug] ** Already cached !! **')
log ('\n\n\n[finished] nothing left in todolist.\n\n')
def _start_thread(self, client_conn, client_addr):
r = client_conn.recv(config['MAX_LEN'])
request = Request(r)
if (request.host is None) :
client_conn.close()
return
client_conn.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
cacheData = cache.get(request.path)
if cacheData is not None:
client_conn.send(cacheData)
client_conn.close()
return
# Site Blocker
if blocked_sites.isBlocked(request.host):
client_conn.send(blocked_sites.not_allowed(request.host).encode())
return
# blocked_sites.print()
print("REQUESTED HOST: " + request.host)
server_conn = None
fst = True
try :
if (request.command == "GET") :
if (request.port is None) :
request.port = 80
server_conn = self._create_sock(request)
server_conn.sendall(r)
# print("GET: " + request.path)
filter = contentfilter.ContentFilter("config")
while 1:
data = server_conn.recv(config['MAX_LEN']) # receive data from web server
if (len(data) > 0):
if filter.notGZIPPath(request.path):
client_conn.send(data)
continue
try:
data_filtered = filter.storeData(data)
client_conn.send(data_filtered)
if data_filtered != b"":
cache.add(request.path, data_filtered)
except UnicodeDecodeError:
client_conn.send(data)
except OSError as e:
if "Not a gzipped file" in str(e):
client_conn.send(data)
else:
break
elif (request.command == "CONNECT") :
if (request.port is None) :
request.port = 443
server_conn = self._create_sock(request)
conn_resp = "HTTP/1.1 200 Connection established\r\n"+"Proxy-agent: Pyx\r\n"+"\r\n"
client_conn.sendall(conn_resp.encode())
# print("sent established connection")
data = client_conn.recv(config['MAX_LEN'])
server_conn.send(data)
# print("data sent")
while (1) :
data = server_conn.recv(config['MAX_LEN'])
if (len(data) > 0):
client_conn.send(data)
else:
break
if server_conn is not None:
server_conn.close()
if client_conn:
client_conn.close()
except socket.error as error_msg:
print ("ERROR: ",client_addr, error_msg)
if server_conn:
server_conn.close()
if client_conn:
client_conn.close()
