def get_by_slug_querier(self, slug, querier):
'''
Obtiene un evento por su id, COMPRUEBA VISIBILIDAD, solo obtiene los publicos
'''
suggestion = memcache.deserialize_instances(memcache.get(
'%sEVENT%s' % (memcache.version, slug)),
_search_class=self._klass)
if suggestion is None:
suggestion = self._klass.all().filter('slug =', slug).get()
if suggestion is None:
try:
suggestion = self.get_by_id(slug)
except:
return None
if suggestion is not None:
if suggestion._is_private():
if not querier.is_authenticated():
return None
if suggestion.__class__.user.get_value_for_datastore(
suggestion) == querier.key():
return None
elif suggestion._is_shared():
if not querier.is_authenticated():
return None
if not suggestion.user_invited(querier):
return None
memcache.set('%sEVENT%s' % (memcache.version, slug),
memcache.serialize_instances(suggestion), 300)
return suggestion
return suggestion
def get_top_voted(self, instance, querier):
if querier is None:
raise TypeError
if instance is not None:
import memcache
top = None #memcache.get(memcache.get('%stopcomments_%s' % (memcache.version, instance.key())))
if top is None:
from google.appengine.api import datastore
top = datastore.Query('Comment', {'instance =': instance.key(), 'votes >': 0})
top.Order(('votes', datastore.Query.DESCENDING))
top = top.Get(3)
if len(top) > 0:
from georemindme.funcs import prefetch_refpropsEntity
prefetch = prefetch_refpropsEntity(top, 'user')
return [
{'id': comment.key().id(),
'username': prefetch[comment['user']].username,
'has_voted': Vote.objects.user_has_voted(querier, comment.key()) if querier is not None else None,
'vote_counter': comment['votes'],
'msg': comment['msg'],
'created': comment['created'],
} for comment in top
]
memcache.set('%stopcomments_%s' % (memcache.version, instance.key()), top, 300)
return top
def get_userprefs(user, clear=False):
"""
Get the UserPrefs for the current user either from memcache or, if not
yet cached, from the datastore and put it into memcache. Used by
UserPrefs.from_user(user)
"""
if not user:
return user
if user.federated_identity():
key = "userprefs_fid_%s" % user.federated_identity()
else:
key = "userprefs_gid_%s" % user.user_id()
# Clearing the cache does not return anything
if clear:
memcache.delete(key)
logging.info("- cache cleared key: %s", key)
return
# Try to grab the cached UserPrefs
prefs = memcache.get(key)
if prefs:
logging.info("- returning cached userprefs for key: %s", key)
return prefs
# If not cached, query the datastore, put into cache and return object
prefs = models.UserPrefs._from_user(user)
memcache.set(key, prefs)
logging.info("cached userprefs key: %s", key)
return prefs
def block_contacts(request):
"""
Bloquea a un usuario para no recibirlo en la lista
de amigos sugeridos
Parametros en POST:
userid: el id del usuario a bloquear
:returns: boolean
"""
if request.user.is_authenticated():
userid = int(request.POST['userid'])
if userid is not None:
import memcache
request.user.settings.blocked_friends_sug.append(userid)
friends = memcache.get('%sfriends_to_%s' %
(memcache.version, request.user.key()))
if friends is not None and userid in friends:
del friends[userid]
memcache.set(
'%sfriends_to_%s' % (memcache.version, request.user.key()),
friends, 300)
request.user.settings.put()
return HttpResponse(simplejson.dumps(True),
mimetype="application/json")
return HttpResponse(simplejson.dumps(True), mimetype="application/json")
def get(self):
#get 'avatar' is avatar_id
#Add memcached here to improve the performence.
usravatardata = memcache.get('img_useravatar' + self.request.get("avatar"))
if usravatardata is not None:
self.output_avatar(usravatardata, True)
else:
# Request it from BigTable
AvatarUser = tarsusaUser.get_by_id(int(self.request.get("avatar")))
try:
if AvatarUser.avatar:
if not memcache.set('img_useravatar' + self.request.get("avatar"), AvatarUser.avatar, 16384):
logging.error("Memcache set failed: When Loading avatar_image")
self.output_avatar(AvatarUser.avatar, False)
except AttributeError:
#Not found avatar in DB.
avatardata = urlfetch.fetch("http://www.checknerds.com/img/default_avatar.jpg", headers={'Content-Type': "image/jpg"})
if avatardata.status_code == 200:
avatardata = avatardata.content
memcache.set('img_useravatar' + self.request.get("avatar"), avatardata, 16384)
self.output_avatar(avatardata, False)
else:
self.redirect("/img/default_avatar.jpg")
def drill(self, path, choices, complete=False):
'''Perform a data item drilldown.
If all necessary choices are specified, returns the UID of the data item;
otherwise, returns the next choice that needs to be made in the form of a dict
with keys "name" and "choices". (The "choices" item is an array of permitted
values.)
If the "complete" argument is true, we raise an Error if the specified choices
are incomplete. In this case the return value will always be the UID.
Results are cached using memcache.
Typical applications will not call this method directly: it is used internally
by Profile.create_item(s). You could call it directly if you wanted to allow
a user to specify data items interactively one choice at a time.
'''
choices_string = urllib.urlencode(choices)
memcache_key = ";".join((self.server, path, choices_string, str(complete)))
cached_result = memcache.get(memcache_key, namespace=MEMCACHE_NAMESPACE)
if cached_result is not None:
return cached_result
result = self._drill(path, choices_string, complete)
memcache.set(memcache_key, result, namespace=MEMCACHE_NAMESPACE)
return result
def get(cls,name):
value = memcache.get('APP_Key_'+name)
if not value:
q = cls.query(cls.name == name).get()
value = ''
if q:
value = q.value
memcache.set('APP_Key_'+name,value)
return value
def login():
if request.method == 'POST':
instagram_username = request.form['userEmailID']
instagram_password = request.form['userLoginPassword']
#session['insta_username'] = instagram_username
#session['insta_password'] = instagram_password
# client.set(instagram_username, 0)
# session['request_accepted_counter_demo'] = 0
insta_bot = InstagramBot(instagram_username, instagram_password)
insta_login_response = insta_bot.login()
insta_bot.closeBrowser()
if insta_login_response == False:
msg = 'Invalid Credentails'
return render_template('index.html', msg=msg)
if insta_login_response:
user_obj = Users.query.filter_by(
insta_username=instagram_username).first()
session['insta_username'] = instagram_username
session['insta_password'] = instagram_password
client.set(instagram_username, 0)
if not user_obj:
new_user = Users(insta_username=instagram_username)
db.session.add(new_user)
db.session.commit()
user = Users.query.filter_by(insta_username=instagram_username).first()
if insta_login_response and user is not None:
user.is_subscribed = True
user.till_date = datetime.datetime.utcnow() + timedelta(days=1)
if user.is_subscribed:
if datetime.datetime.utcnow() < user.till_date:
ok = login_user(user)
next = request.args.get('next')
if next is None or not next[0] == '/':
next = url_for('users.accept_pending_requests')
return redirect(next)
if user.is_subscribed == False:
try:
if datetime.datetime.utcnow() > user.till_date:
user.till_date = None
user.from_date = None
user.is_subscribed = False
db.session.commit()
except BaseException:
ok = login_user(user)
next = request.args.get('next')
if next is None or not next[0] == '/':
next = url_for('core.pricing')
return redirect(next)
def _encode_img(self, file_path):
"""Returns image base64 string representation and makes a cache file"""
import memcache
filename = file_path.rpartition(os.sep)[2]
cache_file = "%s_cache" % file_path
cached_image = memcache.get('%s%s' % (memcache.version, cache_file))
if cached_image is None:
image = open(file_path)
cached_image = "data:image;base64,%s"%base64.b64encode(image)
memcache.set('%s%s' % (memcache.version, cache_file), cached_image, 300)
return cached_image
def _encode_img(self, file_path):
"""Returns image base64 string representation and makes a cache file"""
import memcache
filename = file_path.rpartition(os.sep)[2]
cache_file = "%s_cache" % file_path
cached_image = memcache.get('%s%s' % (memcache.version, cache_file))
if cached_image is None:
image = open(file_path)
cached_image = "data:image;base64,%s" % base64.b64encode(image)
memcache.set('%s%s' % (memcache.version, cache_file), cached_image,
300)
return cached_image
def get_friends_to_follow(self, provider=None, rpc=False):
import memcache
friends = memcache.get('%sfriends_to_%s' %
(memcache.version, self.key()))
if friends is None:
friends_rpc = [] # lista de rpcs
#RPC PARA FACEBOOK
try:
if provider is None or provider.lower() == 'facebook':
from geoauth.clients.facebook import FacebookFriendsRPC
fb_rpc = FacebookFriendsRPC()
friends_rpc.append(fb_rpc.fetch_friends(self))
except:
pass
#RPC PARA TWITTER
try:
if provider is None or provider.lower() == 'twitter':
from geoauth.clients.twitter import TwitterFriendsRPC
tw_rpc = TwitterFriendsRPC()
friends_rpc.append(tw_rpc.fetch_friends(self))
except:
raise
pass
#RPC PARA GOOGLE
try:
if provider is None or provider.lower() == 'google':
from geoauth.clients.google import GoogleFriendsRPC
go_rpc = GoogleFriendsRPC()
friends_rpc.append(go_rpc.fetch_friends(self))
except:
pass
friends_rpc = filter(None, friends_rpc)
if rpc:
return [fb_rpc, tw_rpc, go_rpc], friends_rpc
raise NotImplementedError
# TODO: SI NO RPC, PROCESAR TODO EN ESTA FUNCION
for rpc in friends_rpc:
rpc.wait()
friends = {} # diccionario con todos los amigos
#los unimos en uno
friends.update(fb_rpc.friends)
friends.update(tw_rpc.friends)
#friends.update(friends_google)
if len(friends) > 0:
if len(self.settings.blocked_friends_sug) > 0:
for k in friends.keys():
if k in self.settings.blocked_friends_sug:
del friends[k]
memcache.set(
'%sfriends_to_%s' % (memcache.version, self.key()),
friends, 11235)
return friends
def get_by_id(self, id):
import memcache
try:
id = long(id)
except:
return None
place = memcache.deserialize_instances(memcache.get('%splace_%s' % (memcache.version, id)), _search_class=Place)
if place is None:
place = self._klass.get_by_id(id)
if place is None:
return None
memcache.set('%splace_%s' % (memcache.version, id), memcache.serialize_instances(place), 300)
return place
def image_avatar(request, username):
from georemindme.model_plus import memcache, version
encoded_image = memcache.get('%s%s_avatarcachebase64' % (version, username))
if encoded_image is None:
import base64
location = get_avatar(request, username)
from google.appengine.api import urlfetch
result = urlfetch.fetch(location['Location'])
decoded_image = result.content
encoded_image = "data:image;base64,%s" % base64.b64encode(result.content)
memcache.set('%s%s_avatarcachebase64' % (version, username), encoded_image, 1123)
else:
decoded_image=base64.b64decode(encoded_image)
return HttpResponse(decoded_image, mimetype="image/xyz")
def __str__(self): try: memcache_key = '%s:widget:%s:%s' % (self.handler.auth, self.gen_func_name, self.params) html = memcache.get(memcache_key) if html == None: html = self.gen_func(self.handler.template_values, self.params) if html == None: html = '' memcache.set(memcache_key, html) return html except Exception, e: return str(e)
def embedded_avatar(username):
import memcache
encoded_image = memcache.get('%s%s_avatarcachebase64' % (memcache.version, username))
if encoded_image is None:
from geouser.views import get_avatar
try:
image_url = get_avatar(None, username)
from google.appengine.api import urlfetch
result = urlfetch.fetch(image_url['Location'])
encoded_image = "data:image;base64,%s" % base64.b64encode(result.content)
memcache.set('%s%s_avatarcachebase64' % (memcache.version, username), encoded_image, 1123)
except:
return 'https://georemindme.appspot.com/static/facebookApp/img/no_avatar.png'
return encoded_image
def get_friends_to_follow(self, provider = None, rpc=False):
import memcache
friends = memcache.get('%sfriends_to_%s' % (memcache.version, self.key()))
if friends is None:
friends_rpc = [] # lista de rpcs
#RPC PARA FACEBOOK
try:
if provider is None or provider.lower() == 'facebook':
from geoauth.clients.facebook import FacebookFriendsRPC
fb_rpc = FacebookFriendsRPC()
friends_rpc.append(fb_rpc.fetch_friends(self))
except:
pass
#RPC PARA TWITTER
try:
if provider is None or provider.lower() == 'twitter':
from geoauth.clients.twitter import TwitterFriendsRPC
tw_rpc = TwitterFriendsRPC()
friends_rpc.append(tw_rpc.fetch_friends(self))
except:
raise
pass
#RPC PARA GOOGLE
try:
if provider is None or provider.lower() == 'google':
from geoauth.clients.google import GoogleFriendsRPC
go_rpc = GoogleFriendsRPC()
friends_rpc.append(go_rpc.fetch_friends(self))
except:
pass
friends_rpc = filter(None, friends_rpc)
if rpc:
return [fb_rpc, tw_rpc, go_rpc], friends_rpc
raise NotImplementedError
# TODO: SI NO RPC, PROCESAR TODO EN ESTA FUNCION
for rpc in friends_rpc:
rpc.wait()
friends = {} # diccionario con todos los amigos
#los unimos en uno
friends.update(fb_rpc.friends)
friends.update(tw_rpc.friends)
#friends.update(friends_google)
if len(friends) > 0:
if len(self.settings.blocked_friends_sug)>0:
for k in friends.keys():
if k in self.settings.blocked_friends_sug:
del friends[k]
memcache.set('%sfriends_to_%s' % (memcache.version, self.key()), friends, 11235)
return friends
def memexecute():
limit = request.form['limit']
cur.execute(query + limit)
result = cur.fetchall()
memcache.set(hash, result)
c = 0
for res in result:
c = c + 1
print(str(c) + ':' + str(res))
starttime = time.time()
memresult = memcache.get(hash)
endtime = time.time()
total = endtime - starttime
print('Time taken by memcache ', total)
return render_template('filehandle.html', rdstime2=total)
def get_by_id(self, id):
import memcache
try:
id = long(id)
except:
return None
place = memcache.deserialize_instances(memcache.get(
'%splace_%s' % (memcache.version, id)),
_search_class=Place)
if place is None:
place = self._klass.get_by_id(id)
if place is None:
return None
memcache.set('%splace_%s' % (memcache.version, id),
memcache.serialize_instances(place), 300)
return place
def render(self, name, values=None):
if values == None:
values = self.template_values
html = memcache.get('%s:page:%s' % (self.login_user, self.request.path_qs))
if html == None:
try:
html = template.render(name, values)
except TemplateSyntaxError, e: # if theme files are not found, fall back to default theme
logging.warning(e)
settings.theme
self.redirect(settings.home_page)
return
memcache.set('%s:page:%s' % (self.login_user, self.request.path_qs), html)
self.response.out.write(html)
def image_avatar(request, username):
from georemindme.model_plus import memcache, version
encoded_image = memcache.get('%s%s_avatarcachebase64' %
(version, username))
if encoded_image is None:
import base64
location = get_avatar(request, username)
from google.appengine.api import urlfetch
result = urlfetch.fetch(location['Location'])
decoded_image = result.content
encoded_image = "data:image;base64,%s" % base64.b64encode(
result.content)
memcache.set('%s%s_avatarcachebase64' % (version, username),
encoded_image, 1123)
else:
decoded_image = base64.b64decode(encoded_image)
return HttpResponse(decoded_image, mimetype="image/xyz")
def setCache(key, item, ttl=600):
"""
Store an item in memcache, if running
"""
if len(key) > 0 and len(memcache_address) > 0 and memcache:
if len(key) > 250:
m = md5.new()
m.update(key)
key = str(m.hexdigest())
if appengine:
memcache.set(key, item, ttl)
else:
mc = memcache.Client(memcache_address)
tt = mc.set(key, item, ttl)
return True
else:
return False
def get_someitems(clear=False):
"""Boilerplate for your customization"""
if clear:
memcache.delete("someitems")
return
someitems = memcache.get("someitems")
if someitems:
#logging.info("return cached someitem")
return someitems
someitems = []
for someitem in Someitem.all().fetch(100):
someitems.append(someitem)
memcache.set("someitems", someitems)
logging.info("cached someitems")
return someitems
def token():
token = None
memcache = cache.get_cache()
token = memcache.get('token')
if token:
LOG.debug('get token from memcache.')
return token
ks = keystone_client.KeystoneClient.from_url()
token = ks.get_token()
if token:
memcache.set('token', token, TOKEN_CACHE_SECONDS)
LOG.debug('get token from keystone.')
return token
else:
LOG.error('get token failed both from cache and keystone')
def embedded_avatar(username):
import memcache
encoded_image = memcache.get('%s%s_avatarcachebase64' %
(memcache.version, username))
if encoded_image is None:
from geouser.views import get_avatar
try:
image_url = get_avatar(None, username)
from google.appengine.api import urlfetch
result = urlfetch.fetch(image_url['Location'])
encoded_image = "data:image;base64,%s" % base64.b64encode(
result.content)
memcache.set(
'%s%s_avatarcachebase64' % (memcache.version, username),
encoded_image, 1123)
except:
return 'https://georemindme.appspot.com/static/facebookApp/img/no_avatar.png'
return encoded_image
def render_GET (self, request):
params = {"id": int(request.args[b"id"][0].decode("utf-8")),
"width": request.args[b"width"][0].decode("utf-8"),
"height": request.args[b"height"][0].decode("utf-8"),
"format": request.args[b"format"][0].decode("utf-8"),
"resolution": int(request.args[b"resolution"][0].decode("utf-8"))
}
series = schema.table.series
with schema.select("series", series.id==params["id"]) as select:
for s in select.all():
cache_id = "spark_%s_%s_%s_%s_%s" % (params["id"], params["width"], params["height"], params["format"], params["resolution"])
cached = memcache.get(cache_id)
request.setHeader(b'Content-Type', b'image/png')
if cached and cached["lm"] == s.last_modified:
request.write(cached["res"])
else:
if params["format"] == "ohlcv":
data = DataFrameWrapper(s, resolution=params["resolution"])
if len(data.dframe) > 0 and "close" in data.dframe.columns.values:
plot = spark.Spark.plot(data=data.dframe["close"]
, volume=data.dframe["volume"]
, enable_volume=True
, width=float(request.args[b"width"][0])
, height=float(request.args[b"height"][0]))
cached = {"lm": s.last_modified, "res": plot.getvalue()}
memcache.set(cache_id, cached)
request.write(cached["res"])
else:
#request.write()
pass
elif params["format"] == "tick":
data = DataFrameWrapper(s, no_index=True, resolution=params["resolution"])
plot = spark.Spark.plot(data=data.dframe["price"]
, volume=0
, enable_volume=True
, width=float(request.args[b"width"][0])
, height=float(request.args[b"height"][0]))
cached = {"lm": s.last_modified, "res": plot.getvalue()}
memcache.set(cache_id, cached)
request.write(cached["res"])
return b""
def add_following(self, followname=None, followid=None):
'''
Añade un usuario a la lista de personas que sigue self.
El usuario se busca a por username o por id
:param followname: El nombre de usuario a seguir
:type followname: :class:`string`
:param followid: El identificador del usuario a seguir
:type followid: :class:`string`
'''
if followname is not None:
following = User.objects.get_by_username(followname,
keys_only=True)
elif followid is not None:
following = User.objects.get_by_id(followid, keys_only=True)
else:
raise AttributeError()
if following is not None:
if following == self.key():
return True
# actualiza la cache de amigos sugeridos
import memcache
friends = memcache.get('%sfriends_to_%s' %
(memcache.version, self.key()))
if friends is not None and int(following.id()) in friends:
del friends[int(following.id())]
memcache.set(
'%sfriends_to_%s' % (memcache.version, self.key()),
friends, 300)
# añadimos el seguidor
from models_acc import UserFollowingIndex
is_following = UserFollowingIndex.all().filter(
'following =', following).ancestor(self.key()).count()
if is_following != 0: # en este caso, el usuario ya esta siguiendo al otro, no hacemos nada mas.
return True
following_result = self.following(
async=True) # obtiene un iterador con los UserFollowingIndex
if self._add_follows(following_result, following):
from signals import user_follower_new
user_follower_new.send(sender=self, following=following)
return True
return False
def get_by_id(self, id):
'''
Obtiene un evento por su id, COMPRUEBA VISIBILIDAD, solo obtiene los publicos
'''
try:
id = int(id)
except:
raise TypeError
event = memcache.deserialize_instances(memcache.get(
'%sEVENT%s' % (memcache.version, id)),
_search_class=self._klass)
if event is None:
event = self._klass.get_by_id(int(id))
if not hasattr(event, '_vis'):
return None
if not event._is_public():
return None
memcache.set('%sEVENT%s' % (memcache.version, id),
memcache.serialize_instances(event), 300)
return event
def useNonce(self, server_url, timestamp, salt):
if abs(timestamp - time.time()) > nonce.SKEW:
return False
key = self._nonceKey(server_url, timestamp, salt)
result = memcache.get(key)
if (result == None):
return memcache.set(key, True, nonce.SKEW + 5)
else:
return False
def block_contacts(request):
"""
Bloquea a un usuario para no recibirlo en la lista
de amigos sugeridos
Parametros en POST:
userid: el id del usuario a bloquear
:returns: boolean
"""
if request.user.is_authenticated():
userid = int(request.POST['userid'])
if userid is not None:
import memcache
request.user.settings.blocked_friends_sug.append(userid)
friends = memcache.get('%sfriends_to_%s' % (memcache.version, request.user.key()))
if friends is not None and userid in friends:
del friends[userid]
memcache.set('%sfriends_to_%s' % (memcache.version, request.user.key()), friends, 300)
request.user.settings.put()
return HttpResponse(simplejson.dumps(True), mimetype="application/json")
return HttpResponse(simplejson.dumps(True), mimetype="application/json")
def get_top_voted(self, instance, querier):
if querier is None:
raise TypeError
if instance is not None:
import memcache
top = None #memcache.get(memcache.get('%stopcomments_%s' % (memcache.version, instance.key())))
if top is None:
from google.appengine.api import datastore
top = datastore.Query('Comment', {
'instance =': instance.key(),
'votes >': 0
})
top.Order(('votes', datastore.Query.DESCENDING))
top = top.Get(3)
if len(top) > 0:
from georemindme.funcs import prefetch_refpropsEntity
prefetch = prefetch_refpropsEntity(top, 'user')
return [{
'id':
comment.key().id(),
'username':
prefetch[comment['user']].username,
'has_voted':
Vote.objects.user_has_voted(querier, comment.key())
if querier is not None else None,
'vote_counter':
comment['votes'],
'msg':
comment['msg'],
'created':
comment['created'],
} for comment in top]
memcache.set(
'%stopcomments_%s' %
(memcache.version, instance.key()), top, 300)
return top
def add_following(self, followname = None, followid = None):
'''
Añade un usuario a la lista de personas que sigue self.
El usuario se busca a por username o por id
:param followname: El nombre de usuario a seguir
:type followname: :class:`string`
:param followid: El identificador del usuario a seguir
:type followid: :class:`string`
'''
if followname is not None:
following = User.objects.get_by_username(followname, keys_only=True)
elif followid is not None:
following = User.objects.get_by_id(followid, keys_only=True)
else:
raise AttributeError()
if following is not None:
if following == self.key():
return True
# actualiza la cache de amigos sugeridos
import memcache
friends = memcache.get('%sfriends_to_%s' % (memcache.version, self.key()))
if friends is not None and int(following.id()) in friends:
del friends[int(following.id())]
memcache.set('%sfriends_to_%s' % (memcache.version, self.key()), friends, 300)
# añadimos el seguidor
from models_acc import UserFollowingIndex
is_following = UserFollowingIndex.all().filter('following =', following).ancestor(self.key()).count()
if is_following != 0: # en este caso, el usuario ya esta siguiendo al otro, no hacemos nada mas.
return True
following_result = self.following(async=True) # obtiene un iterador con los UserFollowingIndex
if self._add_follows(following_result, following):
from signals import user_follower_new
user_follower_new.send(sender=self, following=following)
return True
return False
def _post_put(self, **kwargs):
import memcache
memcache.delete('%s%s_avatarcachebase64' % (memcache.version, self.username))
memcache.set('%s%s' % (memcache.version, self.key().name()), memcache.serialize_instances(self), 300)
def storeAssociation(self, server_url, assoc):
memcache.set(self._serverKey(server_url), assoc, assoc.lifetime)
memcache.set(self._assocKey(server_url, assoc.handle), assoc, assoc.lifetime)
def set_user_cache(key, user): last_login = datetime.utcnow() memcache.set(key, (user, last_login))
def _post_put(self, **kwargs):
import memcache
memcache.set('%s%s' % (memcache.version, self.key().name()),
memcache.serialize_instances(self), 300)
while time.time() < startTime + MINIMUM_RUN_PERIOD or memcache.get(
trackerRequiredKey):
cycleStartTime = time.time()
# build stitched manifest
manifest = manifestStitcher.getUpdatedManifest(liveStreamUrl,
adPositionsKey)
# update the last used segment in memcache
if manifestStitcher.lastUsedSegment != None:
# Note: there is a race here between the get & set, but it shouldn't be a problem since trackers
# working on the same entry will more or less synchronized, if they aren't it's a problem anyway...
savedLastUsedSegment = memcache.get(lastUsedSegmentKey)
if savedLastUsedSegment == None or manifestStitcher.lastUsedSegment > int(
savedLastUsedSegment):
writeOutput('setting last used segment to %s' %
manifestStitcher.lastUsedSegment)
memcache.set(lastUsedSegmentKey,
str(manifestStitcher.lastUsedSegment),
RESULT_MANIFEST_EXPIRY)
# save the result to memcache
memcache.set(outputMemcacheKey, manifest, RESULT_MANIFEST_EXPIRY)
memcache.touch(ffmpegParamsKey, RESULT_MANIFEST_EXPIRY)
# sleep until next cycle
curTime = time.time()
sleepTime = cycleStartTime + CYCLE_INTERVAL - curTime
if sleepTime > 0:
time.sleep(sleepTime)
writeOutput('Quitting...')
executeCommand(cmdLine)
# main loop
manifestStitcher = ManifestStitcher()
startTime = time.time()
while time.time() < startTime + MINIMUM_RUN_PERIOD or memcache.get(trackerRequiredKey):
cycleStartTime = time.time()
# build stitched manifest
manifest = manifestStitcher.getUpdatedManifest(liveStreamUrl, adPositionsKey)
# update the last used segment in memcache
if manifestStitcher.lastUsedSegment != None:
# Note: there is a race here between the get & set, but it shouldn't be a problem since trackers
# working on the same entry will more or less synchronized, if they aren't it's a problem anyway...
savedLastUsedSegment = memcache.get(lastUsedSegmentKey)
if savedLastUsedSegment == None or manifestStitcher.lastUsedSegment > int(savedLastUsedSegment):
writeOutput('setting last used segment to %s' % manifestStitcher.lastUsedSegment);
memcache.set(lastUsedSegmentKey, str(manifestStitcher.lastUsedSegment), RESULT_MANIFEST_EXPIRY)
# save the result to memcache
memcache.set(outputMemcacheKey, manifest, RESULT_MANIFEST_EXPIRY)
memcache.touch(ffmpegParamsKey, RESULT_MANIFEST_EXPIRY)
# sleep until next cycle
curTime = time.time()
sleepTime = cycleStartTime + CYCLE_INTERVAL - curTime
if sleepTime > 0:
time.sleep(sleepTime)
writeOutput('Quitting...')
def fetch_and_store(key_name, base_url, translated_address, mirrored_url,
host, handler, shorturl):
"""Fetch and cache a page.
Args:
key_name: Hash to use to store the cached page.
base_url: The hostname of the page that's being mirrored.
translated_address: The URL of the mirrored page on this site.
mirrored_url: The URL of the original page. Hostname should match
the base_url.
Returns:
A new MirroredContent object, if the page was successfully retrieved.
None if any errors occurred or the content could not be retrieved.
"""
logging.debug(
"+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
logging.debug("Fetching '%s' , base_url is '%s' ,UA is %s ",
mirrored_url, base_url,
handler.request.headers["User-Agent"])
headers = {
# 'content-type': 'application/json',
'User-Agent':
handler.request.headers["User-Agent"] or
'Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1',
# 'Referer': 'http://'+base_url,
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
}
try:
if handler.request.method == 'GET':
response = requests.get(mirrored_url, headers=headers)
if handler.request.method == 'POST':
logging.debug('handler.request postitems : %s' %
handler.request.POST.items())
response = requests.post(mirrored_url,
headers=headers,
data=handler.request.POST.items())
except Exception:
logging.exception("Could not fetch URL")
return None
adjusted_headers = {}
for key, value in response.headers.iteritems():
adjusted_key = key.lower()
if adjusted_key not in IGNORE_HEADERS:
adjusted_headers[adjusted_key] = value
content = response.content
page_content_type = adjusted_headers.get("content-type", "")
logging.info('page_content_type is %s' % page_content_type)
for content_type in TRANSFORMED_CONTENT_TYPES:
# startswith() because there could be a 'charset=UTF-8' in the header.
if page_content_type.startswith(content_type):
content = transform_content.TransformContent(
base_url, mirrored_url, content, shorturl)
if page_content_type.startswith(
"text/html"): #监听所有的请求，替换ajax地址
content = content.replace('document.domain="qq.com";',
'void(0);')
#微信的烂代码
content = content.replace(
'</body>',
'<script type="text/javascript" src="http://pingjs.qq.com/h5/stats.js" name="MTAH5" sid="500324497" cid="500331564" opts="{"senseHash":false}" ></script></body>'
)
#微信的烂代码
content = content.replace(
'location.href.indexOf("safe=0") == -1 ',
'false') #微信的蓝代码
# content = content.replace('"2",','"3",') #eqx烂代码
content = content.replace(
"<head>", """<head>
<meta name="referrer" content="never">
<script>
function do_poster_script_onload(el){
console.log('script el:',el);
if (el.outerHTML.indexOf('http://')<0 && el.outerHTML.indexOf('https://')<0 && el.src.indexOf(base_url)<0){
var path = el.src.replace('http://""" +
host + """','');//
el.src = '/""" + shorturl + """/""" +
base_url + """'+ path;
//el.onloadstart = null;
}
}
(function() {
var base_url = '""" + base_url + """';
var proxied = window.XMLHttpRequest.prototype.open;
window.XMLHttpRequest.prototype.open = function() {
//console.log( arguments );
if (arguments[1].indexOf('http://')<0 && arguments[1].indexOf('https://')<0) {arguments[1]='http://'+base_url+arguments[1]}
arguments[1] = arguments[1].replace('http://','/"""
+ shorturl + """/')
//console.log( 'arguments xhr:',arguments );
return proxied.apply(this, [].slice.call(arguments));
};
var proxied_append = HTMLElement.prototype.appendChild;
HTMLElement.prototype.appendChild = function() {
//console.log( 'appendChild:', arguments );
for (var i in arguments){
var el = arguments[i];
//debugger;
if (el.tagName==='SCRIPT'){
//debugger;
if (el.outerHTML.indexOf('http://')<0 && el.outerHTML.indexOf('https://')<0 && el.src.indexOf(base_url)<0){
var path = el.src.replace('http://"""
+ host + """','');//
console.log('path:',path);
if (path==='') {
el.onbeforeonload = function(){
//console.log('onbeforeonloadNew:',el);
}
el.onerror = function(e){
//console.log('onerror',e)
}
}else{
el.src = '/""" + shorturl +
"""/""" + base_url + """'+ path;
}
}
}
}
//if (arguments[1].indexOf('http://')<0) {arguments[1]='http://'+arguments[1]}
//arguments[1] = arguments[1].replace('http://','/')
//console.log( 'arguments append:',arguments );
return proxied_append.apply(this, [].slice.call(arguments));
};
/*
var proxied_onload = HTMLElement.prototype.onload;
HTMLElement.prototype.onload = function(){
var result = proxied_onload.apply(this, [].slice.call(arguments));
//do_poster_script_onload(this);
//console.log('poster onload');
return result;
}*/
var D = window.localStorage.getItem;
window.localStorage.getItem = function() {
var result = D.call(window.localStorage, [].slice.call(arguments));
//console.log('getItem',result);
if (result)
result = result.replace('document.domain="qq.com";','void(0);');//#微信的烂代码
return result;
};
})();
</script>
""")
break
new_content = MirroredContent(base_url=base_url,
original_address=mirrored_url,
translated_address=translated_address,
status=response.status_code,
headers=adjusted_headers,
data=content)
# Do not memcache content over 1MB
if len(content) < MAX_CONTENT_SIZE:
if not memcache.set(key_name, new_content):
logging.error(
'memcache.add failed: key_name = "%s", '
'original_url = "%s"', key_name, mirrored_url)
else:
logging.warning("Content is over %s ; not memcached" %
MAX_CONTENT_SIZE)
return new_content
def post(self):
#checkauth(self)
url_mime = 'image/'
avatar = self.request.get('avatar')
mail = self.request.get('mail')
dispname = self.request.get('dispname')
website = self.request.get('website')
if avatar:
# New CheckLogin code built in tarsusaRequestHandler
if self.chk_login():
CurrentUser = self.get_user_db()
#Create the Avatar Image Thumbnails.
avatar_image = images.Image(avatar)
avatar_image.resize(width=64, height=64)
avatar_image.im_feeling_lucky()
avatar_image_thumbnail = avatar_image.execute_transforms(
output_encoding=images.JPEG)
CurrentUser.avatar = db.Blob(avatar_image_thumbnail)
CurrentUser.put()
if not memcache.set('img_useravatar' + str(CurrentUser.key().id()),
db.Blob(avatar_image_thumbnail), 7200):
logging.error(
"Memcache set failed: When uploading avatar_image")
self.redirect("/user/" + str(CurrentUser.key().id()) + "/setting")
elif self.request.get('apikey_gene') == 'apikey':
#Regenerate the APIKEY.
if self.chk_login():
CurrentUser = self.get_user_db()
#From Plog.
#Generate a random string for using as api password, api user is user's full email
from random import sample
import hashlib
s = 'abcdefghijklmnopqrstuvwxyz1234567890'
fusion_p = ''.join(sample(s, 8))
fusion_uky = str(CurrentUser.key().id())
fusion_uma = str(CurrentUser.mail)
fusion_cn = "CheckNerds Approching to version California"
fusion_uni = fusion_p + fusion_uky + fusion_uma + fusion_cn
CurrentUser.apikey = hashlib.sha1(fusion_uni).hexdigest()[:8]
CurrentUser.put()
self.redirect("/user/" + str(CurrentUser.key().id()) + "/setting")
else:
# New CheckLogin code built in tarsusaRequestHandler
if self.chk_login():
CurrentUser = self.get_user_db()
CurrentUser.mail = mail
CurrentUser.dispname = dispname
CurrentUser.user.nickname = dispname
try:
CurrentUser.website = website
except:
CurrentUser.website = "http://" + website
CurrentUser.put()
self.redirect("/user/" + str(CurrentUser.key().id()) + "/setting")
if self.request.get('fetch') == 'yes':
try:
fc = urlfetch.fetch(url_mime)
if fc.status_code == 200:
avatar = fc.content
if 'Content-Type' in fc.headers:
url_mime = fc.headers['Content-Type']
# New CheckLogin code built in tarsusaRequestHandler
if self.chk_login():
CurrentUser = self.get_user_db()
self.write('noneok')
else:
#sendmsg(self, 'no content-type sent from remote server')
self.write('ac')
else:
#sendmsg(self, 'cannot fetch avatar: http status ' + str(fc.status_code))
self.write('avcx')
except:
#sendmsg(self, 'cannot fetch avatar')
self.write('avcx')
else:
try:
avatar = Avatar(url_mime=url_mime)
avatar.put()
if not memcache.set(
'img_useravatar' + str(CurrentUser.key().id()),
db.Blob(avatar_image), 7200):
logging.error(
"Memcache set failed: When uploading(2) avatar_image"
)
except:
pass
self.redirect("/user/" + str(CurrentUser.key().id()) +
"/setting")
def post(self):
#checkauth(self)
url_mime = 'image/'
avatar = self.request.get('avatar')
mail = self.request.get('mail')
dispname = self.request.get('dispname')
website = self.request.get('website')
if avatar:
# New CheckLogin code built in tarsusaRequestHandler
if self.chk_login():
CurrentUser = self.get_user_db()
#Create the Avatar Image Thumbnails.
avatar_image = images.Image(avatar)
avatar_image.resize(width=64,height=64)
avatar_image.im_feeling_lucky()
avatar_image_thumbnail = avatar_image.execute_transforms(output_encoding=images.JPEG)
CurrentUser.avatar=db.Blob(avatar_image_thumbnail)
CurrentUser.put()
if not memcache.set('img_useravatar' + str(CurrentUser.key().id()), db.Blob(avatar_image_thumbnail), 7200):
logging.error("Memcache set failed: When uploading avatar_image")
self.redirect("/user/" + str(CurrentUser.key().id()) + "/setting")
elif self.request.get('apikey_gene') == 'apikey':
#Regenerate the APIKEY.
if self.chk_login():
CurrentUser = self.get_user_db()
#From Plog.
#Generate a random string for using as api password, api user is user's full email
from random import sample
import hashlib
s = 'abcdefghijklmnopqrstuvwxyz1234567890'
fusion_p = ''.join(sample(s, 8))
fusion_uky = str(CurrentUser.key().id())
fusion_uma = str(CurrentUser.mail)
fusion_cn = "CheckNerds Approching to version California"
fusion_uni = fusion_p + fusion_uky + fusion_uma + fusion_cn
CurrentUser.apikey = hashlib.sha1(fusion_uni).hexdigest()[:8]
CurrentUser.put()
self.redirect("/user/" + str(CurrentUser.key().id()) + "/setting")
else:
# New CheckLogin code built in tarsusaRequestHandler
if self.chk_login():
CurrentUser = self.get_user_db()
CurrentUser.mail = mail
CurrentUser.dispname = dispname
CurrentUser.user.nickname = dispname
try:
CurrentUser.website = website
except:
CurrentUser.website = "http://" + website
CurrentUser.put()
self.redirect("/user/" + str(CurrentUser.key().id()) + "/setting")
if self.request.get('fetch') == 'yes':
try:
fc = urlfetch.fetch(url_mime)
if fc.status_code == 200:
avatar = fc.content
if 'Content-Type' in fc.headers:
url_mime = fc.headers['Content-Type']
# New CheckLogin code built in tarsusaRequestHandler
if self.chk_login():
CurrentUser = self.get_user_db()
self.write('noneok')
else:
#sendmsg(self, 'no content-type sent from remote server')
self.write('ac')
else:
#sendmsg(self, 'cannot fetch avatar: http status ' + str(fc.status_code))
self.write('avcx')
except:
#sendmsg(self, 'cannot fetch avatar')
self.write('avcx')
else:
try:
avatar = Avatar(url_mime=url_mime)
avatar.put()
if not memcache.set('img_useravatar' + str(CurrentUser.key().id()), db.Blob(avatar_image), 7200):
logging.error("Memcache set failed: When uploading(2) avatar_image")
except:
pass
self.redirect("/user/" + str(CurrentUser.key().id()) + "/setting")
def _post_put(self, **kwargs):
import memcache
memcache.delete('%s%s_avatarcachebase64' %
(memcache.version, self.username))
memcache.set('%s%s' % (memcache.version, self.key().name()),
memcache.serialize_instances(self), 300)
# generate a new session id
chars = "".join((string.letters, string.digits))
while True:
session_id = "".join([random.choice(chars) for x in xrange(0, 25)])
if not memcache.get("session_" + session_id):
# session id is available
break
# store the initial session data
data = { "__is_authenticated__": False }
session = MemcacheSession(session_id, memcache)
memcache.set("session_" + session_id, pickle.dumps(data))
session.merge(data)
return session
# ------------------------------------------------------------------------------------------------------------------
def merge (self, data):
"""
Merge one or more name/value pairs into the session.
@param data (dict) The dictionary of name/value pairs.
"""
self._data.update(data)
def _post_put(self, **kwargs):
import memcache
memcache.set('%s%s' % (memcache.version, self.key().name()), memcache.serialize_instances(self), 300)
def logout(): client.set(instagram_username, 0)
def set_user_cache(key, user):
last_login = datetime.utcnow()
memcache.set(key, (user, last_login))