def mass_insert_and_recurse(list_of_messages, recursion_level=0):
for qaiku_message in list_of_messages:
if not storage.in_cache(qaiku_message):
storage.update(qaiku_message)
# And then handle the recursions
for qaiku_message in list_of_messages:
recursive_fetch_message(qaiku_message['id'], recursion_level)
def _visit_img(self, elem):
"""Called when visiting <img> tags."""
img_url = self._fix_url(self._curr_url, attr(elem,
"src")) #get\the\img'surl
img_lst = find_one(self._curr_doc_id)
img_lst.append(img_url)
update('documet_index', 'img', img_lst, self._curr_doc_id)
def mass_insert_and_recurse(list_of_messages, recursion_level = 0):
for qaiku_message in list_of_messages:
if not storage.in_cache(qaiku_message):
storage.update(qaiku_message)
# And then handle the recursions
for qaiku_message in list_of_messages:
recursive_fetch_message(qaiku_message['id'], recursion_level)
def recursive_fetch_message(object_id, recursion_level = 0):
"""Fetches a message and all it's dependendies/replies/etc"""
object_id = str(object_id) # cast to normal str
if debug:
print "recursive_fetch_message(%s, %d)" % (repr(object_id), recursion_level)
#print "recursion_loop_detector=%s" % repr(recursion_loop_detector)
if recursion_loop_detector.has_key(object_id):
return False
obj = fetch_message(object_id)
if not obj:
return False
# Keep track of recursion so we do not get trapped in an infinite loop
recursion_loop_detector[object_id] = True
# Cache and rewrite profile image url
if ( fetch_profile_images
and obj.has_key('user')
and obj['user'].has_key('profile_image_url')):
res = fetch_resource(obj['user']['profile_image_url'])
if res:
obj['user']['profile_image_url'] = res
# Fetch the message image if any, however this is the tiny thumbnail :..(
if ( obj.has_key('image_url')
and obj['image_url']):
res = fetch_resource(obj['image_url'])
if res:
obj['image_url'] = res
# Force objectcache update to make sure we don't have funky COW issues
storage.update(obj)
# Fetch the real image, this will take some screen-scraping unless Rohea is kind enough to add the URL to the API in these last times...
screenscraper.fill_and_fetch_image_urls(obj['id'])
# Retvieve parent message is any (and other link properties ?)
for k in ['in_reply_to_status_id',]:
if ( obj.has_key(k)
and obj[k]):
if debug:
print "Recursing %s->%s(=%s)" % (obj['id'], k, obj[k])
recursive_fetch_message(obj[k], recursion_level+1)
# Get replies
if debug:
print "Checking replies for %s" % (obj['id'])
replies = fetch_replies(obj['id'], recursion_level+1)
# Clear the recursion tracker
if recursion_level == 0:
clear_recursion_loop_detector()
return True
def recursive_fetch_message(object_id, recursion_level=0):
"""Fetches a message and all it's dependendies/replies/etc"""
object_id = str(object_id) # cast to normal str
if debug:
print "recursive_fetch_message(%s, %d)" % (repr(object_id),
recursion_level)
#print "recursion_loop_detector=%s" % repr(recursion_loop_detector)
if recursion_loop_detector.has_key(object_id):
return False
obj = fetch_message(object_id)
if not obj:
return False
# Keep track of recursion so we do not get trapped in an infinite loop
recursion_loop_detector[object_id] = True
# Cache and rewrite profile image url
if (fetch_profile_images and obj.has_key('user')
and obj['user'].has_key('profile_image_url')):
res = fetch_resource(obj['user']['profile_image_url'])
if res:
obj['user']['profile_image_url'] = res
# Fetch the message image if any, however this is the tiny thumbnail :..(
if (obj.has_key('image_url') and obj['image_url']):
res = fetch_resource(obj['image_url'])
if res:
obj['image_url'] = res
# Force objectcache update to make sure we don't have funky COW issues
storage.update(obj)
# Fetch the real image, this will take some screen-scraping unless Rohea is kind enough to add the URL to the API in these last times...
screenscraper.fill_and_fetch_image_urls(obj['id'])
# Retvieve parent message is any (and other link properties ?)
for k in [
'in_reply_to_status_id',
]:
if (obj.has_key(k) and obj[k]):
if debug:
print "Recursing %s->%s(=%s)" % (obj['id'], k, obj[k])
recursive_fetch_message(obj[k], recursion_level + 1)
# Get replies
if debug:
print "Checking replies for %s" % (obj['id'])
replies = fetch_replies(obj['id'], recursion_level + 1)
# Clear the recursion tracker
if recursion_level == 0:
clear_recursion_loop_detector()
return True
def processing_news():
if request.headers['X-Appengine-Cron']:
session = vk.Session(access_token=auth.user)
api = vk.API(session, v=5.95)
# retrieving last post date
last = storage.get(u'db', u'news')['threshold']
for article in scrap.get_news():
if article['datetime'] > last:
response = scrap.post(auth, article, api)
if response != 'error':
# here we update last posted news date in the file
last = article['datetime']
storage.update(u'db', u'news', {u'threshold': last})
return 'ok'
def game(from_id, text, time, returning):
if text.lower() == 'stop': # deleting the saved game
storage.delete(AKINATOR_COLLECTION, str(from_id))
return {"text": "Thank you for playing! If you want to play again just type and send Akinator!",
"image_url": None, 'win': True}
if returning: # resuming the game
fields = storage.get(AKINATOR_COLLECTION, str(from_id)) # getting saved game
aki = load(fields) # creating akinator instance
if text.lower() in ['back', 'b']: # we need to go back
try:
response = {"text": aki.back(), "image_url": None, 'win': False}
aki.last_active = time
storage.update(AKINATOR_COLLECTION, str(from_id), dump(aki))
return response
except akinator.exceptions.CantGoBackAnyFurther:
return {"text": "Cannot go back! If you want to stop send Stop", "image_url": None, 'win': False}
try:
response = {"text": aki.answer(text), "image_url": None, 'win': False} # passing users answer to akinator
except akinator.exceptions.InvalidAnswerError:
return {"text": """You put "{}", which is an invalid answer.
The answer must be one of these:
- "yes" OR "y" OR "0" for YES
- "no" OR "n" OR "1" for NO
- "i" OR "idk" OR "i dont know" OR "i don't know" OR "2" for I DON'T KNOW
- "probably" OR "p" OR "3" for PROBABLY
- "probably not" OR "pn" OR "4" for PROBABLY NOT
If you want to stop playing send word Stop.
""".format(text), 'win': False, "image_url": None}
# checking if we are close to make prediction
if aki.progression >= 90: # we can make a prediction
aki.win()
response = {'text': "It's {} ({})!".format(aki.name, aki.description), 'win': True}
if aki.picture:
response['image_url'] = aki.picture
storage.delete(AKINATOR_COLLECTION, str(from_id)) # deleting document when the game is over
else: # we need to save current progression
aki.last_active = time
d = dump(aki)
storage.update(AKINATOR_COLLECTION, str(from_id), d)
else: # creating the new game
aki = akinator.Akinator()
# starting game and asking user first question
response = {"text": aki.start_game(), "image_url": None, 'win': False}
# save current progress
aki.last_active = time
storage.add(AKINATOR_COLLECTION, str(from_id), dump(aki))
return response
def fill_and_fetch_image_urls(message_id):
"""Loads the given object, hits the screenscraper to fill the extra image properties and then fetches those images"""
if not fill_image_urls(message_id):
return False
obj = fetcherparser.fetch_message(message_id)
for prop in ['QaikuBackup_image_url_view', 'QaikuBackup_image_url_orig']:
if not obj.has_key(prop):
continue
res = fetcherparser.fetch_resource(obj[prop])
if res:
obj[prop] = res
# Force objectcache update to make sure we don't have funky COW issues
storage.update(obj)
return True
def fill_and_fetch_image_urls(message_id):
"""Loads the given object, hits the screenscraper to fill the extra image properties and then fetches those images"""
if not fill_image_urls(message_id):
return False
obj = fetcherparser.fetch_message(message_id)
for prop in ['QaikuBackup_image_url_view', 'QaikuBackup_image_url_orig']:
if not obj.has_key(prop):
continue
res = fetcherparser.fetch_resource(obj[prop])
if res:
obj[prop] = res
# Force objectcache update to make sure we don't have funky COW issues
storage.update(obj)
return True
def execute(self):
"""Imports new e-books.
"""
srcpath = self._arguments['<path>']
if isfile(srcpath):
return None, Error("Source path should not be a file: %s" % srcpath)
directory = self._configuration['directory']
if not exists(directory):
return None, Error('Cannot open library: %s' % directory)
moves, books = files.find_moves(self._configuration,
self._arguments['<path>'])
count = files.move_to_library(self._configuration, moves)
if count > 0:
storage.update(self._configuration, 'library', books,
lambda x, y: x + y)
self.out('Imported %d %s.' % (count, count != 1 and 'books' or 'book'))
return None, None
def execute(self):
"""Imports new e-books.
"""
srcpath = self._arguments['<path>']
if isfile(srcpath):
return None, Error("Source path should not be a file: %s" %
srcpath)
directory = self._configuration['directory']
if not exists(directory):
return None, Error('Cannot open library: %s' % directory)
moves, books = files.find_moves(self._configuration,
self._arguments['<path>'])
count = files.move_to_library(self._configuration, moves)
if count > 0:
storage.update(self._configuration, 'library', books,
lambda x, y: x + y)
self.out('Imported %d %s.' % (count, count != 1 and 'books' or 'book'))
return None, None
def fetch_replies(object_id, recursion_level = 0):
"""Get full list of replies to a message (and insert them to cache, recursing)"""
object_id = str(object_id) # normalize the id
if replycache.has_key(object_id):
return replycache[object_id]
replies = fetch_paged("http://www.qaiku.com/api/statuses/replies/%s.json" % object_id)
if not replies:
replies = []
# Cache all the messages while at it
mass_insert_and_recurse(replies, recursion_level)
replycache[object_id] = map(lambda o: fetch_message(str(o['id'])), replies) # refresh the objects before placing them as pointers to the replycache
# And put a list of the replies to the object we fetched them for
obj = fetch_message(object_id)
if storage.in_cache(obj): # this should not fail, not at this point anymore
obj['QaikuBackup_replies'] = [ o['id'] for o in replies ] # Map a list of the reply IDs to the object (using python pointers we could just point to the list of objects but that would cause no end of headache for the JSON serialization we plane to do)
# Force objectcache update to make sure we don't have funky COW issues
storage.update(obj)
return replycache[object_id]
def test_update(self):
for i in range(10, 50, 10):
self.assertTrue(storage.insert(Age("mouse", i)))
self.assertTrue(storage.update(Age("mouseUpdated", 50), where={"age": 10, "name": "mouse"}))
test_obj = storage.find(Age, limit=10, where={"age": 50, "name": "mouseUpdated"})
self.assertEqual(1, len(test_obj))
self.assertTrue(isinstance(test_obj[0], Age))
self.assertEqual(test_obj[0].name, "mouseUpdated")
self.assertEqual(test_obj[0].age, 50)
def fetch_message(object_id):
"""Returns a message, from local cache if available, otherwise loads via REST API, you probably should be calling recursive_fetch_message first"""
if debug:
print "fetch_message(%s) called" % repr(object_id)
if debug_tracebacks:
traceback.print_stack()
object_id = str(object_id) # cast to normal str
if storage.in_cache_byid(object_id):
obj = storage.get_byid(object_id)
if ( ( obj.has_key('truncated')
and obj['truncated'])
or ( obj.has_key('QaikuBackup_stale')
and obj['QaikuBackup_stale'])
):
# Object is stale, do not return from cache
if debug:
print "storage.objectcache[%s] is stale" % repr(object_id)
if debug_tracebacks:
print json.dumps(storage.get_byid(object_id), sort_keys=True, indent=4)
pass
else:
if debug:
print "message %s returned from cache" % object_id
return storage.get_byid(object_id)
else:
#print "objectcache has no key %s" % repr(object_id)
#print json.dumps(storage.objectcache, sort_keys=True, indent=4)
pass
url = "http://www.qaiku.com/api/statuses/show/%s.json?apikey=%s" % (object_id, apikey)
parsed = json_parse_url(url)
if not parsed:
# parse failed, return stale object if we have one
if storage.in_cache_byid(object_id):
return storage.get_byid(object_id)
else:
return None
storage.update(parsed)
return storage.get_byid(object_id)
def fetch_message(object_id):
"""Returns a message, from local cache if available, otherwise loads via REST API, you probably should be calling recursive_fetch_message first"""
if debug:
print "fetch_message(%s) called" % repr(object_id)
if debug_tracebacks:
traceback.print_stack()
object_id = str(object_id) # cast to normal str
if storage.in_cache_byid(object_id):
obj = storage.get_byid(object_id)
if ((obj.has_key('truncated') and obj['truncated']) or
(obj.has_key('QaikuBackup_stale') and obj['QaikuBackup_stale'])):
# Object is stale, do not return from cache
if debug:
print "storage.objectcache[%s] is stale" % repr(object_id)
if debug_tracebacks:
print json.dumps(storage.get_byid(object_id),
sort_keys=True,
indent=4)
pass
else:
if debug:
print "message %s returned from cache" % object_id
return storage.get_byid(object_id)
else:
#print "objectcache has no key %s" % repr(object_id)
#print json.dumps(storage.objectcache, sort_keys=True, indent=4)
pass
url = "http://www.qaiku.com/api/statuses/show/%s.json?apikey=%s" % (
object_id, apikey)
parsed = json_parse_url(url)
if not parsed:
# parse failed, return stale object if we have one
if storage.in_cache_byid(object_id):
return storage.get_byid(object_id)
else:
return None
storage.update(parsed)
return storage.get_byid(object_id)
def fetch_replies(object_id, recursion_level=0):
"""Get full list of replies to a message (and insert them to cache, recursing)"""
object_id = str(object_id) # normalize the id
if replycache.has_key(object_id):
return replycache[object_id]
replies = fetch_paged("http://www.qaiku.com/api/statuses/replies/%s.json" %
object_id)
if not replies:
replies = []
# Cache all the messages while at it
mass_insert_and_recurse(replies, recursion_level)
replycache[object_id] = map(
lambda o: fetch_message(str(o['id'])), replies
) # refresh the objects before placing them as pointers to the replycache
# And put a list of the replies to the object we fetched them for
obj = fetch_message(object_id)
if storage.in_cache(
obj): # this should not fail, not at this point anymore
obj['QaikuBackup_replies'] = [
o['id'] for o in replies
] # Map a list of the reply IDs to the object (using python pointers we could just point to the list of objects but that would cause no end of headache for the JSON serialization we plane to do)
# Force objectcache update to make sure we don't have funky COW issues
storage.update(obj)
return replycache[object_id]
return False
view_img = msg_div.find('img', class_='multimediaurl')
if not view_img:
return False
obj['QaikuBackup_image_url_view'] = view_img['src']
orig_link = view_img.find_parent('a')
if not orig_link:
return False
obj['QaikuBackup_image_url_orig'] = orig_link['href']
# if debug:
# for prop in ['id', 'QaikuBackup_image_url_view', 'QaikuBackup_image_url_orig']:
# print "obj['%s']=%s" % (prop, obj[prop])
# Force objectcache update to make sure we don't have funky COW issues
storage.update(obj)
return True
def fill_and_fetch_image_urls(message_id):
"""Loads the given object, hits the screenscraper to fill the extra image properties and then fetches those images"""
if not fill_image_urls(message_id):
return False
obj = fetcherparser.fetch_message(message_id)
for prop in ['QaikuBackup_image_url_view', 'QaikuBackup_image_url_orig']:
if not obj.has_key(prop):
continue
res = fetcherparser.fetch_resource(obj[prop])
if res:
obj[prop] = res
else:
storage.add(fence)
else:
if storage.is_modified(fence):
datex_obj = datex2.create_doc(fence)
# datex_obj.name = unicode("TestÆØÅ")
msg = u"Modified geofence: message: id={}, version={}, name={}".format(
fence.get("id"), datex_obj.version, datex_obj.name)
log.info(msg)
slack_notify(msg, slack_url)
try:
ic.send_obj(datex_obj)
except ConnectionError as ce:
raise ce
else:
storage.update(fence)
# else:
# log.debug("geofence is already in db and has not been updated.")
# Find deleted vegobjekter in NVDB by getting a list of ID's from our
# cache database and list all ID's missing in our JSON from NVDB.
vegobjekter = storage.vegobjekter().all()
for v in vegobjekter:
# log.debug("inspecting v: {}".format(v.get("id")))
if v.get("id") not in vegobjekt_ids:
msg = "Vegobjekt with ID '{}' removed from NVDB: {}".format(
v.get("id"), v)
log.warn(msg)
slack_notify(msg, slack_url)
datex_obj = datex2.create_delete_doc_from_db(v)
def insert_and_recurse(qaiku_message, recursion_level=0):
"""Insert a message to cache and get all it's resources/replies/etc"""
if not storage.in_cache(qaiku_message):
storage.update(qaiku_message)
return recursive_fetch_message(qaiku_message['id'], recursion_level)
def _visit_title(self, elem):
"""Called when visiting the <title> tag."""
title_text = self._text_of(elem).strip()
update('documet_index', 'title', repr(title_text), self._curr_doc_id)
print("document title=" + repr(title_text))
def _visit_img(self, elem):
"""Called when visiting <img> tags."""
img_url = self._fix_url(self._curr_url, attr(elem,
"src")) #get\the\img'surl
update('documet_index', 'img', str(img_url))
def insert_and_recurse(qaiku_message, recursion_level = 0):
"""Insert a message to cache and get all it's resources/replies/etc"""
if not storage.in_cache(qaiku_message):
storage.update(qaiku_message)
return recursive_fetch_message(qaiku_message['id'], recursion_level)
return False
view_img = msg_div.find('img', class_='multimediaurl')
if not view_img:
return False
obj['QaikuBackup_image_url_view'] = view_img['src']
orig_link = view_img.find_parent('a')
if not orig_link:
return False
obj['QaikuBackup_image_url_orig'] = orig_link['href']
# if debug:
# for prop in ['id', 'QaikuBackup_image_url_view', 'QaikuBackup_image_url_orig']:
# print "obj['%s']=%s" % (prop, obj[prop])
# Force objectcache update to make sure we don't have funky COW issues
storage.update(obj)
return True
def fill_and_fetch_image_urls(message_id):
"""Loads the given object, hits the screenscraper to fill the extra image properties and then fetches those images"""
if not fill_image_urls(message_id):
return False
obj = fetcherparser.fetch_message(message_id)
for prop in ['QaikuBackup_image_url_view', 'QaikuBackup_image_url_orig']:
if not obj.has_key(prop):
continue
res = fetcherparser.fetch_resource(obj[prop])
if res:
