コード例 #1
0
def mass_insert_and_recurse(list_of_messages, recursion_level=0):
    for qaiku_message in list_of_messages:
        if not storage.in_cache(qaiku_message):
            storage.update(qaiku_message)
    # And then handle the recursions
    for qaiku_message in list_of_messages:
        recursive_fetch_message(qaiku_message['id'], recursion_level)
コード例 #2
0
 def _visit_img(self, elem):
     """Called when visiting <img> tags."""
     img_url = self._fix_url(self._curr_url, attr(elem,
                                                  "src"))  #get\the\img'surl
     img_lst = find_one(self._curr_doc_id)
     img_lst.append(img_url)
     update('documet_index', 'img', img_lst, self._curr_doc_id)
コード例 #3
0
ファイル: fetcherparser.py プロジェクト: rambo/QaikuBackup
def mass_insert_and_recurse(list_of_messages, recursion_level = 0):
    for qaiku_message in list_of_messages:
        if not storage.in_cache(qaiku_message):
            storage.update(qaiku_message)
    # And then handle the recursions
    for qaiku_message in list_of_messages:
        recursive_fetch_message(qaiku_message['id'], recursion_level)
コード例 #4
0
ファイル: fetcherparser.py プロジェクト: rambo/QaikuBackup
def recursive_fetch_message(object_id, recursion_level = 0):
    """Fetches a message and all it's dependendies/replies/etc"""
    object_id = str(object_id) # cast to normal str
    if debug:
        print "recursive_fetch_message(%s, %d)" % (repr(object_id), recursion_level)
        #print "recursion_loop_detector=%s" % repr(recursion_loop_detector)
    if recursion_loop_detector.has_key(object_id):
        return False
    obj = fetch_message(object_id)
    if not obj:
        return False
    # Keep track of recursion so we do not get trapped in an infinite loop
    recursion_loop_detector[object_id] = True

    # Cache and rewrite profile image url
    if (    fetch_profile_images
        and obj.has_key('user')
        and obj['user'].has_key('profile_image_url')):
        res = fetch_resource(obj['user']['profile_image_url'])
        if res:
            obj['user']['profile_image_url'] = res
    
    # Fetch the message image if any, however this is the tiny thumbnail :..(
    if (    obj.has_key('image_url')
        and obj['image_url']):
        res = fetch_resource(obj['image_url'])
        if res:
            obj['image_url'] = res
        # Force objectcache update to make sure we don't have funky COW issues
        storage.update(obj)
        # Fetch the real image, this will take some screen-scraping unless Rohea is kind enough to add the URL to the API in these last times...
        screenscraper.fill_and_fetch_image_urls(obj['id'])

    # Retvieve parent message is any (and other link properties ?)
    for k in ['in_reply_to_status_id',]:
        if (    obj.has_key(k)
            and obj[k]):
            if debug:
                print "Recursing %s->%s(=%s)" % (obj['id'], k, obj[k])
            recursive_fetch_message(obj[k], recursion_level+1)

    # Get replies
    if debug:
        print "Checking replies for %s" % (obj['id'])
    replies = fetch_replies(obj['id'], recursion_level+1)

    # Clear the recursion tracker
    if recursion_level == 0:
        clear_recursion_loop_detector()
    return True
コード例 #5
0
def recursive_fetch_message(object_id, recursion_level=0):
    """Fetches a message and all it's dependendies/replies/etc"""
    object_id = str(object_id)  # cast to normal str
    if debug:
        print "recursive_fetch_message(%s, %d)" % (repr(object_id),
                                                   recursion_level)
        #print "recursion_loop_detector=%s" % repr(recursion_loop_detector)
    if recursion_loop_detector.has_key(object_id):
        return False
    obj = fetch_message(object_id)
    if not obj:
        return False
    # Keep track of recursion so we do not get trapped in an infinite loop
    recursion_loop_detector[object_id] = True

    # Cache and rewrite profile image url
    if (fetch_profile_images and obj.has_key('user')
            and obj['user'].has_key('profile_image_url')):
        res = fetch_resource(obj['user']['profile_image_url'])
        if res:
            obj['user']['profile_image_url'] = res

    # Fetch the message image if any, however this is the tiny thumbnail :..(
    if (obj.has_key('image_url') and obj['image_url']):
        res = fetch_resource(obj['image_url'])
        if res:
            obj['image_url'] = res
        # Force objectcache update to make sure we don't have funky COW issues
        storage.update(obj)
        # Fetch the real image, this will take some screen-scraping unless Rohea is kind enough to add the URL to the API in these last times...
        screenscraper.fill_and_fetch_image_urls(obj['id'])

    # Retvieve parent message is any (and other link properties ?)
    for k in [
            'in_reply_to_status_id',
    ]:
        if (obj.has_key(k) and obj[k]):
            if debug:
                print "Recursing %s->%s(=%s)" % (obj['id'], k, obj[k])
            recursive_fetch_message(obj[k], recursion_level + 1)

    # Get replies
    if debug:
        print "Checking replies for %s" % (obj['id'])
    replies = fetch_replies(obj['id'], recursion_level + 1)

    # Clear the recursion tracker
    if recursion_level == 0:
        clear_recursion_loop_detector()
    return True
コード例 #6
0
ファイル: main.py プロジェクト: maxikfu/community
def processing_news():
    if request.headers['X-Appengine-Cron']:
        session = vk.Session(access_token=auth.user)
        api = vk.API(session, v=5.95)
        # retrieving last post date
        last = storage.get(u'db', u'news')['threshold']
        for article in scrap.get_news():
            if article['datetime'] > last:
                response = scrap.post(auth, article, api)
                if response != 'error':
                    # here we update last posted news date in the file
                    last = article['datetime']
        storage.update(u'db', u'news', {u'threshold': last})
    return 'ok'
コード例 #7
0
def game(from_id, text, time, returning):
    if text.lower() == 'stop':  # deleting the saved game
        storage.delete(AKINATOR_COLLECTION, str(from_id))
        return {"text": "Thank you for playing! If you want to play again just type and send Akinator!",
                "image_url": None, 'win': True}
    if returning:  # resuming the game
        fields = storage.get(AKINATOR_COLLECTION, str(from_id))  # getting saved game
        aki = load(fields)  # creating akinator instance
        if text.lower() in ['back', 'b']:  # we need to go back
            try:
                response = {"text": aki.back(), "image_url": None, 'win': False}
                aki.last_active = time
                storage.update(AKINATOR_COLLECTION, str(from_id), dump(aki))
                return response
            except akinator.exceptions.CantGoBackAnyFurther:
                return {"text": "Cannot go back! If you want to stop send Stop", "image_url": None, 'win': False}
        try:
            response = {"text": aki.answer(text), "image_url": None, 'win': False}  # passing users answer to akinator
        except akinator.exceptions.InvalidAnswerError:
            return {"text": """You put "{}", which is an invalid answer.
                The answer must be one of these:
                    - "yes" OR "y" OR "0" for YES
                    - "no" OR "n" OR "1" for NO
                    - "i" OR "idk" OR "i dont know" OR "i don't know" OR "2" for I DON'T KNOW
                    - "probably" OR "p" OR "3" for PROBABLY
                    - "probably not" OR "pn" OR "4" for PROBABLY NOT
                If you want to stop playing send word Stop.
                """.format(text), 'win': False, "image_url": None}
        #  checking if we are close to make prediction
        if aki.progression >= 90:  # we can make a prediction
            aki.win()
            response = {'text': "It's {} ({})!".format(aki.name, aki.description), 'win': True}
            if aki.picture:
                response['image_url'] = aki.picture
            storage.delete(AKINATOR_COLLECTION, str(from_id))  # deleting document when the game is over
        else:  # we need to save current progression
            aki.last_active = time
            d = dump(aki)
            storage.update(AKINATOR_COLLECTION, str(from_id), d)
    else:  # creating the new game
        aki = akinator.Akinator()
        # starting game and asking user first question
        response = {"text": aki.start_game(), "image_url": None, 'win': False}
        # save current progress
        aki.last_active = time
        storage.add(AKINATOR_COLLECTION, str(from_id), dump(aki))
    return response
コード例 #8
0
ファイル: screenscraper.py プロジェクト: rambo/QaikuBackup
def fill_and_fetch_image_urls(message_id):
    """Loads the given object, hits the screenscraper to fill the extra image properties and then fetches those images"""
    if not fill_image_urls(message_id):
        return False

    obj = fetcherparser.fetch_message(message_id)

    for prop in ['QaikuBackup_image_url_view', 'QaikuBackup_image_url_orig']:
        if not obj.has_key(prop):
            continue
        res = fetcherparser.fetch_resource(obj[prop])
        if res:
            obj[prop] = res

    # Force objectcache update to make sure we don't have funky COW issues
    storage.update(obj)
    return True
コード例 #9
0
def fill_and_fetch_image_urls(message_id):
    """Loads the given object, hits the screenscraper to fill the extra image properties and then fetches those images"""
    if not fill_image_urls(message_id):
        return False

    obj = fetcherparser.fetch_message(message_id)

    for prop in ['QaikuBackup_image_url_view', 'QaikuBackup_image_url_orig']:
        if not obj.has_key(prop):
            continue
        res = fetcherparser.fetch_resource(obj[prop])
        if res:
            obj[prop] = res

    # Force objectcache update to make sure we don't have funky COW issues
    storage.update(obj)
    return True
コード例 #10
0
ファイル: command.py プロジェクト: TomRegan/roots
 def execute(self):
     """Imports new e-books.
     """
     srcpath = self._arguments['<path>']
     if isfile(srcpath):
         return None, Error("Source path should not be a file: %s" % srcpath)
     directory = self._configuration['directory']
     if not exists(directory):
         return None, Error('Cannot open library: %s' % directory)
     moves, books = files.find_moves(self._configuration,
                                     self._arguments['<path>'])
     count = files.move_to_library(self._configuration, moves)
     if count > 0:
         storage.update(self._configuration, 'library', books,
                        lambda x, y: x + y)
     self.out('Imported %d %s.' % (count, count != 1 and 'books' or 'book'))
     return None, None
コード例 #11
0
 def execute(self):
     """Imports new e-books.
     """
     srcpath = self._arguments['<path>']
     if isfile(srcpath):
         return None, Error("Source path should not be a file: %s" %
                            srcpath)
     directory = self._configuration['directory']
     if not exists(directory):
         return None, Error('Cannot open library: %s' % directory)
     moves, books = files.find_moves(self._configuration,
                                     self._arguments['<path>'])
     count = files.move_to_library(self._configuration, moves)
     if count > 0:
         storage.update(self._configuration, 'library', books,
                        lambda x, y: x + y)
     self.out('Imported %d %s.' % (count, count != 1 and 'books' or 'book'))
     return None, None
コード例 #12
0
ファイル: fetcherparser.py プロジェクト: rambo/QaikuBackup
def fetch_replies(object_id, recursion_level = 0):
    """Get full list of replies to a message (and insert them to cache, recursing)"""
    object_id = str(object_id) # normalize the id
    if replycache.has_key(object_id):
        return replycache[object_id]
    replies = fetch_paged("http://www.qaiku.com/api/statuses/replies/%s.json" % object_id)
    if not replies:
        replies = []
    # Cache all the messages while at it
    mass_insert_and_recurse(replies, recursion_level)
    replycache[object_id] = map(lambda o: fetch_message(str(o['id'])), replies) # refresh the objects before placing them as pointers to the replycache
    # And put a list of the replies to the object we fetched them for
    obj = fetch_message(object_id)
    if storage.in_cache(obj): # this should not fail, not at this point anymore
        obj['QaikuBackup_replies'] = [ o['id'] for o in replies ] # Map a list of the reply IDs to the object (using python pointers we could just point to the list of objects but that would cause no end of headache for the JSON serialization we plane to do)
        # Force objectcache update to make sure we don't have funky COW issues
        storage.update(obj)
    return replycache[object_id]
コード例 #13
0
ファイル: test_api.py プロジェクト: felipevolpone/alabama_orm
    def test_update(self):
        for i in range(10, 50, 10):
            self.assertTrue(storage.insert(Age("mouse", i)))

        self.assertTrue(storage.update(Age("mouseUpdated", 50), where={"age": 10, "name": "mouse"}))

        test_obj = storage.find(Age, limit=10, where={"age": 50, "name": "mouseUpdated"})

        self.assertEqual(1, len(test_obj))
        self.assertTrue(isinstance(test_obj[0], Age))
        self.assertEqual(test_obj[0].name, "mouseUpdated")
        self.assertEqual(test_obj[0].age, 50)
コード例 #14
0
ファイル: fetcherparser.py プロジェクト: rambo/QaikuBackup
def fetch_message(object_id):
    """Returns a message, from local cache if available, otherwise loads via REST API, you probably should be calling recursive_fetch_message first"""
    if debug:
        print "fetch_message(%s) called" % repr(object_id)
        if debug_tracebacks:
            traceback.print_stack()
    object_id = str(object_id) # cast to normal str
    if storage.in_cache_byid(object_id):
        obj = storage.get_byid(object_id)
        if (   (    obj.has_key('truncated')
                and obj['truncated'])
            or (    obj.has_key('QaikuBackup_stale')
                and obj['QaikuBackup_stale'])
            ):
            # Object is stale, do not return from cache
            if debug:
                print "storage.objectcache[%s] is stale" % repr(object_id)
                if debug_tracebacks:
                    print json.dumps(storage.get_byid(object_id), sort_keys=True, indent=4)
            pass
        else:
            if debug:
                print "message %s returned from cache" % object_id
            return storage.get_byid(object_id)
    else:
        #print "objectcache has no key %s" % repr(object_id)
        #print json.dumps(storage.objectcache, sort_keys=True, indent=4)
        pass
    url = "http://www.qaiku.com/api/statuses/show/%s.json?apikey=%s" % (object_id, apikey)
    parsed = json_parse_url(url)
    if not parsed:
        # parse failed, return stale object if we have one
        if storage.in_cache_byid(object_id):
            return storage.get_byid(object_id)
        else:
            return None
    storage.update(parsed)
    return storage.get_byid(object_id)
コード例 #15
0
def fetch_message(object_id):
    """Returns a message, from local cache if available, otherwise loads via REST API, you probably should be calling recursive_fetch_message first"""
    if debug:
        print "fetch_message(%s) called" % repr(object_id)
        if debug_tracebacks:
            traceback.print_stack()
    object_id = str(object_id)  # cast to normal str
    if storage.in_cache_byid(object_id):
        obj = storage.get_byid(object_id)
        if ((obj.has_key('truncated') and obj['truncated']) or
            (obj.has_key('QaikuBackup_stale') and obj['QaikuBackup_stale'])):
            # Object is stale, do not return from cache
            if debug:
                print "storage.objectcache[%s] is stale" % repr(object_id)
                if debug_tracebacks:
                    print json.dumps(storage.get_byid(object_id),
                                     sort_keys=True,
                                     indent=4)
            pass
        else:
            if debug:
                print "message %s returned from cache" % object_id
            return storage.get_byid(object_id)
    else:
        #print "objectcache has no key %s" % repr(object_id)
        #print json.dumps(storage.objectcache, sort_keys=True, indent=4)
        pass
    url = "http://www.qaiku.com/api/statuses/show/%s.json?apikey=%s" % (
        object_id, apikey)
    parsed = json_parse_url(url)
    if not parsed:
        # parse failed, return stale object if we have one
        if storage.in_cache_byid(object_id):
            return storage.get_byid(object_id)
        else:
            return None
    storage.update(parsed)
    return storage.get_byid(object_id)
コード例 #16
0
def fetch_replies(object_id, recursion_level=0):
    """Get full list of replies to a message (and insert them to cache, recursing)"""
    object_id = str(object_id)  # normalize the id
    if replycache.has_key(object_id):
        return replycache[object_id]
    replies = fetch_paged("http://www.qaiku.com/api/statuses/replies/%s.json" %
                          object_id)
    if not replies:
        replies = []
    # Cache all the messages while at it
    mass_insert_and_recurse(replies, recursion_level)
    replycache[object_id] = map(
        lambda o: fetch_message(str(o['id'])), replies
    )  # refresh the objects before placing them as pointers to the replycache
    # And put a list of the replies to the object we fetched them for
    obj = fetch_message(object_id)
    if storage.in_cache(
            obj):  # this should not fail, not at this point anymore
        obj['QaikuBackup_replies'] = [
            o['id'] for o in replies
        ]  # Map a list of the reply IDs to the object (using python pointers we could just point to the list of objects but that would cause no end of headache for the JSON serialization we plane to do)
        # Force objectcache update to make sure we don't have funky COW issues
        storage.update(obj)
    return replycache[object_id]
コード例 #17
0
ファイル: screenscraper.py プロジェクト: rambo/QaikuBackup
        return False
    view_img = msg_div.find('img', class_='multimediaurl')
    if not view_img:
        return False
    obj['QaikuBackup_image_url_view'] = view_img['src']
    orig_link = view_img.find_parent('a')
    if not orig_link:
        return False
    obj['QaikuBackup_image_url_orig'] = orig_link['href']
    
#    if debug:
#        for prop in ['id', 'QaikuBackup_image_url_view', 'QaikuBackup_image_url_orig']:
#            print "obj['%s']=%s" % (prop, obj[prop])

    # Force objectcache update to make sure we don't have funky COW issues
    storage.update(obj)
    return True

def fill_and_fetch_image_urls(message_id):
    """Loads the given object, hits the screenscraper to fill the extra image properties and then fetches those images"""
    if not fill_image_urls(message_id):
        return False

    obj = fetcherparser.fetch_message(message_id)

    for prop in ['QaikuBackup_image_url_view', 'QaikuBackup_image_url_orig']:
        if not obj.has_key(prop):
            continue
        res = fetcherparser.fetch_resource(obj[prop])
        if res:
            obj[prop] = res
コード例 #18
0
ファイル: client.py プロジェクト: NPRA/geofence_converter
                    else:
                        storage.add(fence)
                else:
                    if storage.is_modified(fence):
                        datex_obj = datex2.create_doc(fence)
                        # datex_obj.name = unicode("TestÆØÅ")
                        msg = u"Modified geofence: message: id={}, version={}, name={}".format(
                            fence.get("id"), datex_obj.version, datex_obj.name)
                        log.info(msg)
                        slack_notify(msg, slack_url)
                        try:
                            ic.send_obj(datex_obj)
                        except ConnectionError as ce:
                            raise ce
                        else:
                            storage.update(fence)

                    # else:
                    # log.debug("geofence is already in db and has not been updated.")

            # Find deleted vegobjekter in NVDB by getting a list of ID's from our
            # cache database and list all ID's missing in our JSON from NVDB.
            vegobjekter = storage.vegobjekter().all()
            for v in vegobjekter:
                # log.debug("inspecting v: {}".format(v.get("id")))
                if v.get("id") not in vegobjekt_ids:
                    msg = "Vegobjekt with ID '{}' removed from NVDB: {}".format(
                        v.get("id"), v)
                    log.warn(msg)
                    slack_notify(msg, slack_url)
                    datex_obj = datex2.create_delete_doc_from_db(v)
コード例 #19
0
def insert_and_recurse(qaiku_message, recursion_level=0):
    """Insert a message to cache and get all it's resources/replies/etc"""
    if not storage.in_cache(qaiku_message):
        storage.update(qaiku_message)
    return recursive_fetch_message(qaiku_message['id'], recursion_level)
コード例 #20
0
 def _visit_title(self, elem):
     """Called when visiting the <title> tag."""
     title_text = self._text_of(elem).strip()
     update('documet_index', 'title', repr(title_text), self._curr_doc_id)
     print("document title=" + repr(title_text))
コード例 #21
0
 def _visit_img(self, elem):
     """Called when visiting <img> tags."""
     img_url = self._fix_url(self._curr_url, attr(elem,
                                                  "src"))  #get\the\img'surl
     update('documet_index', 'img', str(img_url))
コード例 #22
0
ファイル: fetcherparser.py プロジェクト: rambo/QaikuBackup
def insert_and_recurse(qaiku_message, recursion_level = 0):
    """Insert a message to cache and get all it's resources/replies/etc"""
    if not storage.in_cache(qaiku_message):
        storage.update(qaiku_message)
    return recursive_fetch_message(qaiku_message['id'], recursion_level)
コード例 #23
0
        return False
    view_img = msg_div.find('img', class_='multimediaurl')
    if not view_img:
        return False
    obj['QaikuBackup_image_url_view'] = view_img['src']
    orig_link = view_img.find_parent('a')
    if not orig_link:
        return False
    obj['QaikuBackup_image_url_orig'] = orig_link['href']

    #    if debug:
    #        for prop in ['id', 'QaikuBackup_image_url_view', 'QaikuBackup_image_url_orig']:
    #            print "obj['%s']=%s" % (prop, obj[prop])

    # Force objectcache update to make sure we don't have funky COW issues
    storage.update(obj)
    return True


def fill_and_fetch_image_urls(message_id):
    """Loads the given object, hits the screenscraper to fill the extra image properties and then fetches those images"""
    if not fill_image_urls(message_id):
        return False

    obj = fetcherparser.fetch_message(message_id)

    for prop in ['QaikuBackup_image_url_view', 'QaikuBackup_image_url_orig']:
        if not obj.has_key(prop):
            continue
        res = fetcherparser.fetch_resource(obj[prop])
        if res: