Ejemplo n.º 1
0
 def comments(self, key, data):
     """Save all comments"""
     count = data[key]['count']
     if count == 0:
         return
     comments = [count, ]
     for x in xrange(data[key]['count']):
         (comment_data, json_stuff) = call_api("wall.getComments",
                                             [("owner_id", self.args.id),
                                                 ("post_id", data["id"]),
                                                 ("sort", "asc"),
                                                 ("offset", x),
                                                 ("count", 1),
                                                 ("preview_length", 0),
                                                 ("need_likes", 1),
                                                 ("v", 4.4),
                                              ], self.args)
         comments.append(comment_data[1])
         cdata = defaultdict(lambda: '', comment_data[1])
         pp = PostParser(self.post_directory, 'comments', self.args)
         pp(('comment to ',self.number), cdata, json_stuff)
     json_data = json.dumps(comments, indent=4, ensure_ascii=False)
     f_name = path.join(self.post_directory, 'comments.json')
     out_file = open(f_name, 'a+')
     out_file.write(json_data.encode('utf-8'))
     out_file.close()
Ejemplo n.º 2
0
 def comments(self, key, data):
     """Save all comments"""
     count = data[key]['count']
     if count == 0:
         return
     comments = [
         count,
     ]
     for x in xrange(data[key]['count']):
         (comment_data, json_stuff) = call_api("wall.getComments", [
             ("owner_id", self.args.id),
             ("post_id", data["id"]),
             ("sort", "asc"),
             ("offset", x),
             ("count", 1),
             ("preview_length", 0),
             ("need_likes", 1),
             ("v", 4.4),
         ], self.args)
         comments.append(comment_data[1])
         cdata = defaultdict(lambda: '', comment_data[1])
         pp = PostParser(self.post_directory, 'comments', self.args)
         pp(('comment to ', self.number), cdata, json_stuff)
     json_data = json.dumps(comments, indent=4, ensure_ascii=False)
     f_name = path.join(self.post_directory, 'comments.json')
     out_file = open(f_name, 'a+')
     out_file.write(json_data.encode('utf-8'))
     out_file.close()
Ejemplo n.º 3
0
    def dl_audio(self, data):
        initial_data = data
        aid = data["aid"]
        owner = data["owner_id"]
        request = "{}_{}".format(owner, aid)
        (audio_data, json_stuff) = call_api("audio.getById", [
            ("audios", request),
        ], self.args)
        album = 'no_album'
        try:
            data = audio_data[0]
            artist = data['artist'][:100]
            title = data['title'][:100]
            name = u"{} - {}.mp3".format(artist, title)
            #album = data['album']  # API changed, no time to fix
            #album = get_album_name(owner, album, self.args)
            #album = escape(album)
            make_dir(self.post_directory, album)
            self.save_url(data["url"], name, album)
        except IndexError:  # deleted :(
            logging.warning("Deleted track: {}".format(str(initial_data)))
            return

        # store lyrics if any
        try:
            lid = data["lyrics_id"]
        except KeyError:
            return
        (lyrics_data, json_stuff) = call_api("audio.getLyrics", [
            ("lyrics_id", lid),
        ], self.args)
        text = lyrics_data["text"].encode('utf-8')
        name = escape(name)
        f_name = path.join(self.post_directory, album)
        f_name = path.join(f_name, name + '.txt')
        # escape!
        out_file = open(f_name, 'a+')
        out_file.write(text)
        out_file.write('\n')
        out_file.close()
Ejemplo n.º 4
0
    def dl_audio(self, data):
        initial_data = data
        aid = data["aid"]
        owner = data["owner_id"]
        request = "{}_{}".format(owner, aid)
        (audio_data, json_stuff) = call_api("audio.getById", [("audios", request), ], self.args)
        album = 'no_album'
        try:
            data = audio_data[0]
            artist = data['artist'][:100]
            title= data['title'][:100]
            name = u"{} - {}.mp3".format(artist, title)
            #album = data['album']  # API changed, no time to fix
            #album = get_album_name(owner, album, self.args)
            #album = escape(album)
            make_dir(self.post_directory, album)
            self.save_url(data["url"], name, album)
        except IndexError: # deleted :(
            logging.warning("Deleted track: {}".format(str(initial_data)))
            return

        # store lyrics if any
        try:
            lid = data["lyrics_id"]
        except KeyError:
            return
        (lyrics_data, json_stuff) = call_api("audio.getLyrics", [("lyrics_id", lid), ], self.args)
        text = lyrics_data["text"].encode('utf-8')
        name = escape(name)
        f_name = path.join(self.post_directory, album)
        f_name = path.join(f_name, name+'.txt')
        # escape!
        out_file = open(f_name, 'a+')
        out_file.write(text)
        out_file.write('\n')
        out_file.close()
Ejemplo n.º 5
0
    def dl_note(self, data):
        """Download note, not comments"""
        (note_data, json_stuff) = call_api("notes.getById", [
            ("owner_id", data["owner_id"]),
            ("nid", data["nid"]),
            ], self.args)
        stuff = u"<h1>{title}</h1>\n{text}".format(**note_data)
        ndir = make_dir(self.post_directory, 'note_'+note_data["id"])
        f_name = path.join(ndir, 'text.html')
        out_file = open(f_name, 'a+')
        out_file.write(stuff.encode("utf-8"))
        out_file.close()

        ndata = json.dumps(note_data, indent=4, ensure_ascii=False)

        f_name = path.join(ndir, 'raw.json')
        out_file = open(f_name, 'a+')
        out_file.write(ndata.encode("utf-8"))
        out_file.close()
Ejemplo n.º 6
0
    def dl_note(self, data):
        """Download note, not comments"""
        (note_data, json_stuff) = call_api("notes.getById", [
            ("owner_id", data["owner_id"]),
            ("nid", data["nid"]),
        ], self.args)
        stuff = u"<h1>{title}</h1>\n{text}".format(**note_data)
        ndir = make_dir(self.post_directory, 'note_' + note_data["id"])
        f_name = path.join(ndir, 'text.html')
        out_file = open(f_name, 'a+')
        out_file.write(stuff.encode("utf-8"))
        out_file.close()

        ndata = json.dumps(note_data, indent=4, ensure_ascii=False)

        f_name = path.join(ndir, 'raw.json')
        out_file = open(f_name, 'a+')
        out_file.write(ndata.encode("utf-8"))
        out_file.close()
Ejemplo n.º 7
0
def main():
    """Main function"""

    args = arg_parse()
    args.access_rights = ["wall", "audio", "friends", "notes", "video", "docs"]
    args.token = auth(args) if args.token is None else args.token
    if args.token is None:
        raise RuntimeError("Access token not found")


    if 'wall' in args.mode:
        #determine posts count
        (response, json_stuff) = call_api("wall.get", [("owner_id", args.id), ("count", 1), ("offset", 0)], args)
        count = response[0]
        logging.info("Total posts: {}".format(count))
        print("Wall dowload start")
        args.wall_start, args.wall_end, total = ranges(args.wall_start, args.wall_end, count)
        counter = 0.0  # float for %
        post_parser = PostParser(args.directory, str(args.id), args)
        for x in xrange(args.wall_start, args.wall_end):
            if args.verbose and counter % 10 == 0:
                print("\nDone: {:.2%} ({})".format(counter / total, int(counter)))
            (post, json_stuff) = call_api("wall.get", [("owner_id", args.id), ("count", 1), ("offset", x)], args)
            process_post(("wall post", x), post, post_parser, json_stuff)
            counter += 1
        if args.verbose:
            print("\nDone: {:.2%} ({})".format(float(total) / total, int(total)))

    if 'audio' in args.mode:
        #determine audio count
        (response, json_stuff) = call_api("audio.getCount", [("oid", args.id)], args)
        count = response
        logging.info("Total audio tracks: {}".format(count))
        print("Audio dowload start")
        args.audio_start, args.audio_end, total = ranges(args.audio_start, args.audio_end, count)
        counter = 0.0  # float for %
        #audio_dir = os.path.join(str(args.id), 'audio')
        audio_dir = str(args.id)
        post_parser = PostParser(args.directory, audio_dir, args)
        id_param = "uid" if args.id > 0 else "gid"
        args.id *= -1 if args.id < 0 else 1
        for x in xrange(args.audio_start, args.audio_end):
            if args.verbose and counter % 10 == 0:
                print("\nDone: {:.2%} ({})".format(counter / total, int(counter)))
            (audio, json_stuff) = call_api("audio.get", [(id_param, args.id), ("count", 1), ("offset", x)], args)
            process_audio(("audiotrack", x), audio, post_parser, json_stuff)
            counter += 1
        if args.verbose:
            print("\nDone: {:.2%} ({})".format(float(total) / total, int(total)))

    if 'video' in args.mode:
        raise NotImplementedError("Video mode is not written yet, sorry :(")
    if 'notes' in args.mode:
        raise NotImplementedError("Notes mode is not written yet, sorry :(")
    if 'docs' in args.mode:
        # get ALL docs
        (response, json_stuff) = call_api("docs.get", [("oid", args.id)], args)
        count = response[0]
        data = response[1:]
        logging.info("Total documents: {}".format(count))
        print("Wall dowload start")
        args.docs_start, args.docs_end, total = ranges(args.docs_start, args.docs_end, count)
        counter = 0.0  # float for %
        docs_dir = str(args.id)
        post_parser = PostParser(args.directory, docs_dir, args)
        data = data[args.docs_start:args.docs_end]
        num = args.docs_start
        for x in data:
            if args.verbose and counter % 10 == 0:
                print("\nDone: {:.2%} ({})".format(counter / total, int(counter)))
            process_doc(("document", num), x, post_parser, json_stuff)
            counter += 1
            num += 1
        if args.verbose:
            print("\nDone: {:.2%} ({})".format(float(total) / total, int(total)))