def comments(self, key, data):
"""Save all comments"""
count = data[key]['count']
if count == 0:
return
comments = [count, ]
for x in xrange(data[key]['count']):
(comment_data, json_stuff) = call_api("wall.getComments",
[("owner_id", self.args.id),
("post_id", data["id"]),
("sort", "asc"),
("offset", x),
("count", 1),
("preview_length", 0),
("need_likes", 1),
("v", 4.4),
], self.args)
comments.append(comment_data[1])
cdata = defaultdict(lambda: '', comment_data[1])
pp = PostParser(self.post_directory, 'comments', self.args)
pp(('comment to ',self.number), cdata, json_stuff)
json_data = json.dumps(comments, indent=4, ensure_ascii=False)
f_name = path.join(self.post_directory, 'comments.json')
out_file = open(f_name, 'a+')
out_file.write(json_data.encode('utf-8'))
out_file.close()
def comments(self, key, data):
"""Save all comments"""
count = data[key]['count']
if count == 0:
return
comments = [
count,
]
for x in xrange(data[key]['count']):
(comment_data, json_stuff) = call_api("wall.getComments", [
("owner_id", self.args.id),
("post_id", data["id"]),
("sort", "asc"),
("offset", x),
("count", 1),
("preview_length", 0),
("need_likes", 1),
("v", 4.4),
], self.args)
comments.append(comment_data[1])
cdata = defaultdict(lambda: '', comment_data[1])
pp = PostParser(self.post_directory, 'comments', self.args)
pp(('comment to ', self.number), cdata, json_stuff)
json_data = json.dumps(comments, indent=4, ensure_ascii=False)
f_name = path.join(self.post_directory, 'comments.json')
out_file = open(f_name, 'a+')
out_file.write(json_data.encode('utf-8'))
out_file.close()
def dl_audio(self, data):
initial_data = data
aid = data["aid"]
owner = data["owner_id"]
request = "{}_{}".format(owner, aid)
(audio_data, json_stuff) = call_api("audio.getById", [
("audios", request),
], self.args)
album = 'no_album'
try:
data = audio_data[0]
artist = data['artist'][:100]
title = data['title'][:100]
name = u"{} - {}.mp3".format(artist, title)
#album = data['album'] # API changed, no time to fix
#album = get_album_name(owner, album, self.args)
#album = escape(album)
make_dir(self.post_directory, album)
self.save_url(data["url"], name, album)
except IndexError: # deleted :(
logging.warning("Deleted track: {}".format(str(initial_data)))
return
# store lyrics if any
try:
lid = data["lyrics_id"]
except KeyError:
return
(lyrics_data, json_stuff) = call_api("audio.getLyrics", [
("lyrics_id", lid),
], self.args)
text = lyrics_data["text"].encode('utf-8')
name = escape(name)
f_name = path.join(self.post_directory, album)
f_name = path.join(f_name, name + '.txt')
# escape!
out_file = open(f_name, 'a+')
out_file.write(text)
out_file.write('\n')
out_file.close()
def dl_audio(self, data):
initial_data = data
aid = data["aid"]
owner = data["owner_id"]
request = "{}_{}".format(owner, aid)
(audio_data, json_stuff) = call_api("audio.getById", [("audios", request), ], self.args)
album = 'no_album'
try:
data = audio_data[0]
artist = data['artist'][:100]
title= data['title'][:100]
name = u"{} - {}.mp3".format(artist, title)
#album = data['album'] # API changed, no time to fix
#album = get_album_name(owner, album, self.args)
#album = escape(album)
make_dir(self.post_directory, album)
self.save_url(data["url"], name, album)
except IndexError: # deleted :(
logging.warning("Deleted track: {}".format(str(initial_data)))
return
# store lyrics if any
try:
lid = data["lyrics_id"]
except KeyError:
return
(lyrics_data, json_stuff) = call_api("audio.getLyrics", [("lyrics_id", lid), ], self.args)
text = lyrics_data["text"].encode('utf-8')
name = escape(name)
f_name = path.join(self.post_directory, album)
f_name = path.join(f_name, name+'.txt')
# escape!
out_file = open(f_name, 'a+')
out_file.write(text)
out_file.write('\n')
out_file.close()
def dl_note(self, data):
"""Download note, not comments"""
(note_data, json_stuff) = call_api("notes.getById", [
("owner_id", data["owner_id"]),
("nid", data["nid"]),
], self.args)
stuff = u"<h1>{title}</h1>\n{text}".format(**note_data)
ndir = make_dir(self.post_directory, 'note_'+note_data["id"])
f_name = path.join(ndir, 'text.html')
out_file = open(f_name, 'a+')
out_file.write(stuff.encode("utf-8"))
out_file.close()
ndata = json.dumps(note_data, indent=4, ensure_ascii=False)
f_name = path.join(ndir, 'raw.json')
out_file = open(f_name, 'a+')
out_file.write(ndata.encode("utf-8"))
out_file.close()
def dl_note(self, data):
"""Download note, not comments"""
(note_data, json_stuff) = call_api("notes.getById", [
("owner_id", data["owner_id"]),
("nid", data["nid"]),
], self.args)
stuff = u"<h1>{title}</h1>\n{text}".format(**note_data)
ndir = make_dir(self.post_directory, 'note_' + note_data["id"])
f_name = path.join(ndir, 'text.html')
out_file = open(f_name, 'a+')
out_file.write(stuff.encode("utf-8"))
out_file.close()
ndata = json.dumps(note_data, indent=4, ensure_ascii=False)
f_name = path.join(ndir, 'raw.json')
out_file = open(f_name, 'a+')
out_file.write(ndata.encode("utf-8"))
out_file.close()
def main():
"""Main function"""
args = arg_parse()
args.access_rights = ["wall", "audio", "friends", "notes", "video", "docs"]
args.token = auth(args) if args.token is None else args.token
if args.token is None:
raise RuntimeError("Access token not found")
if 'wall' in args.mode:
#determine posts count
(response, json_stuff) = call_api("wall.get", [("owner_id", args.id), ("count", 1), ("offset", 0)], args)
count = response[0]
logging.info("Total posts: {}".format(count))
print("Wall dowload start")
args.wall_start, args.wall_end, total = ranges(args.wall_start, args.wall_end, count)
counter = 0.0 # float for %
post_parser = PostParser(args.directory, str(args.id), args)
for x in xrange(args.wall_start, args.wall_end):
if args.verbose and counter % 10 == 0:
print("\nDone: {:.2%} ({})".format(counter / total, int(counter)))
(post, json_stuff) = call_api("wall.get", [("owner_id", args.id), ("count", 1), ("offset", x)], args)
process_post(("wall post", x), post, post_parser, json_stuff)
counter += 1
if args.verbose:
print("\nDone: {:.2%} ({})".format(float(total) / total, int(total)))
if 'audio' in args.mode:
#determine audio count
(response, json_stuff) = call_api("audio.getCount", [("oid", args.id)], args)
count = response
logging.info("Total audio tracks: {}".format(count))
print("Audio dowload start")
args.audio_start, args.audio_end, total = ranges(args.audio_start, args.audio_end, count)
counter = 0.0 # float for %
#audio_dir = os.path.join(str(args.id), 'audio')
audio_dir = str(args.id)
post_parser = PostParser(args.directory, audio_dir, args)
id_param = "uid" if args.id > 0 else "gid"
args.id *= -1 if args.id < 0 else 1
for x in xrange(args.audio_start, args.audio_end):
if args.verbose and counter % 10 == 0:
print("\nDone: {:.2%} ({})".format(counter / total, int(counter)))
(audio, json_stuff) = call_api("audio.get", [(id_param, args.id), ("count", 1), ("offset", x)], args)
process_audio(("audiotrack", x), audio, post_parser, json_stuff)
counter += 1
if args.verbose:
print("\nDone: {:.2%} ({})".format(float(total) / total, int(total)))
if 'video' in args.mode:
raise NotImplementedError("Video mode is not written yet, sorry :(")
if 'notes' in args.mode:
raise NotImplementedError("Notes mode is not written yet, sorry :(")
if 'docs' in args.mode:
# get ALL docs
(response, json_stuff) = call_api("docs.get", [("oid", args.id)], args)
count = response[0]
data = response[1:]
logging.info("Total documents: {}".format(count))
print("Wall dowload start")
args.docs_start, args.docs_end, total = ranges(args.docs_start, args.docs_end, count)
counter = 0.0 # float for %
docs_dir = str(args.id)
post_parser = PostParser(args.directory, docs_dir, args)
data = data[args.docs_start:args.docs_end]
num = args.docs_start
for x in data:
if args.verbose and counter % 10 == 0:
print("\nDone: {:.2%} ({})".format(counter / total, int(counter)))
process_doc(("document", num), x, post_parser, json_stuff)
counter += 1
num += 1
if args.verbose:
print("\nDone: {:.2%} ({})".format(float(total) / total, int(total)))