def debug():
"""For WIP, debug, ect function calls"""
session = sql_functions.connect_to_db()
# tumblr
tumblr_post_dict = {u'reblog_key': u'fZE1iow1', u'reblog': {u'comment': u'', u'tree_html': u'<p><a href="http://asklalalexxi.tumblr.com/post/115588359781/i-normally-dont-like-yogurt-because-of-the" class="tumblr_blog">asklalalexxi</a>:</p><blockquote><p>I normally dont like yogurt because of the consistency. but omg\u2026.ive been starving~ Since the infections gone, I can eat dairy now, and this ironically doesn\u2019t make my stomach upset.\xa0</p></blockquote>', u'trail': [{u'blog': {u'theme': {u'title_font_weight': u'bold', u'title_color': u'#444444', u'header_bounds': 0, u'title_font': u'Gibson', u'link_color': u'#529ECC', u'header_image_focused': u'http://assets.tumblr.com/images/default_header/optica_pattern_08.png?_v=f0f055039bb6136b9661cf2227b535c2', u'show_description': True, u'show_header_image': True, u'header_stretch': True, u'body_font': u'Helvetica Neue', u'show_title': True, u'header_image_scaled': u'http://assets.tumblr.com/images/default_header/optica_pattern_08.png?_v=f0f055039bb6136b9661cf2227b535c2', u'avatar_shape': u'square', u'show_avatar': True, u'background_color': u'#FAFAFA', u'header_image': u'http://assets.tumblr.com/images/default_header/optica_pattern_08.png?_v=f0f055039bb6136b9661cf2227b535c2'}, u'name': u'asklalalexxi'}, u'comment': u'<p>I normally dont like yogurt because of the consistency. but omg\u2026.ive been starving~ Since the infections gone, I can eat dairy now, and this ironically doesn\u2019t make my stomach upset.\xa0</p>', u'post': {u'id': u'115588359781'}}]}, u'player': u'<embed type="application/x-shockwave-flash" src="http://assets.tumblr.com/swf/audio_player.swf?audio_file=https%3A%2F%2Fwww.tumblr.com%2Faudio_file%2Faskbuttonsmom%2F115588501253%2Ftumblr_nmcidl4oPW1sl74rq&color=FFFFFF" height="27" width="207" quality="best" wmode="opaque"></embed>', u'id': 115588501253L, u'post_url': u'http://askbuttonsmom.tumblr.com/post/115588501253/asklalalexxi-i-normally-dont-like-yogurt-because', u'plays': 1550, u'tags': [], u'highlighted': [], u'state': u'published', u'track_name': u'When you havent eaten in a while...', u'short_url': u'http://tmblr.co/ZUGffq1hfd8y5', u'type': u'audio', u'format': u'html', u'timestamp': 1428257396, u'note_count': 67, u'date': u'2015-04-05 18:09:56 GMT', u'slug': u'asklalalexxi-i-normally-dont-like-yogurt-because', u'album_art': u'http://31.media.tumblr.com/tumblr_nmcidl4oPW1sl74rqo1_1428257289_cover.png', u'blog_name': u'askbuttonsmom', u'audio_url': u'https://www.tumblr.com/audio_file/askbuttonsmom/115588501253/tumblr_nmcidl4oPW1sl74rq', u'caption': u'<p><a href="http://asklalalexxi.tumblr.com/post/115588359781/i-normally-dont-like-yogurt-because-of-the" class="tumblr_blog">asklalalexxi</a>:</p><blockquote><p>I normally dont like yogurt because of the consistency. but omg\u2026.ive been starving~ Since the infections gone, I can eat dairy now, and this ironically doesn\u2019t make my stomach upset.\xa0</p></blockquote>', u'audio_type': u'tumblr', u'audio_source_url': u'https://www.tumblr.com/audio_file/askbuttonsmom/115588501253/tumblr_nmcidl4oPW1sl74rq', u'embed': u'<iframe class="tumblr_audio_player tumblr_audio_player_115588501253" src="http://askbuttonsmom.tumblr.com/post/115588501253/audio_player_iframe/askbuttonsmom/tumblr_nmcidl4oPW1sl74rq?audio_file=https%3A%2F%2Fwww.tumblr.com%2Faudio_file%2Faskbuttonsmom%2F115588501253%2Ftumblr_nmcidl4oPW1sl74rq" frameborder="0" allowtransparency="true" scrolling="no" width="500" height="169"></iframe>'}
#tumblr_result = handle_audio_posts(session,tumblr_post_dict)
# soundcloud
soundcloud_post_dict_1 = {u'reblog_key': u'S6VWj0Cb', u'reblog': {u'comment': u'', u'tree_html': u'<p><a class="tumblr_blog" href="http://waltzforluma.tumblr.com/post/111622677961/or-your-computer-could-overheat-while-youre">waltzforluma</a>:</p><blockquote><p>Or, your computer could overheat while you\u2019re listening to \u201cDeath Rag\u201d from Future Vision, and burst into flames!</p></blockquote>', u'trail': [{u'blog': {u'theme': {u'title_font_weight': u'regular', u'header_full_height': 262, u'title_color': u'#444444', u'header_bounds': u'0,623,262,157', u'background_color': u'#FAFAFA', u'link_color': u'#529ECC', u'header_image_focused': u'http://static.tumblr.com/a50dd34705b42b1479c2535a15461b00/oevxq7m/ZrTn5ivly/tumblr_static_tumblr_static_57wsbbc6rz0g0gk4ww8k884wk_focused_v3.png', u'show_description': True, u'header_full_width': 898, u'avatar_shape': u'circle', u'header_focus_width': 466, u'show_header_image': True, u'body_font': u'Helvetica Neue', u'show_title': True, u'header_stretch': True, u'header_image_scaled': u'http://static.tumblr.com/a50dd34705b42b1479c2535a15461b00/oevxq7m/g9cn5ivlx/tumblr_static_57wsbbc6rz0g0gk4ww8k884wk_2048_v2.png', u'show_avatar': True, u'header_focus_height': 262, u'title_font': u'Garamond Classic FS', u'header_image': u'http://static.tumblr.com/a50dd34705b42b1479c2535a15461b00/oevxq7m/g9cn5ivlx/tumblr_static_57wsbbc6rz0g0gk4ww8k884wk.png'}, u'name': u'waltzforluma'}, u'comment': u'<p>Or, your computer could overheat while you\u2019re listening to \u201cDeath Rag\u201d from Future Vision, and burst into flames!</p>', u'post': {u'id': u'111622677961'}}]}, u'player': u'<iframe src="https://w.soundcloud.com/player/?url=https%3A%2F%2Fapi.soundcloud.com%2Ftracks%2F192213990&visual=true&liking=false&sharing=false&auto_play=false&show_comments=false&continuous_play=false&origin=tumblr" frameborder="0" allowtransparency="true" class="soundcloud_audio_player" width="500" height="500"></iframe>', u'id': 113020390888L, u'post_url': u'http://doscoon.tumblr.com/post/113020390888/waltzforluma-or-your-computer-could-overheat', u'source_title': u'waltzforluma', u'format': u'html', u'highlighted': [], u'state': u'published', u'track_name': u'Steven Universe - Death Rag', u'short_url': u'http://tmblr.co/ZlYOqv1fGYate', u'type': u'audio', u'tags': [], u'timestamp': 1425776404, u'note_count': 1014, u'source_url': u'http://waltzforluma.tumblr.com/post/111622677961/or-your-computer-could-overheat-while-youre', u'date': u'2015-03-08 01:00:04 GMT', u'plays': 38933, u'slug': u'waltzforluma-or-your-computer-could-overheat', u'album_art': u'http://38.media.tumblr.com/tumblr_nk3re1A1Cf1qzqb72_1424489834_cover.jpg', u'blog_name': u'doscoon', u'is_external': True, u'audio_url': u'https://api.soundcloud.com/tracks/192213990/stream?client_id=3cQaPshpEeLqMsNFAUw1Q', u'caption': u'<p><a class="tumblr_blog" href="http://waltzforluma.tumblr.com/post/111622677961/or-your-computer-could-overheat-while-youre">waltzforluma</a>:</p><blockquote><p>Or, your computer could overheat while you\u2019re listening to \u201cDeath Rag\u201d from Future Vision, and burst into flames!</p></blockquote>', u'audio_type': u'soundcloud', u'audio_source_url': u'https://soundcloud.com/aivisura/steven-universe-death-rag', u'embed': u'<iframe src="https://w.soundcloud.com/player/?url=https%3A%2F%2Fapi.soundcloud.com%2Ftracks%2F192213990&visual=true&liking=false&sharing=false&auto_play=false&show_comments=false&continuous_play=false&origin=tumblr" frameborder="0" allowtransparency="true" class="soundcloud_audio_player" width="500" height="500"></iframe>'}
#soundcloud_result_1 = handle_audio_posts(session,soundcloud_post_dict_1)
# soundcloud
soundcloud_post_dict_2 = {u'reblog_key': u'S6VWj0Cb', u'reblog': {u'comment': u'', u'tree_html': u'<p><a class="tumblr_blog" href="http://waltzforluma.tumblr.com/post/111622677961/or-your-computer-could-overheat-while-youre">waltzforluma</a>:</p><blockquote><p>Or, your computer could overheat while you\u2019re listening to \u201cDeath Rag\u201d from Future Vision, and burst into flames!</p></blockquote>', u'trail': [{u'blog': {u'theme': {u'title_font_weight': u'regular', u'header_full_height': 262, u'title_color': u'#444444', u'header_bounds': u'0,623,262,157', u'background_color': u'#FAFAFA', u'link_color': u'#529ECC', u'header_image_focused': u'http://static.tumblr.com/a50dd34705b42b1479c2535a15461b00/oevxq7m/ZrTn5ivly/tumblr_static_tumblr_static_57wsbbc6rz0g0gk4ww8k884wk_focused_v3.png', u'show_description': True, u'header_full_width': 898, u'avatar_shape': u'circle', u'header_focus_width': 466, u'show_header_image': True, u'body_font': u'Helvetica Neue', u'show_title': True, u'header_stretch': True, u'header_image_scaled': u'http://static.tumblr.com/a50dd34705b42b1479c2535a15461b00/oevxq7m/g9cn5ivlx/tumblr_static_57wsbbc6rz0g0gk4ww8k884wk_2048_v2.png', u'show_avatar': True, u'header_focus_height': 262, u'title_font': u'Garamond Classic FS', u'header_image': u'http://static.tumblr.com/a50dd34705b42b1479c2535a15461b00/oevxq7m/g9cn5ivlx/tumblr_static_57wsbbc6rz0g0gk4ww8k884wk.png'}, u'name': u'waltzforluma'}, u'comment': u'<p>Or, your computer could overheat while you\u2019re listening to \u201cDeath Rag\u201d from Future Vision, and burst into flames!</p>', u'post': {u'id': u'111622677961'}}]}, u'player': u'<iframe src="https://w.soundcloud.com/player/?url=https%3A%2F%2Fapi.soundcloud.com%2Ftracks%2F192213990&visual=true&liking=false&sharing=false&auto_play=false&show_comments=false&continuous_play=false&origin=tumblr" frameborder="0" allowtransparency="true" class="soundcloud_audio_player" width="500" height="500"></iframe>', u'id': 113020390888L, u'post_url': u'http://doscoon.tumblr.com/post/113020390888/waltzforluma-or-your-computer-could-overheat', u'source_title': u'waltzforluma', u'format': u'html', u'highlighted': [], u'state': u'published', u'track_name': u'Steven Universe - Death Rag', u'short_url': u'http://tmblr.co/ZlYOqv1fGYate', u'type': u'audio', u'tags': [], u'timestamp': 1425776404, u'note_count': 1014, u'source_url': u'http://waltzforluma.tumblr.com/post/111622677961/or-your-computer-could-overheat-while-youre', u'date': u'2015-03-08 01:00:04 GMT', u'plays': 38933, u'slug': u'waltzforluma-or-your-computer-could-overheat', u'album_art': u'http://38.media.tumblr.com/tumblr_nk3re1A1Cf1qzqb72_1424489834_cover.jpg', u'blog_name': u'doscoon', u'is_external': True, u'audio_url': u'https://api.soundcloud.com/tracks/192213990/stream?client_id=3cQaPshpEeLqMsNFAUw1Q', u'caption': u'<p><a class="tumblr_blog" href="http://waltzforluma.tumblr.com/post/111622677961/or-your-computer-could-overheat-while-youre">waltzforluma</a>:</p><blockquote><p>Or, your computer could overheat while you\u2019re listening to \u201cDeath Rag\u201d from Future Vision, and burst into flames!</p></blockquote>', u'audio_type': u'soundcloud', u'audio_source_url': u'https://soundcloud.com/aivisura/steven-universe-death-rag', u'embed': u'<iframe src="https://w.soundcloud.com/player/?url=https%3A%2F%2Fapi.soundcloud.com%2Ftracks%2F192213990&visual=true&liking=false&sharing=false&auto_play=false&show_comments=false&continuous_play=false&origin=tumblr" frameborder="0" allowtransparency="true" class="soundcloud_audio_player" width="500" height="500"></iframe>'}
soundcloud_result_2 = handle_audio_posts(session,soundcloud_post_dict_2)
logging.info("soundcloud_result_2:"+repr(soundcloud_result_2))
# Dead soundcloud
soundcloud_post_dict_3 = {u'reblog_key': u'UygtqfHO', u'reblog': {u'comment': u'', u'tree_html': u''}, u'audio_url': None, u'id': 96589998856L, u'album': u'Pure Heroine - EP', u'highlighted': [], u'plays': 463291, u'format': u'html', u'post_url': u'http://tepidtwisted.tumblr.com/post/96589998856', u'recommended_source': None, u'state': u'published', u'track_name': u'Everybody Wants To Rule The World', u'short_url': u'http://tmblr.co/ZrDCJw1PzDci8', u'type': u'audio', u'tags': [], u'timestamp': 1409799977, u'note_count': 62060, u'source_url': u'http://audio-trip.tumblr.com/post/84453352328', u'trail': [], u'player': u'<iframe src="https://w.soundcloud.com/player/?url=&visual=true&liking=false&sharing=false&auto_play=false&show_comments=false&continuous_play=false&origin=tumblr" frameborder="0" allowtransparency="true" class="soundcloud_audio_player" width="500" height="500"></iframe>', u'date': u'2014-09-04 03:06:17 GMT', u'source_title': u'audio-trip', u'slug': u'', u'album_art': u'http://33.media.tumblr.com/tumblr_n4wy9ee6BY1rhgjr7o1_1398977906_cover.jpg', u'blog_name': u'tepidtwisted', u'is_external': True, u'artist': u'Lorde', u'caption': u'', u'audio_type': u'soundcloud', u'audio_source_url': None, u'embed': u'<iframe src="https://w.soundcloud.com/player/?url=&visual=true&liking=false&sharing=false&auto_play=false&show_comments=false&continuous_play=false&origin=tumblr" frameborder="0" allowtransparency="true" class="soundcloud_audio_player" width="500" height="500"></iframe>'}
soundcloud_result_3 = handle_audio_posts(session,soundcloud_post_dict_3)
logging.info("soundcloud_result_3:"+repr(soundcloud_result_3))
# bandcamp
bandcamp_post_dict = {u'reblog_key': u'AbxPRz4A', u'short_url': u'http://tmblr.co/ZE5FbyhlEyOo', u'audio_url': u'http://popplers5.bandcamp.com/download/track?enc=mp3-128&fsig=ca33bbe78b527483940c050bd627a33d&id=2851867171&nl=1&stream=1&ts=1428792368.0', u'player': u'<iframe class="bandcamp_audio_player" width="500" height="120" src="http://bandcamp.com/EmbeddedPlayer/size=medium/bgcol=ffffff/linkcol=0687f5/notracklist=true/transparent=true/track=2851867171/" allowtransparency="true" frameborder="0"></iframe>', u'id': 46963344946, u'album': u'Hunter Hunted - EP', u'post_url': u'http://staff.tumblr.com/post/46963344946/listen-up-our-audio-player-just-got-super-rad', u'source_title': u'Bandcamp', u'tags': [u'hunter hunted', u'keep together', u'music on tumblr', u'bands on tumblr'], u'highlighted': [], u'state': u'published', u'track_name': u'Keep Together', u'type': u'audio', u'featured_in_tag': [u'Music', u'Design'], u'format': u'html', u'timestamp': 1364937324, u'note_count': 5105, u'source_url': u'http://hunterhuntedmusic.bandcamp.com/track/keep-together', u'date': u'2013-04-02 21:15:24 GMT', u'plays': 110209, u'slug': u'listen-up-our-audio-player-just-got-super-rad', u'album_art': u'http://31.media.tumblr.com/tumblr_mkn8sgrMDN1qz8q0h_dSDeaHscmUuv4sl8epf8mAC32IU_cover.jpg', u'blog_name': u'staff', u'is_external': True, u'artist': u'Hunter Hunted', u'caption': u'<p>Listen up: our audio player just got super rad!</p>\n<ul><li><span>Fancy new audio visualizer</span></li>\n<li><span>Bigger album art</span></li>\n<li><span>Click and drag to skip around</span></li>\n</ul><p>Hit play and happy listening!</p>', u'audio_type': u'bandcamp', u'audio_source_url': u'http://popplers5.bandcamp.com/download/track?enc=mp3-128&fsig=ca33bbe78b527483940c050bd627a33d&id=2851867171&nl=1&stream=1&ts=1428792368.0', u'embed': u'<iframe class="bandcamp_audio_player" width="100%" height="120" src="http://bandcamp.com/EmbeddedPlayer/size=medium/bgcol=ffffff/linkcol=0687f5/notracklist=true/transparent=true/track=2851867171/" allowtransparency="true" frameborder="0"></iframe>'}
#bandcamp_result = handle_audio_posts(session,bandcamp_post_dict)
# Spotify
spotify_post_dict = {u'reblog_key': u'AbxPRz4A', u'short_url': u'http://tmblr.co/ZE5FbyhlEyOo', u'audio_url': u'http://popplers5.bandcamp.com/download/track?enc=mp3-128&fsig=ca33bbe78b527483940c050bd627a33d&id=2851867171&nl=1&stream=1&ts=1428792368.0', u'player': u'<iframe class="bandcamp_audio_player" width="500" height="120" src="http://bandcamp.com/EmbeddedPlayer/size=medium/bgcol=ffffff/linkcol=0687f5/notracklist=true/transparent=true/track=2851867171/" allowtransparency="true" frameborder="0"></iframe>', u'id': 46963344946, u'album': u'Hunter Hunted - EP', u'post_url': u'http://staff.tumblr.com/post/46963344946/listen-up-our-audio-player-just-got-super-rad', u'source_title': u'Bandcamp', u'tags': [u'hunter hunted', u'keep together', u'music on tumblr', u'bands on tumblr'], u'highlighted': [], u'state': u'published', u'track_name': u'Keep Together', u'type': u'audio', u'featured_in_tag': [u'Music', u'Design'], u'format': u'html', u'timestamp': 1364937324, u'note_count': 5105, u'source_url': u'http://hunterhuntedmusic.bandcamp.com/track/keep-together', u'date': u'2013-04-02 21:15:24 GMT', u'plays': 110209, u'slug': u'listen-up-our-audio-player-just-got-super-rad', u'album_art': u'http://31.media.tumblr.com/tumblr_mkn8sgrMDN1qz8q0h_dSDeaHscmUuv4sl8epf8mAC32IU_cover.jpg', u'blog_name': u'staff', u'is_external': True, u'artist': u'Hunter Hunted', u'caption': u'<p>Listen up: our audio player just got super rad!</p>\n<ul><li><span>Fancy new audio visualizer</span></li>\n<li><span>Bigger album art</span></li>\n<li><span>Click and drag to skip around</span></li>\n</ul><p>Hit play and happy listening!</p>', u'audio_type': u'bandcamp', u'audio_source_url': u'http://popplers5.bandcamp.com/download/track?enc=mp3-128&fsig=ca33bbe78b527483940c050bd627a33d&id=2851867171&nl=1&stream=1&ts=1428792368.0', u'embed': u'<iframe class="bandcamp_audio_player" width="100%" height="120" src="http://bandcamp.com/EmbeddedPlayer/size=medium/bgcol=ffffff/linkcol=0687f5/notracklist=true/transparent=true/track=2851867171/" allowtransparency="true" frameborder="0"></iframe>'}
#spotify_result = handle_audio_posts(session,spotify_post_dict)
return
def post_producer(post_queue, target_blog=None):
"""Keep a queue-like object filled with post dicts"""
# Connect to the DB
database_session = sql_functions.connect_to_db()
# Keep the queue supplied with posts
counter = 0
while True:
if post_queue.qsize() < 100:
logging.info(
"Adding more posts to post queue. Posts added so far: " +
repr(counter))
new_posts = list_new_posts(database_session=database_session,
max_rows=1000,
target_blog=target_blog)
if len(new_posts) == 0:
# Tell consumers to stop by flooding the queue with None objects
#logging.error("No more posts to process, exiting.")
for n in xrange(100):
post_queue.put(None)
return
for new_post in new_posts:
assert (post_queue.full() is False)
counter += 1
post_queue.put(new_post)
else:
time.sleep(1)
def generate_stats_page():
logging.info("Generating stats page...")
# Connect to the DB
session = sql_functions.connect_to_db()
page_html = "Rowcounts:" + "\r\n"
# Get rowcounts
raw_posts_rowcount = session.query(
sqlalchemy.func.count(RawPosts.primary_key)).scalar()
media_rowcount = session.query(sqlalchemy.func.count(
Media.media_id)).scalar()
posts_rowcount = session.query(sqlalchemy.func.count(
posts.post_id)).scalar()
blogs_rowcount = session.query(sqlalchemy.func.count(
blogs.blog_id)).scalar()
# Add values to page
page_html += "raw_posts: " + splitthousands(
str(raw_posts_rowcount)) + "\r\n"
page_html += "posts: " + splitthousands(str(posts_rowcount)) + "\r\n"
page_html += "blogs: " + splitthousands(str(blogs_rowcount)) + "\r\n"
page_html += "Media: " + splitthousands(str(media_rowcount)) + "\r\n"
page_html += "\r\n"
page_html += "\r\n"
page_html += "\r\n"
page_html += "Ratios:" + "\r\n"
page_html += "raw_posts/posts: " + repr(
(raw_posts_rowcount / posts_rowcount)) + "\r\n"
page_html += "raw_posts: " + splitthousands(
str(raw_posts_rowcount)) + ", posts: " + splitthousands(
str(posts_rowcount)) + "\r\n"
logging.debug(page_html)
save_file(file_path="stats.txt",
data=page_html,
force_save=True,
allow_fail=False)
logging.info("Generated stats page.")
def blog_consumer(blog_url_queue):
"Process posts for a queue-like object containing blog URLs"
logging.debug("Consumer function started.")
# Connect to DB
database_session = sql_functions.connect_to_db()
# Process posts from the queue
c = 0# Counter for number of posts processed
while True:
c += 1
if c%100 == 0:
logging.info(repr(c)+" posts processed by this process")
#suicide_timer = threading.Timer(1200, suicider_media)# Kill after 20 minutes (1200 seconds)
#suicide_timer.start()
blog_url = blog_url_queue.get(timeout=600)
if blog_url is None:# Stop if None object is put into the queue
logging.info("Post consumer recieved None object as exit signal")
break# Stop doing work and exit thread/process
save_blog(blog_url)
#suicide_timer.cancel()# Remove suicide timer after each blog
continue
# Disconnect from DB
database_session.close()
logging.debug("Consumer function exiting.")
return
def debug():
"""For WIP, debug, ect function calls"""
session = sql_functions.connect_to_db()
# unusual url u'http://fallfeatherspony.tumblr.com/post/109567450555/askloona-\u1d1b-\u029c-\u1d07-\u028f-\u1d21-\u1d07-\u0280-\u1d07'
strange_url_result_1 = run_yt_dl_single(
session,
download_url=
u'http://fallfeatherspony.tumblr.com/post/109567450555/askloona-\u1d1b-\u029c-\u1d07-\u028f-\u1d21-\u1d07-\u0280-\u1d07',
extractor_used="DEBUG:strange",
audio_id=None,
video_id=None)
logging.info("strange_url_result_1: " + repr(strange_url_result_1))
#return
# gfycat
gfycat_result = run_yt_dl_single(
session,
download_url="http://gfycat.com/MerryAdolescentAlaskankleekai",
extractor_used="DEBUG:linked_gfycat",
audio_id=None,
video_id=None)
logging.debug("gfycat_result: " + repr(gfycat_result))
# youtube https://www.youtube.com/embed/JIQ_jjfDsw0
youtube_result = run_yt_dl_single(
session,
download_url="https://www.youtube.com/embed/JIQ_jjfDsw0",
extractor_used="DEBUG:youtube",
audio_id=None,
video_id=None)
return
def generate_stats_page():
logging.info("Generating stats page...")
# Connect to the DB
session = sql_functions.connect_to_db()
page_html = "Rowcounts:"+"\r\n"
# Get rowcounts
raw_posts_rowcount = session.query(sqlalchemy.func.count(RawPosts.primary_key)).scalar()
media_rowcount = session.query(sqlalchemy.func.count(Media.media_id)).scalar()
posts_rowcount = session.query(sqlalchemy.func.count(posts.post_id)).scalar()
blogs_rowcount = session.query(sqlalchemy.func.count(blogs.blog_id)).scalar()
# Add values to page
page_html += "raw_posts: "+splitthousands(str(raw_posts_rowcount))+"\r\n"
page_html += "posts: "+splitthousands(str(posts_rowcount))+"\r\n"
page_html += "blogs: "+splitthousands(str(blogs_rowcount))+"\r\n"
page_html += "Media: "+splitthousands(str(media_rowcount))+"\r\n"
page_html += "\r\n"
page_html += "\r\n"
page_html += "\r\n"
page_html += "Ratios:"+"\r\n"
page_html += "raw_posts/posts: "+repr((raw_posts_rowcount/posts_rowcount))+"\r\n"
page_html += "raw_posts: "+splitthousands(str(raw_posts_rowcount))+", posts: "+splitthousands(str(posts_rowcount))+"\r\n"
logging.debug(page_html)
save_file(
file_path="stats.txt",
data=page_html,
force_save=True,
allow_fail=False
)
logging.info("Generated stats page.")
def debug():
"""For WIP, debug, ect function calls"""
session = sql_functions.connect_to_db()
# unusual url u'http://fallfeatherspony.tumblr.com/post/109567450555/askloona-\u1d1b-\u029c-\u1d07-\u028f-\u1d21-\u1d07-\u0280-\u1d07'
strange_url_result_1 = run_yt_dl_single(
session,
download_url = u'http://fallfeatherspony.tumblr.com/post/109567450555/askloona-\u1d1b-\u029c-\u1d07-\u028f-\u1d21-\u1d07-\u0280-\u1d07',
extractor_used = "DEBUG:strange",
audio_id=None,
video_id=None
)
logging.info("strange_url_result_1: "+repr(strange_url_result_1))
#return
# gfycat
gfycat_result = run_yt_dl_single(
session,
download_url = "http://gfycat.com/MerryAdolescentAlaskankleekai",
extractor_used = "DEBUG:linked_gfycat",
audio_id=None,
video_id=None
)
logging.debug("gfycat_result: "+repr(gfycat_result))
# youtube https://www.youtube.com/embed/JIQ_jjfDsw0
youtube_result = run_yt_dl_single(
session,
download_url = "https://www.youtube.com/embed/JIQ_jjfDsw0",
extractor_used = "DEBUG:youtube",
audio_id=None,
video_id=None
)
return
def save_blog(blog_url):
"""Save one tumblr blog"""
logging.info("Saving blog: "+repr(blog_url))
try:
# Connect to DB
session = sql_functions.connect_to_db()
# Instantiate blog, collect blog metadata
blog = tumblr_blog(session, consumer_key = config.consumer_key, blog_url=blog_url)
# Ensure blog actually exists
if blog.blog_exists is False:
logging.error("Blog does not exist! "+repr(blog_url))
appendlist(
blog_url,
list_file_path="tumblr_failed_list.txt",
initial_text="# List of failed items.\n"
)
return
# Add new posts for the blog
posts = blog.save_new_posts(config.max_pages_to_check)
logging.info("Finished saving blog: "+repr(blog_url))
appendlist(blog_url,list_file_path=config.done_list_path,initial_text="# List of completed items.\n")
except Exception, e:# Log exceptions and pass them on
logging.critical("Unhandled exception in save_blog()!")
logging.exception(e)
raise
def post_producer(post_queue,target_blog=None):
"""Keep a queue-like object filled with post dicts"""
# Connect to the DB
database_session = sql_functions.connect_to_db()
# Keep the queue supplied with posts
counter = 0
while True:
if post_queue.qsize() < 100:
logging.info("Adding more posts to post queue. Posts added so far: "+repr(counter))
new_posts = list_new_posts(
database_session=database_session,
max_rows=1000,
target_blog=target_blog
)
if len(new_posts) == 0:
# Tell consumers to stop by flooding the queue with None objects
#logging.error("No more posts to process, exiting.")
for n in xrange(100):
post_queue.put(None)
return
for new_post in new_posts:
assert(post_queue.full() is False)
counter += 1
post_queue.put(new_post)
else:
time.sleep(1)
def debug():
"""For WIP, debug, ect function calls"""
session = sql_functions.connect_to_db()
print "foo"
# Album
album_media_id_list = save_album(
session,
album_link = "http://imgur.com/a/AMibi#0"
)
logging.debug("album_media_id_list:"+repr(album_media_id_list))
# Multiple image link
multiple_image_link_media_id_list = save_imgur(
session,
link = "http://imgur.com/UQ1j8OT,2PvFmxV#1"
)
logging.debug("multiple_image_link_media_id_list:"+repr(multiple_image_link_media_id_list))
# Single image link
single_image_link_media_id_list = save_imgur(
session,
link = "http://imgur.com/UQ1j8OT"
)
logging.debug("single_image_link_media_id_list:"+repr(single_image_link_media_id_list))
logging.debug(repr(locals()))
return
def debug():
"""Code for debugging during programming goes here so everything is logged to file"""
session = sql_functions.connect_to_db()
# Debug video DB check
#sql_functions.check_if_video_in_db(connection,media_url="https://www.youtube.com/embed/lGIEmH3BoyA",youtube_id=None,sha512base64_hash=None,post_id=None)
#return
# Debug images
logging.debug("Debug images")
image_post_dict = {u'highlighted': [], u'reblog_key': u'RSNOnudd', u'format': u'html', u'timestamp': 1401396780, u'note_count': 429, u'tags': [u'p**n', u'furry', u'anthro', u'art', u'fantasy', u'compilation', u'myart', u'futa', u'female', u'nude', u'werewolf'], 'link_to_hash_dict': {}, u'photos': [{u'caption': u'My character Gwen, the hermaphrodite Unicorn. Short for I Guinevere.', u'original_size': {u'url': u'http://41.media.tumblr.com/51dc06d26888063e978967b9effdd79d/tumblr_n6csptiJ5u1rzato1o1_1280.jpg', u'width': 1280, u'height': 1739}, u'alt_sizes': [{u'url': u'http://41.media.tumblr.com/51dc06d26888063e978967b9effdd79d/tumblr_n6csptiJ5u1rzato1o1_1280.jpg', u'width': 1280, u'height': 1739}, {u'url': u'http://41.media.tumblr.com/51dc06d26888063e978967b9effdd79d/tumblr_n6csptiJ5u1rzato1o1_500.jpg', u'width': 500, u'height': 679}, {u'url': u'http://41.media.tumblr.com/51dc06d26888063e978967b9effdd79d/tumblr_n6csptiJ5u1rzato1o1_400.jpg', u'width': 400, u'height': 543}, {u'url': u'http://40.media.tumblr.com/51dc06d26888063e978967b9effdd79d/tumblr_n6csptiJ5u1rzato1o1_250.jpg', u'width': 250, u'height': 340}, {u'url': u'http://41.media.tumblr.com/51dc06d26888063e978967b9effdd79d/tumblr_n6csptiJ5u1rzato1o1_100.jpg', u'width': 100, u'height': 136}, {u'url': u'http://41.media.tumblr.com/51dc06d26888063e978967b9effdd79d/tumblr_n6csptiJ5u1rzato1o1_75sq.jpg', u'width': 75, u'height': 75}]}, {u'caption': u'A young man and one of his harem concubines.', u'original_size': {u'url': u'http://40.media.tumblr.com/df5d6e743955acef44262810e7e68196/tumblr_n6csptiJ5u1rzato1o2_1280.jpg', u'width': 1280, u'height': 1037}, u'alt_sizes': [{u'url': u'http://40.media.tumblr.com/df5d6e743955acef44262810e7e68196/tumblr_n6csptiJ5u1rzato1o2_1280.jpg', u'width': 1280, u'height': 1037}, {u'url': u'http://40.media.tumblr.com/df5d6e743955acef44262810e7e68196/tumblr_n6csptiJ5u1rzato1o2_500.jpg', u'width': 500, u'height': 405}, {u'url': u'http://41.media.tumblr.com/df5d6e743955acef44262810e7e68196/tumblr_n6csptiJ5u1rzato1o2_400.jpg', u'width': 400, u'height': 324}, {u'url': u'http://40.media.tumblr.com/df5d6e743955acef44262810e7e68196/tumblr_n6csptiJ5u1rzato1o2_250.jpg', u'width': 250, u'height': 203}, {u'url': u'http://41.media.tumblr.com/df5d6e743955acef44262810e7e68196/tumblr_n6csptiJ5u1rzato1o2_100.jpg', u'width': 100, u'height': 81}, {u'url': u'http://40.media.tumblr.com/df5d6e743955acef44262810e7e68196/tumblr_n6csptiJ5u1rzato1o2_75sq.jpg', u'width': 75, u'height': 75}]}, {u'caption': u'Gift-art for Robotjoe at FA.', u'original_size': {u'url': u'http://40.media.tumblr.com/027e4e40a7b6dd7437ba19bb0bf66394/tumblr_n6csptiJ5u1rzato1o3_1280.jpg', u'width': 1280, u'height': 1280}, u'alt_sizes': [{u'url': u'http://40.media.tumblr.com/027e4e40a7b6dd7437ba19bb0bf66394/tumblr_n6csptiJ5u1rzato1o3_1280.jpg', u'width': 1280, u'height': 1280}, {u'url': u'http://41.media.tumblr.com/027e4e40a7b6dd7437ba19bb0bf66394/tumblr_n6csptiJ5u1rzato1o3_500.jpg', u'width': 500, u'height': 500}, {u'url': u'http://41.media.tumblr.com/027e4e40a7b6dd7437ba19bb0bf66394/tumblr_n6csptiJ5u1rzato1o3_400.jpg', u'width': 400, u'height': 400}, {u'url': u'http://41.media.tumblr.com/027e4e40a7b6dd7437ba19bb0bf66394/tumblr_n6csptiJ5u1rzato1o3_250.jpg', u'width': 250, u'height': 250}, {u'url': u'http://40.media.tumblr.com/027e4e40a7b6dd7437ba19bb0bf66394/tumblr_n6csptiJ5u1rzato1o3_100.jpg', u'width': 100, u'height': 100}, {u'url': u'http://40.media.tumblr.com/027e4e40a7b6dd7437ba19bb0bf66394/tumblr_n6csptiJ5u1rzato1o3_75sq.jpg', u'width': 75, u'height': 75}]}, {u'caption': u'Giftart for Ritts at FA.', u'original_size': {u'url': u'http://41.media.tumblr.com/b04099342f13a3aaad3ef8d7f9f3080f/tumblr_n6csptiJ5u1rzato1o4_1280.jpg', u'width': 1280, u'height': 1152}, u'alt_sizes': [{u'url': u'http://41.media.tumblr.com/b04099342f13a3aaad3ef8d7f9f3080f/tumblr_n6csptiJ5u1rzato1o4_1280.jpg', u'width': 1280, u'height': 1152}, {u'url': u'http://40.media.tumblr.com/b04099342f13a3aaad3ef8d7f9f3080f/tumblr_n6csptiJ5u1rzato1o4_500.jpg', u'width': 500, u'height': 450}, {u'url': u'http://40.media.tumblr.com/b04099342f13a3aaad3ef8d7f9f3080f/tumblr_n6csptiJ5u1rzato1o4_400.jpg', u'width': 400, u'height': 360}, {u'url': u'http://40.media.tumblr.com/b04099342f13a3aaad3ef8d7f9f3080f/tumblr_n6csptiJ5u1rzato1o4_250.jpg', u'width': 250, u'height': 225}, {u'url': u'http://40.media.tumblr.com/b04099342f13a3aaad3ef8d7f9f3080f/tumblr_n6csptiJ5u1rzato1o4_100.jpg', u'width': 100, u'height': 90}, {u'url': u'http://41.media.tumblr.com/b04099342f13a3aaad3ef8d7f9f3080f/tumblr_n6csptiJ5u1rzato1o4_75sq.jpg', u'width': 75, u'height': 75}]}, {u'caption': u'', u'original_size': {u'url': u'http://41.media.tumblr.com/96a2f5867ff3269def55cba3ddb42282/tumblr_n6csptiJ5u1rzato1o5_1280.jpg', u'width': 1153, u'height': 1920}, u'alt_sizes': [{u'url': u'http://41.media.tumblr.com/96a2f5867ff3269def55cba3ddb42282/tumblr_n6csptiJ5u1rzato1o5_1280.jpg', u'width': 1153, u'height': 1920}, {u'url': u'http://40.media.tumblr.com/96a2f5867ff3269def55cba3ddb42282/tumblr_n6csptiJ5u1rzato1o5_500.jpg', u'width': 450, u'height': 750}, {u'url': u'http://40.media.tumblr.com/96a2f5867ff3269def55cba3ddb42282/tumblr_n6csptiJ5u1rzato1o5_400.jpg', u'width': 360, u'height': 600}, {u'url': u'http://40.media.tumblr.com/96a2f5867ff3269def55cba3ddb42282/tumblr_n6csptiJ5u1rzato1o5_250.jpg', u'width': 240, u'height': 400}, {u'url': u'http://41.media.tumblr.com/96a2f5867ff3269def55cba3ddb42282/tumblr_n6csptiJ5u1rzato1o5_100.jpg', u'width': 100, u'height': 167}, {u'url': u'http://36.media.tumblr.com/96a2f5867ff3269def55cba3ddb42282/tumblr_n6csptiJ5u1rzato1o5_75sq.jpg', u'width': 75, u'height': 75}]}, {u'caption': u'She shot herself in the face, or did others? Up to you.', u'original_size': {u'url': u'http://41.media.tumblr.com/879c064933cf138ae169a152dbd717a4/tumblr_n6csptiJ5u1rzato1o6_1280.jpg', u'width': 841, u'height': 1400}, u'alt_sizes': [{u'url': u'http://41.media.tumblr.com/879c064933cf138ae169a152dbd717a4/tumblr_n6csptiJ5u1rzato1o6_1280.jpg', u'width': 841, u'height': 1400}, {u'url': u'http://40.media.tumblr.com/879c064933cf138ae169a152dbd717a4/tumblr_n6csptiJ5u1rzato1o6_500.jpg', u'width': 451, u'height': 750}, {u'url': u'http://41.media.tumblr.com/879c064933cf138ae169a152dbd717a4/tumblr_n6csptiJ5u1rzato1o6_400.jpg', u'width': 360, u'height': 600}, {u'url': u'http://40.media.tumblr.com/879c064933cf138ae169a152dbd717a4/tumblr_n6csptiJ5u1rzato1o6_250.jpg', u'width': 240, u'height': 400}, {u'url': u'http://40.media.tumblr.com/879c064933cf138ae169a152dbd717a4/tumblr_n6csptiJ5u1rzato1o6_100.jpg', u'width': 100, u'height': 166}, {u'url': u'http://36.media.tumblr.com/879c064933cf138ae169a152dbd717a4/tumblr_n6csptiJ5u1rzato1o6_75sq.jpg', u'width': 75, u'height': 75}]}, {u'caption': u"They're now twins.", u'original_size': {u'url': u'http://36.media.tumblr.com/56df999c75b2ea6e10d9e9e3a4248db6/tumblr_n6csptiJ5u1rzato1o7_1280.jpg', u'width': 1153, u'height': 1920}, u'alt_sizes': [{u'url': u'http://36.media.tumblr.com/56df999c75b2ea6e10d9e9e3a4248db6/tumblr_n6csptiJ5u1rzato1o7_1280.jpg', u'width': 1153, u'height': 1920}, {u'url': u'http://36.media.tumblr.com/56df999c75b2ea6e10d9e9e3a4248db6/tumblr_n6csptiJ5u1rzato1o7_500.jpg', u'width': 450, u'height': 750}, {u'url': u'http://41.media.tumblr.com/56df999c75b2ea6e10d9e9e3a4248db6/tumblr_n6csptiJ5u1rzato1o7_400.jpg', u'width': 360, u'height': 600}, {u'url': u'http://40.media.tumblr.com/56df999c75b2ea6e10d9e9e3a4248db6/tumblr_n6csptiJ5u1rzato1o7_250.jpg', u'width': 240, u'height': 400}, {u'url': u'http://41.media.tumblr.com/56df999c75b2ea6e10d9e9e3a4248db6/tumblr_n6csptiJ5u1rzato1o7_100.jpg', u'width': 100, u'height': 167}, {u'url': u'http://41.media.tumblr.com/56df999c75b2ea6e10d9e9e3a4248db6/tumblr_n6csptiJ5u1rzato1o7_75sq.jpg', u'width': 75, u'height': 75}]}, {u'caption': u'This is knot a funny joke.', u'original_size': {u'url': u'http://36.media.tumblr.com/dd7b3f0723d26d0cf9a5daff5cb82e8a/tumblr_n6csptiJ5u1rzato1o8_1280.jpg', u'width': 1000, u'height': 1000}, u'alt_sizes': [{u'url': u'http://36.media.tumblr.com/dd7b3f0723d26d0cf9a5daff5cb82e8a/tumblr_n6csptiJ5u1rzato1o8_1280.jpg', u'width': 1000, u'height': 1000}, {u'url': u'http://40.media.tumblr.com/dd7b3f0723d26d0cf9a5daff5cb82e8a/tumblr_n6csptiJ5u1rzato1o8_500.jpg', u'width': 500, u'height': 500}, {u'url': u'http://40.media.tumblr.com/dd7b3f0723d26d0cf9a5daff5cb82e8a/tumblr_n6csptiJ5u1rzato1o8_400.jpg', u'width': 400, u'height': 400}, {u'url': u'http://36.media.tumblr.com/dd7b3f0723d26d0cf9a5daff5cb82e8a/tumblr_n6csptiJ5u1rzato1o8_250.jpg', u'width': 250, u'height': 250}, {u'url': u'http://40.media.tumblr.com/dd7b3f0723d26d0cf9a5daff5cb82e8a/tumblr_n6csptiJ5u1rzato1o8_100.jpg', u'width': 100, u'height': 100}, {u'url': u'http://40.media.tumblr.com/dd7b3f0723d26d0cf9a5daff5cb82e8a/tumblr_n6csptiJ5u1rzato1o8_75sq.jpg', u'width': 75, u'height': 75}]}, {u'caption': u'Gift-art for Quillu at FA.', u'original_size': {u'url': u'http://41.media.tumblr.com/e02229ba14b4fb3be2866595e371aaa7/tumblr_n6csptiJ5u1rzato1o9_1280.jpg', u'width': 800, u'height': 1410}, u'alt_sizes': [{u'url': u'http://41.media.tumblr.com/e02229ba14b4fb3be2866595e371aaa7/tumblr_n6csptiJ5u1rzato1o9_1280.jpg', u'width': 800, u'height': 1410}, {u'url': u'http://40.media.tumblr.com/e02229ba14b4fb3be2866595e371aaa7/tumblr_n6csptiJ5u1rzato1o9_500.jpg', u'width': 426, u'height': 750}, {u'url': u'http://41.media.tumblr.com/e02229ba14b4fb3be2866595e371aaa7/tumblr_n6csptiJ5u1rzato1o9_400.jpg', u'width': 340, u'height': 600}, {u'url': u'http://41.media.tumblr.com/e02229ba14b4fb3be2866595e371aaa7/tumblr_n6csptiJ5u1rzato1o9_250.jpg', u'width': 227, u'height': 400}, {u'url': u'http://40.media.tumblr.com/e02229ba14b4fb3be2866595e371aaa7/tumblr_n6csptiJ5u1rzato1o9_100.jpg', u'width': 100, u'height': 176}, {u'url': u'http://41.media.tumblr.com/e02229ba14b4fb3be2866595e371aaa7/tumblr_n6csptiJ5u1rzato1o9_75sq.jpg', u'width': 75, u'height': 75}]}, {u'caption': u'Werewolf herm, in heat. Watch out!', u'original_size': {u'url': u'http://36.media.tumblr.com/3a23e75b564f5d790bb440ba2ba6140c/tumblr_n6csptiJ5u1rzato1o10_1280.jpg', u'width': 1280, u'height': 962}, u'alt_sizes': [{u'url': u'http://36.media.tumblr.com/3a23e75b564f5d790bb440ba2ba6140c/tumblr_n6csptiJ5u1rzato1o10_1280.jpg', u'width': 1280, u'height': 962}, {u'url': u'http://41.media.tumblr.com/3a23e75b564f5d790bb440ba2ba6140c/tumblr_n6csptiJ5u1rzato1o10_500.jpg', u'width': 500, u'height': 376}, {u'url': u'http://41.media.tumblr.com/3a23e75b564f5d790bb440ba2ba6140c/tumblr_n6csptiJ5u1rzato1o10_400.jpg', u'width': 400, u'height': 301}, {u'url': u'http://36.media.tumblr.com/3a23e75b564f5d790bb440ba2ba6140c/tumblr_n6csptiJ5u1rzato1o10_250.jpg', u'width': 250, u'height': 188}, {u'url': u'http://36.media.tumblr.com/3a23e75b564f5d790bb440ba2ba6140c/tumblr_n6csptiJ5u1rzato1o10_100.jpg', u'width': 100, u'height': 75}, {u'url': u'http://40.media.tumblr.com/3a23e75b564f5d790bb440ba2ba6140c/tumblr_n6csptiJ5u1rzato1o10_75sq.jpg', u'width': 75, u'height': 75}]}], u'id': 87231460597L, u'post_url': u'http://zaggatar.tumblr.com/post/87231460597/i-thought-i-would-upload-some-of-what-i-think-is', u'caption': u'<p><span>I thought I would upload s</span>ome of what I think is best of my older stuff.</p>\n<p>As you can see, I am guilty for liking horsegirls with big dicks.</p>\n<p>Enjoy.</p>', u'state': u'published', u'short_url': u'http://tmblr.co/Zlxuxu1HFPdJr', u'date': u'2014-05-29 20:53:00 GMT', u'type': u'photo', u'slug': u'i-thought-i-would-upload-some-of-what-i-think-is', u'photoset_layout': u'1111111111', u'blog_name': u'zaggatar'}
#print flatten(image_post_dict)
#new_post_dict = save_media(session,image_post_dict)
#download_image_link(session,"https://derpicdn.net/spns/W1siZiIsIjIwMTQvMDEvMTAvMDJfNDBfMjhfNjUyX2RlcnBpYm9vcnVfYmFubmVyLnBuZyJdXQ.png")
# Debug audio
#logging.debug("Debug audio")
#soundcloud_post_dict = {u'reblog_key': u'S6VWj0Cb', u'reblog': {u'comment': u'', u'tree_html': u'<p><a class="tumblr_blog" href="http://waltzforluma.tumblr.com/post/111622677961/or-your-computer-could-overheat-while-youre">waltzforluma</a>:</p><blockquote><p>Or, your computer could overheat while you\u2019re listening to \u201cDeath Rag\u201d from Future Vision, and burst into flames!</p></blockquote>', u'trail': [{u'blog': {u'theme': {u'title_font_weight': u'regular', u'header_full_height': 262, u'title_color': u'#444444', u'header_bounds': u'0,623,262,157', u'background_color': u'#FAFAFA', u'link_color': u'#529ECC', u'header_image_focused': u'http://static.tumblr.com/a50dd34705b42b1479c2535a15461b00/oevxq7m/ZrTn5ivly/tumblr_static_tumblr_static_57wsbbc6rz0g0gk4ww8k884wk_focused_v3.png', u'show_description': True, u'header_full_width': 898, u'avatar_shape': u'circle', u'header_focus_width': 466, u'show_header_image': True, u'body_font': u'Helvetica Neue', u'show_title': True, u'header_stretch': True, u'header_image_scaled': u'http://static.tumblr.com/a50dd34705b42b1479c2535a15461b00/oevxq7m/g9cn5ivlx/tumblr_static_57wsbbc6rz0g0gk4ww8k884wk_2048_v2.png', u'show_avatar': True, u'header_focus_height': 262, u'title_font': u'Garamond Classic FS', u'header_image': u'http://static.tumblr.com/a50dd34705b42b1479c2535a15461b00/oevxq7m/g9cn5ivlx/tumblr_static_57wsbbc6rz0g0gk4ww8k884wk.png'}, u'name': u'waltzforluma'}, u'comment': u'<p>Or, your computer could overheat while you\u2019re listening to \u201cDeath Rag\u201d from Future Vision, and burst into flames!</p>', u'post': {u'id': u'111622677961'}}]}, u'player': u'<iframe src="https://w.soundcloud.com/player/?url=https%3A%2F%2Fapi.soundcloud.com%2Ftracks%2F192213990&visual=true&liking=false&sharing=false&auto_play=false&show_comments=false&continuous_play=false&origin=tumblr" frameborder="0" allowtransparency="true" class="soundcloud_audio_player" width="500" height="500"></iframe>', u'id': 113020390888L, u'post_url': u'http://doscoon.tumblr.com/post/113020390888/waltzforluma-or-your-computer-could-overheat', u'source_title': u'waltzforluma', u'format': u'html', u'highlighted': [], u'state': u'published', u'track_name': u'Steven Universe - Death Rag', u'short_url': u'http://tmblr.co/ZlYOqv1fGYate', u'type': u'audio', u'tags': [], u'timestamp': 1425776404, u'note_count': 1014, u'source_url': u'http://waltzforluma.tumblr.com/post/111622677961/or-your-computer-could-overheat-while-youre', u'date': u'2015-03-08 01:00:04 GMT', u'plays': 38933, u'slug': u'waltzforluma-or-your-computer-could-overheat', u'album_art': u'http://38.media.tumblr.com/tumblr_nk3re1A1Cf1qzqb72_1424489834_cover.jpg', u'blog_name': u'doscoon', u'is_external': True, u'audio_url': u'https://api.soundcloud.com/tracks/192213990/stream?client_id=3cQaPshpEeLqMsNFAUw1Q', u'caption': u'<p><a class="tumblr_blog" href="http://waltzforluma.tumblr.com/post/111622677961/or-your-computer-could-overheat-while-youre">waltzforluma</a>:</p><blockquote><p>Or, your computer could overheat while you\u2019re listening to \u201cDeath Rag\u201d from Future Vision, and burst into flames!</p></blockquote>', u'audio_type': u'soundcloud', u'audio_source_url': u'https://soundcloud.com/aivisura/steven-universe-death-rag', u'embed': u'<iframe src="https://w.soundcloud.com/player/?url=https%3A%2F%2Fapi.soundcloud.com%2Ftracks%2F192213990&visual=true&liking=false&sharing=false&auto_play=false&show_comments=false&continuous_play=false&origin=tumblr" frameborder="0" allowtransparency="true" class="soundcloud_audio_player" width="500" height="500"></iframe>'}
#audio_handlers.handle_soundcloud_audio(session,soundcloud_post_dict)
# Debug video
logging.debug("Debug video")
# Tumblr video
#tumblr_video_post_dict = {u'reblog_key': u'3bqfxHgy', u'short_url': u'http://tmblr.co/Z_sLQw1eYTSqS', u'thumbnail_width': 480, u'player': [{u'width': 250, u'embed_code': u'<iframe width="250" height="140" id="youtube_iframe" src="https://www.youtube.com/embed/tT5pifkZzEk?feature=oembed&enablejsapi=1&origin=http://safe.txmblr.com&wmode=opaque" frameborder="0" allowfullscreen></iframe>'}, {u'width': 400, u'embed_code': u'<iframe width="400" height="225" id="youtube_iframe" src="https://www.youtube.com/embed/tT5pifkZzEk?feature=oembed&enablejsapi=1&origin=http://safe.txmblr.com&wmode=opaque" frameborder="0" allowfullscreen></iframe>'}, {u'width': 500, u'embed_code': u'<iframe width="500" height="281" id="youtube_iframe" src="https://www.youtube.com/embed/tT5pifkZzEk?feature=oembed&enablejsapi=1&origin=http://safe.txmblr.com&wmode=opaque" frameborder="0" allowfullscreen></iframe>'}], u'id': 112247295260L, u'post_url': u'http://tsitra360.tumblr.com/post/112247295260/my-latest-art-timelapse-is-up-see-how-i-drew', u'tags': [], u'highlighted': [], u'state': u'published', u'html5_capable': True, u'type': u'video', u'format': u'html', u'timestamp': 1425068852, u'note_count': 79, u'video_type': u'youtube', u'date': u'2015-02-27 20:27:32 GMT', u'thumbnail_height': 360, u'permalink_url': u'https://www.youtube.com/watch?v=tT5pifkZzEk', u'slug': u'my-latest-art-timelapse-is-up-see-how-i-drew', u'blog_name': u'tsitra360', u'caption': u'<p>My latest art timelapse is up! See how I drew Berry Swirl on my youtube channel.</p>', u'thumbnail_url': u'https://i.ytimg.com/vi/tT5pifkZzEk/hqdefault.jpg'}
#save_media(session,tumblr_video_post_dict)
# Youtube
youtube_video_post_dict = {u'reblog_key': u'HfjckfH7', u'short_url': u'http://tmblr.co/ZUGffq1cfuHuJ', u'thumbnail_width': 480, u'player': [{u'width': 250, u'embed_code': u'<iframe width="250" height="140" id="youtube_iframe" src="https://www.youtube.com/embed/lGIEmH3BoyA?feature=oembed&enablejsapi=1&origin=http://safe.txmblr.com&wmode=opaque" frameborder="0" allowfullscreen></iframe>'}, {u'width': 400, u'embed_code': u'<iframe width="400" height="225" id="youtube_iframe" src="https://www.youtube.com/embed/lGIEmH3BoyA?feature=oembed&enablejsapi=1&origin=http://safe.txmblr.com&wmode=opaque" frameborder="0" allowfullscreen></iframe>'}, {u'width': 500, u'embed_code': u'<iframe width="500" height="281" id="youtube_iframe" src="https://www.youtube.com/embed/lGIEmH3BoyA?feature=oembed&enablejsapi=1&origin=http://safe.txmblr.com&wmode=opaque" frameborder="0" allowfullscreen></iframe>'}], u'id': 110224285203L, u'post_url': u'http://askbuttonsmom.tumblr.com/post/110224285203/throwback-can-you-believe-its-been-almost-2yrs', u'tags': [u"button's mom", u'hardcopy', u'song', u'shadyvox'], u'highlighted': [], u'state': u'published', u'html5_capable': True, u'type': u'video', u'format': u'html', u'timestamp': 1423197599, u'note_count': 145, u'video_type': u'youtube', u'date': u'2015-02-06 04:39:59 GMT', u'thumbnail_height': 360, u'permalink_url': u'https://www.youtube.com/watch?v=lGIEmH3BoyA', u'slug': u'throwback-can-you-believe-its-been-almost-2yrs', u'blog_name': u'askbuttonsmom', u'caption': u'<p>Throwback! Can you believe it’s been almost 2yrs since this came out? Mommy’s getting old…</p>', u'thumbnail_url': u'https://i.ytimg.com/vi/lGIEmH3BoyA/hqdefault.jpg'}
youtube_dict_two = {u'highlighted': [], u'reblog_key': u'qO3JnfS7', u'player': [{u'width': 250, u'embed_code': False}, {u'width': 400, u'embed_code': False}, {u'width': 500, u'embed_code': False}], u'format': u'html', u'timestamp': 1390412461, u'note_count': 4282, u'tags': [], u'video_type': u'youtube', u'id': 74184911379L, u'post_url': u'http://askbuttonsmom.tumblr.com/post/74184911379/ask-thecrusaders-bar-buddies-dont-worry', u'caption': u'<p><a class="tumblr_blog" href="http://ask-thecrusaders.tumblr.com/post/74162414750/bar-buddies-dont-worry-neon-you-will-have-your">ask-thecrusaders</a>:</p>\n<blockquote>\n<p><strong>"Bar Buddies"</strong><br/><br/>Dont\u2019 worry Neon, you will have your music video soon enough.</p>\n</blockquote>\n<p>Honestly, that Neon Lights is a TERRIBLE influence!! No son of mine will grow up to be a drunken drug-shooting bass dropping hipster! :C</p>', u'state': u'published', u'html5_capable': False, u'reblog': {u'comment': u'<p>Honestly, that Neon Lights is a TERRIBLE influence!! No son of mine will grow up to be a drunken drug-shooting bass dropping hipster! :C</p>', u'tree_html': u'<p><a class="tumblr_blog" href="http://ask-thecrusaders.tumblr.com/post/74162414750/bar-buddies-dont-worry-neon-you-will-have-your">ask-thecrusaders</a>:</p><blockquote>\n<p><strong>"Bar Buddies"</strong><br/><br/>Dont\u2019 worry Neon, you will have your music video soon enough.</p>\n</blockquote>', u'trail': [{u'blog': {u'theme': {u'title_font_weight': u'bold', u'title_color': u'#444444', u'header_bounds': 0, u'title_font': u'Helvetica Neue', u'link_color': u'#529ECC', u'header_image_focused': u'http://assets.tumblr.com/images/default_header/optica_pattern_04.png?_v=7c4e5e82cf797042596e2e64af1c383f', u'show_description': True, u'show_header_image': True, u'header_stretch': True, u'body_font': u'Helvetica Neue', u'show_title': True, u'header_image_scaled': u'http://assets.tumblr.com/images/default_header/optica_pattern_04.png?_v=7c4e5e82cf797042596e2e64af1c383f', u'avatar_shape': u'circle', u'show_avatar': True, u'background_color': u'#F6F6F6', u'header_image': u'http://assets.tumblr.com/images/default_header/optica_pattern_04.png?_v=7c4e5e82cf797042596e2e64af1c383f'}, u'name': u'ask-thecrusaders'}, u'comment': u'<p><strong>"Bar Buddies"</strong><br><br>Dont\u2019 worry Neon, you will have your music video soon enough.</p>', u'post': {u'id': u'74162414750'}}]}, u'short_url': u'http://tmblr.co/ZUGffq155m_eJ', u'date': u'2014-01-22 17:41:01 GMT', u'type': u'video', u'slug': u'ask-thecrusaders-bar-buddies-dont-worry', u'blog_name': u'askbuttonsmom'}
#yt_result_1 = save_media(session,youtube_video_post_dict)
#save_media(session,youtube_dict_two)
# Vine
vine_dict = {u'reblog_key': u'A5DhHt28', u'reblog': {u'comment': u'<p>Have a nice weekend, Tumblr. </p>', u'tree_html': u'', u'trail': []}, u'placement_id': u'{"i":"mF4avY6GyshXjaQmfk0v","v":4,"t":1427193020,"c":{"p":"113540981790","b":"staff"},"d":{"v":{"e":"hjWIUFOYD31"}},"h":"3291f1aa07"}', u'thumbnail_width': 480, u'player': [{u'width': 250, u'embed_code': u'<iframe class="vine-embed" src="https://vine.co/v/hjWIUFOYD31/embed/simple"width="250" height="250" frameborder="0"></iframe><script async src="//platform.vine.co/static/scripts/embed.js" charset="utf-8"></script>'}, {u'width': 400, u'embed_code': u'<iframe class="vine-embed" src="https://vine.co/v/hjWIUFOYD31/embed/simple"width="400" height="400" frameborder="0"></iframe><script async src="//platform.vine.co/static/scripts/embed.js" charset="utf-8"></script>'}, {u'width': 500, u'embed_code': u'<iframe class="vine-embed" src="https://vine.co/v/hjWIUFOYD31/embed/simple"width="500" height="500" frameborder="0"></iframe><script async src="//platform.vine.co/static/scripts/embed.js" charset="utf-8"></script>'}], u'id': 113540981790L, u'post_url': u'http://staff.tumblr.com/post/113540981790/have-a-nice-weekend-tumblr', u'source_title': u'weloveshortvideos.com', u'tags': [], u'highlighted': [], u'state': u'published', u'short_url': u'http://tmblr.co/ZE5Fby1flaUGU', u'html5_capable': True, u'type': u'video', u'format': u'html', u'timestamp': 1426282797, u'note_count': 48309, u'video_type': u'vine', u'source_url': u'http://weloveshortvideos.com', u'date': u'2015-03-13 21:39:57 GMT', u'thumbnail_height': 480, u'permalink_url': u'https://vine.co/v/hjWIUFOYD31', u'slug': u'have-a-nice-weekend-tumblr', u'blog_name': u'staff', u'caption': u'<p>Have a nice weekend, Tumblr.\xa0</p>', u'thumbnail_url': u'http://v.cdn.vine.co/r/thumbs/FE4C8DC8781008139866036658176_1c16044fdd3.3.4.mp4_l_pAXVyCckNVnk2OzdadqNB_6bq4mYoBHpBFRIF8Hi3OdOW1vmjP1TR075G1ZegT.jpg?versionId=abawWSw4Y_QFv2TKPWz6j8N5y7.6LOGq'}
#vine_restult = save_media(session,vine_dict)
# Vimeo
vimeo_dict = {u'reblog_key': u'3BuzwM1q', u'reblog': {u'comment': u'', u'tree_html': u'<p><a href="http://robscorner.tumblr.com/post/110250942998/a-hyperfast-preview-video-for-the-kind-of-content" class="tumblr_blog">robscorner</a>:</p><blockquote><p>A hyperfast preview video for the kind of content I\u2019m featuring on Patreon (patreon.com/robaato)! Slower version will be available for my supporters!<br/>MUSIC: The End (T.E.I.N. Pt. 2) | 12th Planet<br/></p><p>Support for high-resolution art, PSDs, process videos, tutorials, character requests, and more!<br/></p></blockquote>', u'trail': [{u'blog': {u'theme': {u'title_font_weight': u'bold', u'header_full_height': 1071, u'title_color': u'#FFFFFF', u'header_bounds': u'92,1581,978,3', u'title_font': u'Gibson', u'link_color': u'#529ECC', u'header_image_focused': u'http://static.tumblr.com/a5a733e78671519e8eb9cf3700ccfb70/ybimlef/1eon5zyi0/tumblr_static_tumblr_static_2df9bnxrqh1c4c8sgk8448s80_focused_v3.jpg', u'show_description': False, u'header_full_width': 1600, u'header_focus_width': 1578, u'header_stretch': True, u'show_header_image': True, u'body_font': u'Helvetica Neue', u'show_title': True, u'header_image_scaled': u'http://static.tumblr.com/cfa3addece89b58093ea0a8a87082653/ybimlef/FWyn5zyhv/tumblr_static_2df9bnxrqh1c4c8sgk8448s80_2048_v2.png', u'avatar_shape': u'square', u'show_avatar': True, u'header_focus_height': 886, u'background_color': u'#337db1', u'header_image': u'http://static.tumblr.com/cfa3addece89b58093ea0a8a87082653/ybimlef/FWyn5zyhv/tumblr_static_2df9bnxrqh1c4c8sgk8448s80.png'}, u'name': u'robscorner'}, u'comment': u'<p>A hyperfast preview video for the kind of content I\u2019m featuring on Patreon (patreon.com/robaato)! Slower version will be available for my supporters!<br>MUSIC: The End (T.E.I.N. Pt. 2) | 12th Planet<br></p><p>Support for high-resolution art, PSDs, process videos, tutorials, character requests, and more!<br></p>', u'post': {u'id': u'110250942998'}}]}, u'thumbnail_width': 295, u'player': [{u'width': 250, u'embed_code': u'<iframe src="https://player.vimeo.com/video/118912193?title=0&byline=0&portrait=0" width="250" height="156" frameborder="0" title="Hyperfast Preview - Mai (Patreon Process Videos)" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>'}, {u'width': 400, u'embed_code': u'<iframe src="https://player.vimeo.com/video/118912193?title=0&byline=0&portrait=0" width="400" height="250" frameborder="0" title="Hyperfast Preview - Mai (Patreon Process Videos)" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>'}, {u'width': 500, u'embed_code': u'<iframe src="https://player.vimeo.com/video/118912193?title=0&byline=0&portrait=0" width="500" height="312" frameborder="0" title="Hyperfast Preview - Mai (Patreon Process Videos)" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>'}], u'id': 110255840681, u'post_url': u'http://nsfw.kevinsano.com/post/110255840681/robscorner-a-hyperfast-preview-video-for-the-kind', u'source_title': u'robscorner', u'tags': [u'reblog', u'erohua'], u'highlighted': [], u'state': u'published', u'short_url': u'http://tmblr.co/Zo9zBq1chmfsf', u'html5_capable': True, u'type': u'video', u'format': u'html', u'timestamp': 1423238010, u'note_count': 415, u'video_type': u'vimeo', u'source_url': u'http://robscorner.tumblr.com/post/110250942998/a-hyperfast-preview-video-for-the-kind-of-content', u'date': u'2015-02-06 15:53:30 GMT', u'thumbnail_height': 184, u'permalink_url': u'https://vimeo.com/118912193', u'slug': u'robscorner-a-hyperfast-preview-video-for-the-kind', u'blog_name': u'nsfwkevinsano', u'caption': u'<p><a href="http://robscorner.tumblr.com/post/110250942998/a-hyperfast-preview-video-for-the-kind-of-content" class="tumblr_blog">robscorner</a>:</p><blockquote><p>A hyperfast preview video for the kind of content I\u2019m featuring on Patreon (patreon.com/robaato)! Slower version will be available for my supporters!<br/>MUSIC: The End (T.E.I.N. Pt. 2) | 12th Planet<br/></p><p>Support for high-resolution art, PSDs, process videos, tutorials, character requests, and more!<br/></p></blockquote>', u'thumbnail_url': u'https://i.vimeocdn.com/video/506047324_295x166.jpg'}
#save_media(session,vimeo_dict)
#video_handlers.handle_video_posts(session,vimeo_dict)
# Imgur video
imgur_post_dict = {u'highlighted': [], u'reblog_key': u'qX0EtplN', u'player': [{u'width': 250, u'embed_code': u'<iframe class="imgur-embed" width="100%" height="720" frameborder="0" src="http://i.imgur.com/wSBlRyv.gifv#embed"></iframe>'}, {u'width': 400, u'embed_code': u'<iframe class="imgur-embed" width="100%" height="720" frameborder="0" src="http://i.imgur.com/wSBlRyv.gifv#embed"></iframe>'}, {u'width': 500, u'embed_code': u'<iframe class="imgur-embed" width="100%" height="720" frameborder="0" src="http://i.imgur.com/wSBlRyv.gifv#embed"></iframe>'}], u'format': u'html', u'timestamp': 1415466120, u'note_count': 109, u'tags': [], u'thumbnail_width': 0, u'id': 102102282191, u'post_url': u'http://jessicaanner.tumblr.com/post/102102282191/front-view-clothed-large-version-gif-back', u'caption': u'<p><em><strong><a href="http://jessicaanner.tumblr.com/post/101601852991/front-view-clothed-large-version-gif-back">Front View (Clothed)</a> <a href="http://i.imgur.com/fDixfAC.gifv"><span class="auto_link" title="">(Large version)</span></a><a href="http://d.facdn.net/art/benezia/1414952655.benezia_front_armored_optimized.gif"><span class="auto_link" title=""> (GIF)</span></a></strong></em><br/><em><strong><a href="http://jessicaanner.tumblr.com/post/101666148721/front-view-clothed-large-version-gif-back">Back View (Clothed)</a> <a href="http://i.imgur.com/QYfRNeQ.gifv" title="">(Large version)</a> <a href="http://d.facdn.net/art/benezia/1415012804.benezia_back_armored_optimized.gif">(GIF)</a></strong></em><br/><em><strong><a href="http://jessicaanner.tumblr.com/post/101768307896/front-view-clothed-large-version-gif-back">Front View (Nude)</a> <a href="http://i.imgur.com/0N7ir7o.gifv">(Large version)</a> <a href="http://d.facdn.net/art/benezia/1415120393.benezia_front_nude_optimized.gif" title="">(GIF)</a></strong></em><br/><em><strong><a href="http://jessicaanner.tumblr.com/post/101852253284/front-view-clothed-large-version-gif-back">Back View (Nude)</a> <a href="http://i.imgur.com/sP5h9ux.gifv" title="">(Large version)</a> <a href="http://d.facdn.net/art/benezia/1415120590.benezia_back_nude_optimized.gif" title="">(GIF)</a></strong></em><br/><strong><em><a href="http://jessicaanner.tumblr.com/post/101934955336/front-view-clothed-large-version-gif-back">Buttocks Closeup View</a> <a href="http://i.imgur.com/BXMYuxk.gifv" title="">(Large version)</a> <a href="http://i.imgur.com/3bhzRP2.gif">(GIF)</a></em></strong><br/><em><strong><a href="http://jessicaanner.tumblr.com/post/102102282191/front-view-clothed-large-version-gif-back">Crotch Closeup View</a> <a href="http://i.imgur.com/wSBlRyv.gifv">(Large version)</a> <a href="http://i.imgur.com/UiDU1XB.gif">(GIF)</a></strong></em><br/><em><strong><a href="http://jessicaanner.tumblr.com/post/102017653601/front-view-clothed-large-version-gif-back">Bust Closeup View</a> <a href="http://i.imgur.com/S5M6PID.gifv">(Large version)</a> <a href="http://i.imgur.com/BlMYohP.gif">(GIF)</a></strong></em></p>', u'state': u'published', u'html5_capable': False, u'video_type': u'unknown', u'short_url': u'http://tmblr.co/ZLO7Om1V5nI-F', u'date': u'2014-11-08 17:02:00 GMT', u'thumbnail_height': 0, u'thumbnail_url': u'', u'type': u'video', u'slug': u'front-view-clothed-large-version-gif-back', u'blog_name': u'jessicaanner'}
#imgur_result = save_media(session,imgur_post_dict)
# Livestream
livestream_post_dict ={u'reblog_key': u'oapXWQlr', u'reblog': {u'comment': u'<p><span>To reiterate: this an </span><strong>only</strong><span> and an </span><strong>exclusive </strong><span>and it </span><strong>starts in just a few minutes</strong><span>. Hurry on over. </span></p>', u'tree_html': u'<p><a class="tumblr_blog" href="http://92y.tumblr.com/post/101031505431/watch-the-92y-livestream-of-game-of-thrones">92y</a>:</p><blockquote>\n<p>Watch the 92Y Livestream of <strong>Game of Thrones</strong> creator <strong>George R.R. Martin, TONIGHT at 8\xa0pm ET</strong>, in his <strong>only</strong> public U.S. appearance for the release of <a href="http://www.amazon.com/gp/product/B00EGMGGVK/ref=as_li_tl?ie=UTF8&camp=1789&creative=390957&creativeASIN=B00EGMGGVK&linkCode=as2&tag=92y-20&linkId=V3MMY57QIQ7QVFNK"><em>The World of Ice and Fire: The Untold History of Westeros and the Game of Thrones</em></a>. Exclusively on Tumblr!</p>\n</blockquote>', u'trail': [{u'blog': {u'theme': {u'title_font_weight': u'bold', u'title_color': u'#444444', u'header_bounds': u'', u'title_font': u'Gibson', u'link_color': u'#529ECC', u'header_image_focused': u'http://assets.tumblr.com/images/default_header/optica_pattern_13_focused_v3.png?_v=2f4063be1dd2ee91e4eca54332e25191', u'show_description': True, u'show_header_image': True, u'header_stretch': True, u'body_font': u'Helvetica Neue', u'show_title': True, u'header_image_scaled': u'http://assets.tumblr.com/images/default_header/optica_pattern_13_focused_v3.png?_v=2f4063be1dd2ee91e4eca54332e25191', u'avatar_shape': u'square', u'show_avatar': True, u'background_color': u'#FAFAFA', u'header_image': u'http://assets.tumblr.com/images/default_header/optica_pattern_13.png?_v=2f4063be1dd2ee91e4eca54332e25191'}, u'name': u'92y'}, u'comment': u'<p>Watch the 92Y Livestream of <strong>Game of Thrones</strong> creator <strong>George R.R. Martin, TONIGHT at 8\xa0pm ET</strong>, in his <strong>only</strong> public U.S. appearance for the release of <a href="http://www.amazon.com/gp/product/B00EGMGGVK/ref=as_li_tl?ie=UTF8&camp=1789&creative=390957&creativeASIN=B00EGMGGVK&linkCode=as2&tag=92y-20&linkId=V3MMY57QIQ7QVFNK"><em>The World of Ice and Fire: The Untold History of Westeros and the Game of Thrones</em></a>. Exclusively on Tumblr!</p>', u'post': {u'id': u'101031505431'}}]}, u'thumbnail_width': 0, u'player': [{u'width': 250, u'embed_code': u'<iframe src="http://new.livestream.com/accounts/1249127/events/3464519/player?width=560&height=315&autoPlay=true&mute=false" width="250" height="140" frameborder="0" scrolling="no"> </iframe>'}, {u'width': 400, u'embed_code': u'<iframe src="http://new.livestream.com/accounts/1249127/events/3464519/player?width=560&height=315&autoPlay=true&mute=false" width="400" height="225" frameborder="0" scrolling="no"> </iframe>'}, {u'width': 500, u'embed_code': u'<iframe src="http://new.livestream.com/accounts/1249127/events/3464519/player?width=560&height=315&autoPlay=true&mute=false" width="500" height="281" frameborder="0" scrolling="no"> </iframe>'}], u'id': 101038462325, u'post_url': u'http://staff.tumblr.com/post/101038462325/92y-watch-the-92y-livestream-of-game-of-thrones', u'tags': [], u'highlighted': [], u'state': u'published', u'short_url': u'http://tmblr.co/ZE5Fby1U6N9Lr', u'html5_capable': False, u'type': u'video', u'format': u'html', u'timestamp': 1414366397, u'note_count': 917, u'video_type': u'unknown', u'date': u'2014-10-26 23:33:17 GMT', u'thumbnail_height': 0, u'slug': u'92y-watch-the-92y-livestream-of-game-of-thrones', u'blog_name': u'staff', u'caption': u'<p><a class="tumblr_blog" href="http://92y.tumblr.com/post/101031505431/watch-the-92y-livestream-of-game-of-thrones">92y</a>:</p>\n<blockquote>\n<p>Watch the 92Y Livestream of <strong>Game of Thrones</strong> creator <strong>George R.R. Martin, TONIGHT at 8\xa0pm ET</strong>, in his <strong>only</strong> public U.S. appearance for the release of <a href="http://www.amazon.com/gp/product/B00EGMGGVK/ref=as_li_tl?ie=UTF8&camp=1789&creative=390957&creativeASIN=B00EGMGGVK&linkCode=as2&tag=92y-20&linkId=V3MMY57QIQ7QVFNK"><em>The World of Ice and Fire: The Untold History of Westeros and the Game of Thrones</em></a>. Exclusively on Tumblr!</p>\n</blockquote>\n<p><span>To reiterate: this an\xa0</span><strong>only</strong><span>\xa0and an\xa0</span><strong>exclusive\xa0</strong><span>and it\xa0</span><strong>starts in just a few minutes</strong><span>. Hurry on over. \xa0</span></p>', u'thumbnail_url': u''}
#livestream_result = save_media(session,livestream_post_dict)
# Yahoo video
yahoo_post_dict = {u'reblog_key': u'GGWw7A77', u'reblog': {u'comment': u'<p>It’s really happening!</p>', u'tree_html': u'<p><a class="tumblr_blog" href="http://whitehouse.tumblr.com/post/88396016693/obamairl">whitehouse</a>:</p><blockquote>\n<p>President Obama is answering your questions on education and college affordability in his first-ever Tumblr Q&A today.</p>\n<p>Tune in right here at 4 p.m. ET, and make sure to follow us @<a class="tumblelog" href="http://tmblr.co/mWgXp6TEB4GEsC_jKXfrSvw">whitehouse</a>.</p>\n</blockquote>', u'trail': [{u'blog': {u'theme': {u'title_font_weight': u'bold', u'header_full_height': 1056, u'title_color': u'#444444', u'header_bounds': u'43,1500,887,0', u'title_font': u'Gibson', u'link_color': u'#529ECC', u'header_image_focused': u'http://static.tumblr.com/861cd9f032b93a7ace681b4fcb7d05e4/mjqkjev/pEEn56435/tumblr_static_tumblr_static_17trsnvc8xes0og8kgk88coc0_focused_v3.jpg', u'show_description': True, u'header_full_width': 1500, u'header_focus_width': 1500, u'header_stretch': True, u'show_header_image': True, u'body_font': u'Helvetica Neue', u'show_title': True, u'header_image_scaled': u'http://static.tumblr.com/861cd9f032b93a7ace681b4fcb7d05e4/mjqkjev/sgln56432/tumblr_static_17trsnvc8xes0og8kgk88coc0_2048_v2.jpg', u'avatar_shape': u'square', u'show_avatar': True, u'header_focus_height': 844, u'background_color': u'#FAFAFA', u'header_image': u'http://static.tumblr.com/861cd9f032b93a7ace681b4fcb7d05e4/mjqkjev/sgln56432/tumblr_static_17trsnvc8xes0og8kgk88coc0.jpg'}, u'name': u'whitehouse'}, u'comment': u'<p>President Obama is answering your questions on education and college affordability in his first-ever Tumblr Q&A today.</p>\n<p>Tune in right here at 4 p.m. ET, and make sure to follow us @<a class="tumblelog" href="http://tmblr.co/mWgXp6TEB4GEsC_jKXfrSvw">whitehouse</a>.</p>', u'post': {u'id': u'88396016693'}}]}, u'thumbnail_width': 320, u'player': [{u'width': 250, u'embed_code': u'<iframe width="250" height="140" src="https://news.yahoo.com/video/tumblr-goes-white-house-190000218.html?format=embed" frameborder="0" allowfullscreen></iframe>'}, {u'width': 400, u'embed_code': u'<iframe width="400" height="225" src="https://news.yahoo.com/video/tumblr-goes-white-house-190000218.html?format=embed" frameborder="0" allowfullscreen></iframe>'}, {u'width': 500, u'embed_code': u'<iframe width="500" height="281" src="https://news.yahoo.com/video/tumblr-goes-white-house-190000218.html?format=embed" frameborder="0" allowfullscreen></iframe>'}], u'id': 88400573880, u'post_url': u'http://staff.tumblr.com/post/88400573880/whitehouse-president-obama-is-answering-your', u'tags': [u'ObamaIRL'], u'highlighted': [], u'state': u'published', u'short_url': u'http://tmblr.co/ZE5Fby1IL5RMu', u'html5_capable': True, u'type': u'video', u'format': u'html', u'timestamp': 1402430040, u'note_count': 9899, u'video_type': u'yahoo', u'date': u'2014-06-10 19:54:00 GMT', u'thumbnail_height': 180, u'permalink_url': u'https://news.yahoo.com/video/tumblr-goes-white-house-190000218.html', u'slug': u'whitehouse-president-obama-is-answering-your', u'blog_name': u'staff', u'caption': u'<p><a class="tumblr_blog" href="http://whitehouse.tumblr.com/post/88396016693/obamairl">whitehouse</a>:</p>\n<blockquote>\n<p>President Obama is answering your questions on education and college affordability in his first-ever Tumblr Q&A today.</p>\n<p>Tune in right here at 4 p.m. ET, and make sure to follow us @<a class="tumblelog" href="http://tmblr.co/mWgXp6TEB4GEsC_jKXfrSvw">whitehouse</a>.</p>\n</blockquote>\n<p>It\u2019s really happening!</p>', u'thumbnail_url': u'https://s1.yimg.com/uu/api/res/1.2/JW58D_.UFfRLkBOrIemIXw--/dz0zMjA7c209MTtmaT1maWxsO3B5b2ZmPTA7aD0xODA7YXBwaWQ9eXRhY2h5b24-/http://l.yimg.com/os/publish-images/ivy/2014-06-10/912811c0-f0c6-11e3-bb53-bd3ad1c7b3ec_06102014_tumblr_white_house.jpg'}
yahoo_result = save_media(session,yahoo_post_dict)
logging.debug("Closing DB session")
session.commit()
return
def debug():
"""Temp code for debug"""
session = sql_functions.connect_to_db()
read_post(
session,
post_url=
"http://askbuttonsmom.tumblr.com/post/118625666843/happy-mothers-day")
def reset_rows():
"""Reset processed flag to False"""
session = sql_functions.connect_to_db()
update_statement = sqlalchemy.update(RawPosts).where(RawPosts.media_processed == True).\
values(media_processed = False)
session.execute(update_statement)
session.commit()
return
def reset_rows():
"""Reset processed flag to False"""
session = sql_functions.connect_to_db()
update_statement = sqlalchemy.update(RawPosts).where(RawPosts.media_processed == True).\
values(media_processed = False)
session.execute(update_statement)
session.commit()
return
def debug():
"""For WIP, debug, ect function calls"""
session = sql_functions.connect_to_db()
result = download_image_links(session,
media_urls = ["http://blog.crooz.jp/svc/userFrontArticle/ShowFiles/?no=1538&blog_id=53800&file_str=5380015381862e669e0073d13d4175ecae9d5a34b8ff05fe3.jpg&guid=on&vga_flg=0&y=2014&m=02&d=19&wid=480&hei=640",
"http://static.tumblr.com/f6539f27dff5045834f7722e61c02e21/w5cnjnh/fVJnpuk9d/tumblr_static_digrsyxj0eg400so4g4kgsc4k.jpg",
"https://dl.dropboxusercontent.com/u/27379736/NSFWSFM/SWF/TwilightSoloLightParticles.swf"
]
)
logging.debug(result)
def delete_broken_posts():
"""Remove posts that do not have the processed flag set to true in the rawposts table"""
session = sql_functions.connect_to_db()
# -Posts with status mismatch-
# Each post row must have the same timestamp, blog url, and origin id as a rawposts row
# The rawposts row must have media_processed == True
# Choose a blog
blog_statement = sqlalchemy.select([blogs]).\
where(blogs.blog_id < 100)
blog_rows = session.execute(blog_statement)
blog_row = blog_rows.fetchone()
logging.debug(repr(blog_row))
blog_id = 1
# For each post from this blog
select_posts_statement = sqlalchemy.select([posts]).\
where(posts.blog_id == blog_id).\
limit(100)
post_rows = session.execute(select_posts_statement)
for post_row in post_rows:
logging.debug("post_row:" + repr(post_row))
post_id = post_row["post_id"]
# Grab the matching row from rawposts
compare_statement = sqlalchemy.select([RawPosts]).\
where(post_row.source_url == RawPosts.all_posts_post_url).\
where(post_row.timestamp == RawPosts.all_posts_timestamp).\
limit(10)
compare_rows = session.execute(compare_statement)
compare_row = compare_rows.fetchone()
logging.debug("compare_row:" + repr(compare_row))
# If post row exists but processed flag is not true, delete post
if compare_row["media_processed"] == False:
logging.debug("Bad")
# Delete post
delete_post_statement = sqlalchemy.select([posts]).\
where(posts.post_id == post_id)
session.execute(delete_post_statement)
continue
else:
logging.debug("Good")
continue
# Delete child rows
# Delete post row
# Commit changes
session.commit()
def main():
try:
setup_logging(log_file_path=os.path.join("debug",
"display_post-log.txt"),
concise_log_file_path=os.path.join(
"debug", "short-display_post-log.txt"))
# Program
session = sql_functions.connect_to_db()
build_all_posts(session)
## # Generate lists of what we have
## list_domain_posts(
## session,
## blog_domain = "askbuttonsmom.tumblr.com",
## output_path="debug\\list_posts.txt"
## )
##
## list_blogs(
## session,
## output_path="debug\\blog_list.txt"
## )
##
## list_all_media(
## session,
## output_path="debug\\media_list.txt"
## )
##
## # Test generating posts for display
##
## display_post(
## session,
## source_id = 117088487878L,
## output_path="debug\\post_117088487878L.txt",
## )
##
## display_post(
## session,
## source_id = 118863575131L,
## output_path="debug\\post_118863575131L.txt",
## )
##
## display_post(
## session,
## source_id = 118863575131,
## output_path="debug\\post_118863575131.txt",
## )
# /Program
logging.info("Finished, exiting.")
return
except Exception, e: # Log fatal exceptions
logging.critical("Unhandled exception!")
logging.exception(e)
def debug():
"""For WIP, debug, ect function calls"""
session = sql_functions.connect_to_db()
result = download_image_links(
session,
media_urls=[
"http://blog.crooz.jp/svc/userFrontArticle/ShowFiles/?no=1538&blog_id=53800&file_str=5380015381862e669e0073d13d4175ecae9d5a34b8ff05fe3.jpg&guid=on&vga_flg=0&y=2014&m=02&d=19&wid=480&hei=640",
"http://static.tumblr.com/f6539f27dff5045834f7722e61c02e21/w5cnjnh/fVJnpuk9d/tumblr_static_digrsyxj0eg400so4g4kgsc4k.jpg",
"https://dl.dropboxusercontent.com/u/27379736/NSFWSFM/SWF/TwilightSoloLightParticles.swf"
])
logging.debug(result)
def add_blogs(list_file_path="tumblr_todo_list.txt"):
"""Save tumblr blogs from a list"""
logging.info("Adding blogs from list file: " + repr(list_file_path))
# Get the list of blogs we want to add
blog_url_list = import_blog_list(list_file_path)
# Connect to DB
session = sql_functions.connect_to_db()
# Process each URL
for blog_url in blog_url_list:
add_blog(session, blog_url)
logging.info("Finished downloading blogs list")
return
def debug():
"""Temp code for debug"""
session = sql_functions.connect_to_db()
blog_urls = [
#"staff.tumblr.com",
#"testsetsts2.tumblr.com",
"lunarshinestore.tumblr.com",
"atryl.tumblr.com",
]
for blog_url in blog_urls:
dummy_blog_id = sql_functions.add_blog(session,blog_url)
logging.debug("dummy_blog_id: "+repr(dummy_blog_id))
update_blog_theme(session,blog_url)
continue
return
def add_blogs(list_file_path="tumblr_todo_list.txt"):
"""Save tumblr blogs from a list"""
logging.info("Adding blogs from list file: "+repr(list_file_path))
# Get the list of blogs we want to add
blog_url_list = import_blog_list(list_file_path)
# Connect to DB
session = sql_functions.connect_to_db()
# Process each URL
for blog_url in blog_url_list:
add_blog(session,blog_url)
logging.info("Finished downloading blogs list")
return
def debug():
"""Temp code for debug"""
session = sql_functions.connect_to_db()
blog_urls = [
#"staff.tumblr.com",
#"testsetsts2.tumblr.com",
"lunarshinestore.tumblr.com",
"atryl.tumblr.com",
]
for blog_url in blog_urls:
dummy_blog_id = sql_functions.add_blog(session, blog_url)
logging.debug("dummy_blog_id: " + repr(dummy_blog_id))
update_blog_theme(session, blog_url)
continue
return
def main():
try:
setup_logging(log_file_path=os.path.join("debug","asagi_vocaroo_grabber-log.txt"))
session = sql_functions.connect_to_db()
scan_db(
session=session,
output_dir = config.output_dir,
start_id = config.start_id,
stop_id = config.stop_id,
step_number = config.step_number
)
except Exception, e:# Log fatal exceptions
logging.critical("Unhandled exception!")
logging.exception(e)
def process_posts(blog_domain):
"""Find all posts for a given blog"""
# Connect to the DB
session = sql_functions.connect_to_db()
# Grab the posts
posts_query = sqlalchemy.select([RawPosts]).\
where(RawPosts.blog_domain == blog_domain)
posts_rows = session.execute(posts_query)
found_blog_links = []
# Process each post
counter = 0
for posts_row in posts_rows:
counter += 1
logging.debug("Processing post "+repr(counter))
# Grab post data from DB
post_dict = posts_row["raw_post_json"]
# Stringify post data
post_string = repr(post_dict)
# Parse out links
blog_links_in_post = find_blog_links(post_string)
logging.debug("blog_links_in_post: "+repr(blog_links_in_post))
# Append found blogs to list
found_blog_links += blog_links_in_post
# Clean up links
logging.debug("found_blog_links: "+repr(found_blog_links))
kept_blog_links = remove_unwanted_links(found_blog_links)
logging.debug("kept_blog_links: "+repr(kept_blog_links))
output_list = uniquify(kept_blog_links)
logging.debug("output_list: "+repr(output_list))
timestamp_string = datetime.datetime.utcnow().strftime("%Y-%m-%d %H.%M.%S%Z")
# Save found links to file
appendlist(
output_list,
list_file_path="linked_blogs_"+blog_domain+"_"+timestamp_string+".txt",
initial_text="# List of found blog_links.\n"
)
return
def post_consumer(post_queue):
"Process posts for a queue-like object containing post dicts"
logging.debug("Consumer function started.")
# Connect to DB
database_session = sql_functions.connect_to_db()
# Process posts from the queue
c = 0 # Counter for number of posts processed
while True:
c += 1
if c % 100 == 0:
logging.info(repr(c) + " posts processed by this process")
post_row = post_queue.get(timeout=600)
if post_row is None: # Stop if None object is put into the queue
logging.info("Post consumer recieved None object as exit signal")
break # Stop doing work and exit thread/process
process_one_new_posts_media(database_session, post_row)
continue
# Disconnect from DB
database_session.close()
logging.debug("Consumer function exiting.")
return
def post_consumer(post_queue):
"Process posts for a queue-like object containing post dicts"
logging.debug("Consumer function started.")
# Connect to DB
database_session = sql_functions.connect_to_db()
# Process posts from the queue
c = 0# Counter for number of posts processed
while True:
c += 1
if c%100 == 0:
logging.info(repr(c)+" posts processed by this process")
post_row = post_queue.get(timeout=600)
if post_row is None:# Stop if None object is put into the queue
logging.info("Post consumer recieved None object as exit signal")
break# Stop doing work and exit thread/process
process_one_new_posts_media(database_session,post_row)
continue
# Disconnect from DB
database_session.close()
logging.debug("Consumer function exiting.")
return
def list_blogs():
"""Return a list of up to maximum_blogs blogids"""
# Connect to DB
session = sql_functions.connect_to_db()
# Load new blogs
# date_posts_last_saved NULL
select_null_query = sqlalchemy.select([blogs]).\
where(blogs.date_posts_last_saved == sqlalchemy.null()).\
order_by(blogs.date_posts_last_saved.asc())
null_blogs_rows = session.execute(select_null_query)
# Grab blog URLs
null_blog_urls = []
for null_blogs_row in null_blogs_rows:
null_blog_url = null_blogs_row["blog_url"]
null_blog_urls += [null_blog_url]
logging.debug("len(null_blog_urls):"+repr(len(null_blog_urls)))
# date_posts_last_saved NOT NULL
select_query = sqlalchemy.select([blogs]).\
where(blogs.date_posts_last_saved != sqlalchemy.null()).\
order_by(blogs.date_posts_last_saved.asc())
not_null_blogs_rows = session.execute(select_query)
# Grab blog URLs
not_null_blog_urls = []
for not_null_blogs_row in not_null_blogs_rows:
not_null_blog_url = not_null_blogs_row["blog_url"]
not_null_blog_urls += [not_null_blog_url]
logging.debug("len(not_null_blog_urls):"+repr(len(not_null_blog_urls)))
# Disconnect from DB
session.close()
blog_urls = null_blog_urls+not_null_blog_urls
return blog_urls
def debug():
"""where stuff is called to debug and test"""
## save_vocaroo_link(
## vocaroo_link="http://vocaroo.com/i/s0xtktsit8rE",
## output_dir=os.path.join("debug", "output"),
## )
session = sql_functions.connect_to_db()
## find_vocaroo_links_in_db(
## session=session,
## start_id=0,
## stop_id=1000,
## output_dir=os.path.join("debug", "output")
## )
scan_db(
session=session,
output_dir=os.path.join("output"),
start_id=0,
stop_id=None,
step_number=1000
)
return
def debug():
"""Temp code for debug"""
session = sql_functions.connect_to_db()
read_post(session,post_url="http://askbuttonsmom.tumblr.com/post/118625666843/happy-mothers-day")
def drop_tables():
"""Drop all tables except for media and rawposts"""
session = sql_functions.connect_to_db()
Base.metadata.drop_all()
def delete_broken_posts():
"""Remove posts that do not have the processed flag set to true in the rawposts table"""
session = sql_functions.connect_to_db()
# -Posts with status mismatch-
# Each post row must have the same timestamp, blog url, and origin id as a rawposts row
# The rawposts row must have media_processed == True
# Choose a blog
blog_statement = sqlalchemy.select([blogs]).\
where(blogs.blog_id < 100)
blog_rows = session.execute(blog_statement)
blog_row = blog_rows.fetchone()
logging.debug(repr(blog_row))
blog_id = 1
# For each post from this blog
select_posts_statement = sqlalchemy.select([posts]).\
where(posts.blog_id == blog_id).\
limit(100)
post_rows = session.execute(select_posts_statement)
for post_row in post_rows:
logging.debug("post_row:"+repr(post_row))
post_id = post_row["post_id"]
# Grab the matching row from rawposts
compare_statement = sqlalchemy.select([RawPosts]).\
where(post_row.source_url == RawPosts.all_posts_post_url).\
where(post_row.timestamp == RawPosts.all_posts_timestamp).\
limit(10)
compare_rows = session.execute(compare_statement)
compare_row = compare_rows.fetchone()
logging.debug("compare_row:"+repr(compare_row))
# If post row exists but processed flag is not true, delete post
if compare_row["media_processed"] == False:
logging.debug("Bad")
# Delete post
delete_post_statement = sqlalchemy.select([posts]).\
where(posts.post_id == post_id)
session.execute(delete_post_statement)
continue
else:
logging.debug("Good")
continue
# Delete child rows
# Delete post row
# Commit changes
session.commit()
def main():
session = sql_functions.connect_to_db()
dump_posts(
session,
output_file_path=os.path.join("debug","raw_posts_dump.txt")
)
def debug():
"""For WIP, debug, ect function calls"""
session = sql_functions.connect_to_db()
postimg_result = save_postimg_gallery(
session, link="http://postimg.org/gallery/1g80elqce/")
logging.debug("postimg_result")
return
#handle_generic_link(session,link="media.tumblr.com/80638dc40978c286b5d3f18b2bfcf3d6/tumblr_inline_mfb0pxs7lf1qfi5b1.gif")
#fastswf_result = handle_fastswf_link(session,"http://www.fastswf.com/5MH9MZw")
#return
## result = handle_links(
## session,
## post_dict = {u'highlighted': [], u'asking_url': None, u'reblog_key': u'O3UPhQYC', u'format': u'html', u'asking_name': u'Anonymous', u'timestamp': 1430518955, u'note_count': 43253, u'tags': [], u'question': u'lauren will you tell us a story', u'trail': [{u'blog': {u'theme': {u'title_font_weight': u'bold', u'header_full_height': 1226, u'title_color': u'#8f1e03', u'header_bounds': u'87,647,433,31', u'title_font': u'Courier New', u'link_color': u'#ffdd00', u'header_image_focused': u'http://static.tumblr.com/cafca27bb4f69629bf936818efb44e6f/88apzkd/fEXnhmx2i/tumblr_static_tumblr_static_983gvr4ppg4cgkgcs8c8owk4o_focused_v3.png', u'show_description': True, u'header_full_width': 677, u'header_focus_width': 616, u'header_stretch': True, u'show_header_image': True, u'body_font': u'Helvetica Neue', u'show_title': True, u'header_image_scaled': u'http://static.tumblr.com/cafca27bb4f69629bf936818efb44e6f/88apzkd/h29nhmx2f/tumblr_static_983gvr4ppg4cgkgcs8c8owk4o_2048_v2.png', u'avatar_shape': u'circle', u'show_avatar': True, u'header_focus_height': 346, u'background_color': u'#f38321', u'header_image': u'http://static.tumblr.com/cafca27bb4f69629bf936818efb44e6f/88apzkd/h29nhmx2f/tumblr_static_983gvr4ppg4cgkgcs8c8owk4o.png'}, u'name': u'iguanamouth'}, u'content': u'<p>ok heres something</p><p>around a year ago someone asked me to draw <a href="http://iguanamouth.tumblr.com/post/82902281208/draw-danny-devito-as-a-kitty">danny devito as a kitty</a>, spawning this terrible terrible image\xa0</p><figure data-orig-width="500" data-orig-height="406" class="tmblr-full"><img src="https://41.media.tumblr.com/d7549754fd8c427d3393eb702939652f/tumblr_inline_nnld4fWtJH1qmoaae_540.png" alt="image" data-orig-width="500" data-orig-height="406"></figure><p>time passes. a lot of time passes. then two months ago i get an email from a group of people called <a href="https://www.facebook.com/fpoafm">FPOAFM</a> doing a pottery installation event, and theyre going around gathering artwork from artists to put onto cups and dishes to sell,, in exchange for a few pieces with the artists work on them</p><p>and i said SURE you can use some of my stuff \u2026 . but in exchange.\u2026 <b>\xa0i want something with kitty devito on it. </b>i dont care if you put it on anything else, but one item that i get in return has to have this cat man abomination</p><p>i give them my address and a few images and months pass. i forget about it. THEN literally two days ago i get this big package on my doorstep, and INSIDE OF IT\u2026. is the holy grail</p><p>in addition to <a href="https://40.media.tumblr.com/cdd10d787d5bd9c4ab797fca37671cb0/tumblr_nneimxvAoS1rhtthso2_r3_1280.png">two</a> <a href="https://40.media.tumblr.com/7e01bf6e04497f0c9da80ed0b83cc3df/tumblr_nneimxvAoS1rhtthso3_r2_540.png">plates</a> is this incredible porcelain cup with the fabled kitty devito on it, proudly grinning his terrible cat grin</p><figure data-orig-width="800" data-orig-height="885" class="tmblr-full"><img src="https://41.media.tumblr.com/aca0ef3ca89612a513ef9143a439b48d/tumblr_inline_nnldwbZNR91qmoaae_540.png" alt="image" data-orig-width="800" data-orig-height="885"></figure><p>the thing that pushes this cup into the Far Reaches of Awful isnt just the image stamped on it. its that it is one hundred percent made from a mold of a styrofoam cup</p><figure data-orig-width="800" data-orig-height="600" class="tmblr-full"><img src="https://40.media.tumblr.com/7ce202e1b1877d34c0211876cdbb575e/tumblr_inline_nnleakjIpO1qmoaae_540.png" alt="image" data-orig-width="800" data-orig-height="600"></figure><p>its finger presses on the rim, those little lines going around</p><figure data-orig-width="800" data-orig-height="600" class="tmblr-full"><img src="https://36.media.tumblr.com/75f6a2b06915bac6aec356d18f226a47/tumblr_inline_nnleefM9Ty1qmoaae_540.png" alt="image" data-orig-width="800" data-orig-height="600"></figure><p>and all this jargon on the bottom, right under the glaze. the amount of effort that went into reproducing this styrofoam cup is incredible and i can stick it in my shelf and drink soup from it at four in the morning with danny devitos smug cat face looking out over everything i do, forever. follow your dreams</p>', u'post': {u'id': u'117728731307'}, u'is_root_item': True}, {u'blog': {u'theme': {u'title_font_weight': u'bold', u'title_color': u'#FFFFFF', u'header_bounds': 0, u'title_font': u'Gibson', u'link_color': u'#56BC8A', u'header_image_focused': u'http://static.tumblr.com/10e607f9b9b4c588d1cbd6c9f8b564d9/3hpyv0p/nFBnlgttl/tumblr_static_1sssiwavcjs0gs4cggwsok040_2048_v2.gif', u'show_description': True, u'show_header_image': True, u'header_stretch': True, u'body_font': u'Helvetica Neue', u'show_title': True, u'header_image_scaled': u'http://static.tumblr.com/10e607f9b9b4c588d1cbd6c9f8b564d9/3hpyv0p/nFBnlgttl/tumblr_static_1sssiwavcjs0gs4cggwsok040_2048_v2.gif', u'avatar_shape': u'square', u'show_avatar': False, u'background_color': u'#37475c', u'header_image': u'http://static.tumblr.com/10e607f9b9b4c588d1cbd6c9f8b564d9/3hpyv0p/nFBnlgttl/tumblr_static_1sssiwavcjs0gs4cggwsok040.gif'}, u'name': u'staff'}, u'content': u'<p>Have a follow-your-dreams weekend, Tumblr.</p>', u'post': {u'id': u'117887933455'}, u'is_current_item': True}], u'id': 117887933455L, u'post_url': u'http://staff.tumblr.com/post/117887933455/lauren-will-you-tell-us-a-story', u'answer': u'<p><a class="tumblr_blog" href="http://iguanamouth.tumblr.com/post/117728731307/lauren-will-you-tell-us-a-story">iguanamouth</a>:</p>\n\n<blockquote><p>ok heres something</p><p>around a year ago someone asked me to draw <a href="http://iguanamouth.tumblr.com/post/82902281208/draw-danny-devito-as-a-kitty">danny devito as a kitty</a>, spawning this terrible terrible image\xa0</p><figure data-orig-width="500" data-orig-height="406" class="tmblr-full"><img src="https://41.media.tumblr.com/d7549754fd8c427d3393eb702939652f/tumblr_inline_nnld4fWtJH1qmoaae_540.png" alt="image" data-orig-width="500" data-orig-height="406"/></figure><p>time passes. a lot of time passes. then two months ago i get an email from a group of people called <a href="https://www.facebook.com/fpoafm">FPOAFM</a> doing a pottery installation event, and theyre going around gathering artwork from artists to put onto cups and dishes to sell,, in exchange for a few pieces with the artists work on them</p><p>and i said SURE you can use some of my stuff \u2026 . but in exchange.\u2026 <b>\xa0i want something with kitty devito on it. </b>i dont care if you put it on anything else, but one item that i get in return has to have this cat man abomination</p><p>i give them my address and a few images and months pass. i forget about it. THEN literally two days ago i get this big package on my doorstep, and INSIDE OF IT\u2026. is the holy grail</p><p>in addition to <a href="https://40.media.tumblr.com/cdd10d787d5bd9c4ab797fca37671cb0/tumblr_nneimxvAoS1rhtthso2_r3_1280.png">two</a> <a href="https://40.media.tumblr.com/7e01bf6e04497f0c9da80ed0b83cc3df/tumblr_nneimxvAoS1rhtthso3_r2_540.png">plates</a> is this incredible porcelain cup with the fabled kitty devito on it, proudly grinning his terrible cat grin</p><figure data-orig-width="800" data-orig-height="885" class="tmblr-full"><img src="https://41.media.tumblr.com/aca0ef3ca89612a513ef9143a439b48d/tumblr_inline_nnldwbZNR91qmoaae_540.png" alt="image" data-orig-width="800" data-orig-height="885"/></figure><p>the thing that pushes this cup into the Far Reaches of Awful isnt just the image stamped on it. its that it is one hundred percent made from a mold of a styrofoam cup</p><figure data-orig-width="800" data-orig-height="600" class="tmblr-full"><img src="https://40.media.tumblr.com/7ce202e1b1877d34c0211876cdbb575e/tumblr_inline_nnleakjIpO1qmoaae_540.png" alt="image" data-orig-width="800" data-orig-height="600"/></figure><p>its finger presses on the rim, those little lines going around</p><figure data-orig-width="800" data-orig-height="600" class="tmblr-full"><img src="https://36.media.tumblr.com/75f6a2b06915bac6aec356d18f226a47/tumblr_inline_nnleefM9Ty1qmoaae_540.png" alt="image" data-orig-width="800" data-orig-height="600"/></figure><p>and all this jargon on the bottom, right under the glaze. the amount of effort that went into reproducing this styrofoam cup is incredible and i can stick it in my shelf and drink soup from it at four in the morning with danny devitos smug cat face looking out over everything i do, forever. follow your dreams</p></blockquote><p><p>Have a follow-your-dreams weekend, Tumblr.</p></p>', u'state': u'published', u'reblog': {u'tree_html': u'<p><a class="tumblr_blog" href="http://iguanamouth.tumblr.com/post/117728731307/lauren-will-you-tell-us-a-story">iguanamouth</a>:</p><blockquote><p>ok heres something</p><p>around a year ago someone asked me to draw <a href="http://iguanamouth.tumblr.com/post/82902281208/draw-danny-devito-as-a-kitty">danny devito as a kitty</a>, spawning this terrible terrible image\xa0</p><figure data-orig-width="500" data-orig-height="406" class="tmblr-full"><img src="https://41.media.tumblr.com/d7549754fd8c427d3393eb702939652f/tumblr_inline_nnld4fWtJH1qmoaae_540.png" alt="image" data-orig-width="500" data-orig-height="406"/></figure><p>time passes. a lot of time passes. then two months ago i get an email from a group of people called <a href="https://www.facebook.com/fpoafm">FPOAFM</a> doing a pottery installation event, and theyre going around gathering artwork from artists to put onto cups and dishes to sell,, in exchange for a few pieces with the artists work on them</p><p>and i said SURE you can use some of my stuff \u2026 . but in exchange.\u2026 <b>\xa0i want something with kitty devito on it. </b>i dont care if you put it on anything else, but one item that i get in return has to have this cat man abomination</p><p>i give them my address and a few images and months pass. i forget about it. THEN literally two days ago i get this big package on my doorstep, and INSIDE OF IT\u2026. is the holy grail</p><p>in addition to <a href="https://40.media.tumblr.com/cdd10d787d5bd9c4ab797fca37671cb0/tumblr_nneimxvAoS1rhtthso2_r3_1280.png">two</a> <a href="https://40.media.tumblr.com/7e01bf6e04497f0c9da80ed0b83cc3df/tumblr_nneimxvAoS1rhtthso3_r2_540.png">plates</a> is this incredible porcelain cup with the fabled kitty devito on it, proudly grinning his terrible cat grin</p><figure data-orig-width="800" data-orig-height="885" class="tmblr-full"><img src="https://41.media.tumblr.com/aca0ef3ca89612a513ef9143a439b48d/tumblr_inline_nnldwbZNR91qmoaae_540.png" alt="image" data-orig-width="800" data-orig-height="885"/></figure><p>the thing that pushes this cup into the Far Reaches of Awful isnt just the image stamped on it. its that it is one hundred percent made from a mold of a styrofoam cup</p><figure data-orig-width="800" data-orig-height="600" class="tmblr-full"><img src="https://40.media.tumblr.com/7ce202e1b1877d34c0211876cdbb575e/tumblr_inline_nnleakjIpO1qmoaae_540.png" alt="image" data-orig-width="800" data-orig-height="600"/></figure><p>its finger presses on the rim, those little lines going around</p><figure data-orig-width="800" data-orig-height="600" class="tmblr-full"><img src="https://36.media.tumblr.com/75f6a2b06915bac6aec356d18f226a47/tumblr_inline_nnleefM9Ty1qmoaae_540.png" alt="image" data-orig-width="800" data-orig-height="600"/></figure><p>and all this jargon on the bottom, right under the glaze. the amount of effort that went into reproducing this styrofoam cup is incredible and i can stick it in my shelf and drink soup from it at four in the morning with danny devitos smug cat face looking out over everything i do, forever. follow your dreams</p></blockquote>'}, u'short_url': u'http://tmblr.co/ZE5Fby1jognmF', u'date': u'2015-05-01 22:22:35 GMT', u'type': u'answer', u'slug': u'lauren-will-you-tell-us-a-story', u'blog_name': u'staff'}
## )
## logging.debug("result:"+repr(result))
# Dropbox error case
dropboxusercontent_post_dict = {
u'reblog_key':
u'w7hVL658',
u'reblog': {
u'comment':
u'',
u'tree_html':
u'<p><a href="http://verysaltyonions.tumblr.com/post/82259171463/fruitymilkstuff-im-terrible-at-being-lazy-so" class="tumblr_blog" target="_blank">verysaltyonions</a>:</p><blockquote><p><a href="http://fruitymilk.co.uk/post/81236468626/im-terrible-at-being-lazy-so-much-for-my-no" class="tumblr_blog" target="_blank">fruitymilkstuff</a>:</p>\n\n<blockquote><p><em><strong>I\u2019m terrible at being lazy.</strong></em>\xa0So much for my \u2018no content\u2019 weekend.</p>\n<p>I used some time last night to experiment with particles and lights coming off said particles. Magic in this case. Nothing spectacular though!</p>\n<p><em>I was testing out particles some more, trying to figure out how to make them loop and stuff. Turns out I can manage it! That and lighting to match them.\xa0</em></p>\n<p><strong><em>As usual click-through for a MUCH higher quality SWF file!</em></strong></p></blockquote>\n\n<p>Unusual futa-c**k</p></blockquote>'
},
u'id':
82654349960L,
u'post_url':
u'http://aegissp.tumblr.com/post/82654349960/verysaltyonions-fruitymilkstuff-im-terrible',
u'source_title':
u'fruitymilkstuff',
u'image_permalink':
u'http://aegissp.tumblr.com/image/82654349960',
u'tags': [
u'NSFW', u'nsfw:twilight sparkle', u'futa', u'futanari',
u'rule 34', u'clop', u'media'
],
u'highlighted': [],
u'recommended_source':
None,
u'state':
u'published',
u'short_url':
u'http://tmblr.co/ZeuQFu1C_bKg8',
u'type':
u'photo',
u'format':
u'html',
u'timestamp':
1397446549,
u'note_count':
348,
u'source_url':
u'http://fruitymilkstuff.tumblr.com/post/81236468626/im-terrible-at-being-lazy-so-much-for-my-no',
u'photos': [{
u'caption':
u'',
u'original_size': {
u'url':
u'http://33.media.tumblr.com/894426e6f4f1b90b330e986abc26bb00/tumblr_n39x44yu4P1tpeknfo1_500.gif',
u'width': 445,
u'height': 250
},
u'alt_sizes': [{
u'url':
u'http://33.media.tumblr.com/894426e6f4f1b90b330e986abc26bb00/tumblr_n39x44yu4P1tpeknfo1_500.gif',
u'width': 445,
u'height': 250
}, {
u'url':
u'http://33.media.tumblr.com/894426e6f4f1b90b330e986abc26bb00/tumblr_n39x44yu4P1tpeknfo1_400.gif',
u'width': 400,
u'height': 225
}, {
u'url':
u'http://38.media.tumblr.com/894426e6f4f1b90b330e986abc26bb00/tumblr_n39x44yu4P1tpeknfo1_250.gif',
u'width': 250,
u'height': 140
}, {
u'url':
u'http://33.media.tumblr.com/894426e6f4f1b90b330e986abc26bb00/tumblr_n39x44yu4P1tpeknfo1_100.gif',
u'width': 100,
u'height': 56
}, {
u'url':
u'http://33.media.tumblr.com/894426e6f4f1b90b330e986abc26bb00/tumblr_n39x44yu4P1tpeknfo1_75sq.gif',
u'width': 75,
u'height': 75
}]
}],
u'date':
u'2014-04-14 03:35:49 GMT',
u'slug':
u'verysaltyonions-fruitymilkstuff-im-terrible',
u'blog_name':
u'aegissp',
u'trail': [{
u'blog': {
u'active': True,
u'theme': {
u'title_font_weight':
u'bold',
u'title_color':
u'#FFFFFF',
u'header_bounds':
0,
u'title_font':
u'Streetscript',
u'link_color':
u'#529ECC',
u'header_image_focused':
u'http://static.tumblr.com/ef1069245069af066df2ac8411f241c9/4it4dnn/Ipwnk5eq8/tumblr_static_8cqn4964iow0ok8gw80c4cwg8_2048_v2.png',
u'show_description':
True,
u'show_header_image':
True,
u'body_font':
u'Helvetica Neue',
u'show_title':
True,
u'header_stretch':
False,
u'avatar_shape':
u'circle',
u'show_avatar':
True,
u'background_color':
u'#b01e1e',
u'header_image':
u'http://static.tumblr.com/ef1069245069af066df2ac8411f241c9/4it4dnn/Ipwnk5eq8/tumblr_static_8cqn4964iow0ok8gw80c4cwg8.png',
u'header_image_scaled':
u'http://static.tumblr.com/ef1069245069af066df2ac8411f241c9/4it4dnn/Ipwnk5eq8/tumblr_static_8cqn4964iow0ok8gw80c4cwg8_2048_v2.png'
},
u'name': u'fruitymilkstuff'
},
u'content':
u'<p><em><strong>I\u2019m terrible at being lazy.</strong></em>\xa0So much for my \u2018no content\u2019 weekend.</p>\n<p>I used some time last night to experiment with particles and lights coming off said particles. Magic in this case. Nothing spectacular though!</p>\n<p><em>I was testing out particles some more, trying to figure out how to make them loop and stuff. Turns out I can manage it! That and lighting to match them.\xa0</em></p>\n<p><strong><em>As usual click-through for a MUCH higher quality SWF file!</em></strong></p>',
u'post': {
u'id': u'81236468626'
},
u'is_root_item': True
}, {
u'blog': {
u'active': True,
u'theme': {
u'title_font_weight':
u'bold',
u'title_color':
u'#444444',
u'header_bounds':
u'',
u'title_font':
u'Helvetica Neue',
u'link_color':
u'#FFA200',
u'header_image_focused':
u'http://assets.tumblr.com/images/default_header/optica_pattern_08_focused_v3.png?_v=f0f055039bb6136b9661cf2227b535c2',
u'show_description':
True,
u'show_header_image':
True,
u'body_font':
u'Helvetica Neue',
u'show_title':
True,
u'header_stretch':
True,
u'avatar_shape':
u'circle',
u'show_avatar':
True,
u'background_color':
u'#ffffff',
u'header_image':
u'http://assets.tumblr.com/images/default_header/optica_pattern_08.png?_v=f0f055039bb6136b9661cf2227b535c2',
u'header_image_scaled':
u'http://assets.tumblr.com/images/default_header/optica_pattern_08_focused_v3.png?_v=f0f055039bb6136b9661cf2227b535c2'
},
u'name': u'verysaltyonions'
},
u'content': u'<p>Unusual futa-c**k</p>',
u'post': {
u'id': u'82259171463'
}
}],
u'link_url':
u'https://dl.dropboxusercontent.com/u/27379736/NSFWSFM/SWF/TwilightSoloLightParticles.swf',
u'caption':
u'<p><a href="http://verysaltyonions.tumblr.com/post/82259171463/fruitymilkstuff-im-terrible-at-being-lazy-so" class="tumblr_blog" target="_blank">verysaltyonions</a>:</p>\n\n<blockquote><p><a href="http://fruitymilk.co.uk/post/81236468626/im-terrible-at-being-lazy-so-much-for-my-no" class="tumblr_blog" target="_blank">fruitymilkstuff</a>:</p>\n\n<blockquote><p><em><strong>I\u2019m terrible at being lazy.</strong></em>\xa0So much for my \u2018no content\u2019 weekend.</p>\n<p>I used some time last night to experiment with particles and lights coming off said particles. Magic in this case. Nothing spectacular though!</p>\n<p><em>I was testing out particles some more, trying to figure out how to make them loop and stuff. Turns out I can manage it! That and lighting to match them.\xa0</em></p>\n<p><strong><em>As usual click-through for a MUCH higher quality SWF file!</em></strong></p></blockquote>\n\n<p>Unusual futa-c**k</p></blockquote>'
}
dropboxusercontent_result = handle_links(
session, post_dict=dropboxusercontent_post_dict)
logging.debug("dropboxusercontent_result:" +
repr(dropboxusercontent_result))
# malformed link case
malformed_link_post_dict = {
u'reblog_key':
u'wlQEgKpb',
u'reblog': {
u'comment':
u'',
u'tree_html':
u'<p><a href="http://taboolicious.tumblr.com/post/118601139146/happy-milf-day-many-thanks-for-all-those-that" class="tumblr_blog">taboolicious</a>:</p><blockquote><p>happy milf day! many thanks for all those that where present in the raffle and suggested such hot milfs :D</p><p>here is the wallpaper: 1920x1200</p><p><a href="https://dl.dropboxusercontent.com/u/1834642/milf%20paper.jpg">https://dl.dropboxusercontent.com/u/1834642/milf%20paper.jpg</a></p><p>from left to right:</p><p>bananacreamcakecomic: Jenna OC</p><p>jacquesthebird: Kate OC</p><p>damabiath: android 18</p><p>bellend08: Molly Cosset OC</p><p>vic79: Mystique</p><p>ravenlordmercenary: Grace pokemon X/Y</p><p>Th3A1chemist: Vuong OC</p><p>Evenanthy: Harley Quinn</p><p><br/></p><p>if any of the winners wants a wallpaper with just his/her requested girl, pm me :)</p></blockquote>'
},
u'id':
118615261815L,
u'post_url':
u'http://dclzexonask.tumblr.com/post/118615261815/taboolicious-happy-milf-day-many-thanks-for',
u'source_title':
u'taboolicious',
u'image_permalink':
u'http://dclzexonask.tumblr.com/image/118615261815',
u'tags': [],
u'highlighted': [],
u'recommended_source':
None,
u'state':
u'published',
u'short_url':
u'http://tmblr.co/Z7jrQx1kU1K9t',
u'type':
u'photo',
u'format':
u'html',
u'timestamp':
1431273428,
u'note_count':
510,
u'source_url':
u'http://taboolicious.tumblr.com/post/118601139146/happy-milf-day-many-thanks-for-all-those-that',
u'photos': [{
u'caption':
u'',
u'original_size': {
u'url':
u'http://40.media.tumblr.com/4f98bc55f60c925f6755fa1a272ec6d7/tumblr_no4u7pEOCl1s36m6bo1_1280.jpg',
u'width': 1280,
u'height': 800
},
u'alt_sizes': [{
u'url':
u'http://40.media.tumblr.com/4f98bc55f60c925f6755fa1a272ec6d7/tumblr_no4u7pEOCl1s36m6bo1_1280.jpg',
u'width': 1280,
u'height': 800
}, {
u'url':
u'http://40.media.tumblr.com/4f98bc55f60c925f6755fa1a272ec6d7/tumblr_no4u7pEOCl1s36m6bo1_500.jpg',
u'width': 500,
u'height': 313
}, {
u'url':
u'http://36.media.tumblr.com/4f98bc55f60c925f6755fa1a272ec6d7/tumblr_no4u7pEOCl1s36m6bo1_400.jpg',
u'width': 400,
u'height': 250
}, {
u'url':
u'http://40.media.tumblr.com/4f98bc55f60c925f6755fa1a272ec6d7/tumblr_no4u7pEOCl1s36m6bo1_250.jpg',
u'width': 250,
u'height': 156
}, {
u'url':
u'http://40.media.tumblr.com/4f98bc55f60c925f6755fa1a272ec6d7/tumblr_no4u7pEOCl1s36m6bo1_100.jpg',
u'width': 100,
u'height': 63
}, {
u'url':
u'http://40.media.tumblr.com/4f98bc55f60c925f6755fa1a272ec6d7/tumblr_no4u7pEOCl1s36m6bo1_75sq.jpg',
u'width': 75,
u'height': 75
}]
}],
u'date':
u'2015-05-10 15:57:08 GMT',
u'slug':
u'taboolicious-happy-milf-day-many-thanks-for',
u'blog_name':
u'dclzexonask',
u'trail': [{
u'blog': {
u'active': True,
u'theme': {
u'title_font_weight':
u'bold',
u'title_color':
u'#ff0062',
u'header_bounds':
0,
u'title_font':
u'Calluna Sans',
u'link_color':
u'#ff002b',
u'header_image_focused':
u'http://static.tumblr.com/d48ceb5a7ccc9b9eeffdcf3aa9972391/3jhlkdq/1DXn0hvh0/tumblr_static_splash.jpg',
u'show_description':
True,
u'show_header_image':
True,
u'body_font':
u'Lucida Sans',
u'show_title':
True,
u'header_stretch':
True,
u'avatar_shape':
u'circle',
u'show_avatar':
True,
u'background_color':
u'#f6f6f6',
u'header_image':
u'http://static.tumblr.com/d48ceb5a7ccc9b9eeffdcf3aa9972391/3jhlkdq/1DXn0hvh0/tumblr_static_splash.jpg',
u'header_image_scaled':
u'http://static.tumblr.com/d48ceb5a7ccc9b9eeffdcf3aa9972391/3jhlkdq/1DXn0hvh0/tumblr_static_splash.jpg'
},
u'name': u'taboolicious'
},
u'content':
u'<p>happy milf day! many thanks for all those that where present in the raffle and suggested such hot milfs :D</p><p>here is the wallpaper: 1920x1200</p><p><a href="https://dl.dropboxusercontent.com/u/1834642/milf%20paper.jpg">https://dl.dropboxusercontent.com/u/1834642/milf%20paper.jpg</a></p><p>from left to right:</p><p>bananacreamcakecomic: Jenna OC</p><p>jacquesthebird: Kate OC</p><p>damabiath: android 18</p><p>bellend08: Molly Cosset OC</p><p>vic79: Mystique</p><p>ravenlordmercenary: Grace pokemon X/Y</p><p>Th3A1chemist: Vuong OC</p><p>Evenanthy: Harley Quinn</p><p><br></p><p>if any of the winners wants a wallpaper with just his/her requested girl, pm me :)</p>',
u'post': {
u'id': u'118601139146'
},
u'is_root_item': True
}],
u'caption':
u'<p><a href="http://taboolicious.tumblr.com/post/118601139146/happy-milf-day-many-thanks-for-all-those-that" class="tumblr_blog">taboolicious</a>:</p>\n\n<blockquote><p>happy milf day! many thanks for all those that where present in the raffle and suggested such hot milfs :D</p><p>here is the wallpaper: 1920x1200</p><p><a href="https://dl.dropboxusercontent.com/u/1834642/milf%20paper.jpg">https://dl.dropboxusercontent.com/u/1834642/milf%20paper.jpg</a></p><p>from left to right:</p><p>bananacreamcakecomic: Jenna OC</p><p>jacquesthebird: Kate OC</p><p>damabiath: android 18</p><p>bellend08: Molly Cosset OC</p><p>vic79: Mystique</p><p>ravenlordmercenary: Grace pokemon X/Y</p><p>Th3A1chemist: Vuong OC</p><p>Evenanthy: Harley Quinn</p><p><br/></p><p>if any of the winners wants a wallpaper with just his/her requested girl, pm me :)</p></blockquote>'
}
malformed_link_result = handle_links(session,
post_dict=malformed_link_post_dict)
logging.debug("malformed_link_result:" + repr(malformed_link_result))
def main():
session = sql_functions.connect_to_db()
add_skip(session,primary_key=1548538L)
confirm_skip(session)
def main():
session = sql_functions.connect_to_db()
add_skip(session, primary_key=1548538L)
confirm_skip(session)
