def debug(): """For WIP, debug, ect function calls""" session = sql_functions.connect_to_db() # tumblr tumblr_post_dict = {u'reblog_key': u'fZE1iow1', u'reblog': {u'comment': u'', u'tree_html': u'<p><a href="http://asklalalexxi.tumblr.com/post/115588359781/i-normally-dont-like-yogurt-because-of-the" class="tumblr_blog">asklalalexxi</a>:</p><blockquote><p>I normally dont like yogurt because of the consistency. but omg\u2026.ive been starving~ Since the infections gone, I can eat dairy now, and this ironically doesn\u2019t make my stomach upset.\xa0</p></blockquote>', u'trail': [{u'blog': {u'theme': {u'title_font_weight': u'bold', u'title_color': u'#444444', u'header_bounds': 0, u'title_font': u'Gibson', u'link_color': u'#529ECC', u'header_image_focused': u'http://assets.tumblr.com/images/default_header/optica_pattern_08.png?_v=f0f055039bb6136b9661cf2227b535c2', u'show_description': True, u'show_header_image': True, u'header_stretch': True, u'body_font': u'Helvetica Neue', u'show_title': True, u'header_image_scaled': u'http://assets.tumblr.com/images/default_header/optica_pattern_08.png?_v=f0f055039bb6136b9661cf2227b535c2', u'avatar_shape': u'square', u'show_avatar': True, u'background_color': u'#FAFAFA', u'header_image': u'http://assets.tumblr.com/images/default_header/optica_pattern_08.png?_v=f0f055039bb6136b9661cf2227b535c2'}, u'name': u'asklalalexxi'}, u'comment': u'<p>I normally dont like yogurt because of the consistency. but omg\u2026.ive been starving~ Since the infections gone, I can eat dairy now, and this ironically doesn\u2019t make my stomach upset.\xa0</p>', u'post': {u'id': u'115588359781'}}]}, u'player': u'<embed type="application/x-shockwave-flash" src="http://assets.tumblr.com/swf/audio_player.swf?audio_file=https%3A%2F%2Fwww.tumblr.com%2Faudio_file%2Faskbuttonsmom%2F115588501253%2Ftumblr_nmcidl4oPW1sl74rq&color=FFFFFF" height="27" width="207" quality="best" wmode="opaque"></embed>', u'id': 115588501253L, u'post_url': u'http://askbuttonsmom.tumblr.com/post/115588501253/asklalalexxi-i-normally-dont-like-yogurt-because', u'plays': 1550, u'tags': [], u'highlighted': [], u'state': u'published', u'track_name': u'When you havent eaten in a while...', u'short_url': u'http://tmblr.co/ZUGffq1hfd8y5', u'type': u'audio', u'format': u'html', u'timestamp': 1428257396, u'note_count': 67, u'date': u'2015-04-05 18:09:56 GMT', u'slug': u'asklalalexxi-i-normally-dont-like-yogurt-because', u'album_art': u'http://31.media.tumblr.com/tumblr_nmcidl4oPW1sl74rqo1_1428257289_cover.png', u'blog_name': u'askbuttonsmom', u'audio_url': u'https://www.tumblr.com/audio_file/askbuttonsmom/115588501253/tumblr_nmcidl4oPW1sl74rq', u'caption': u'<p><a href="http://asklalalexxi.tumblr.com/post/115588359781/i-normally-dont-like-yogurt-because-of-the" class="tumblr_blog">asklalalexxi</a>:</p><blockquote><p>I normally dont like yogurt because of the consistency. but omg\u2026.ive been starving~ Since the infections gone, I can eat dairy now, and this ironically doesn\u2019t make my stomach upset.\xa0</p></blockquote>', u'audio_type': u'tumblr', u'audio_source_url': u'https://www.tumblr.com/audio_file/askbuttonsmom/115588501253/tumblr_nmcidl4oPW1sl74rq', u'embed': u'<iframe class="tumblr_audio_player tumblr_audio_player_115588501253" src="http://askbuttonsmom.tumblr.com/post/115588501253/audio_player_iframe/askbuttonsmom/tumblr_nmcidl4oPW1sl74rq?audio_file=https%3A%2F%2Fwww.tumblr.com%2Faudio_file%2Faskbuttonsmom%2F115588501253%2Ftumblr_nmcidl4oPW1sl74rq" frameborder="0" allowtransparency="true" scrolling="no" width="500" height="169"></iframe>'} #tumblr_result = handle_audio_posts(session,tumblr_post_dict) # soundcloud soundcloud_post_dict_1 = {u'reblog_key': u'S6VWj0Cb', u'reblog': {u'comment': u'', u'tree_html': u'<p><a class="tumblr_blog" href="http://waltzforluma.tumblr.com/post/111622677961/or-your-computer-could-overheat-while-youre">waltzforluma</a>:</p><blockquote><p>Or, your computer could overheat while you\u2019re listening to \u201cDeath Rag\u201d from Future Vision, and burst into flames!</p></blockquote>', u'trail': [{u'blog': {u'theme': {u'title_font_weight': u'regular', u'header_full_height': 262, u'title_color': u'#444444', u'header_bounds': u'0,623,262,157', u'background_color': u'#FAFAFA', u'link_color': u'#529ECC', u'header_image_focused': u'http://static.tumblr.com/a50dd34705b42b1479c2535a15461b00/oevxq7m/ZrTn5ivly/tumblr_static_tumblr_static_57wsbbc6rz0g0gk4ww8k884wk_focused_v3.png', u'show_description': True, u'header_full_width': 898, u'avatar_shape': u'circle', u'header_focus_width': 466, u'show_header_image': True, u'body_font': u'Helvetica Neue', u'show_title': True, u'header_stretch': True, u'header_image_scaled': u'http://static.tumblr.com/a50dd34705b42b1479c2535a15461b00/oevxq7m/g9cn5ivlx/tumblr_static_57wsbbc6rz0g0gk4ww8k884wk_2048_v2.png', u'show_avatar': True, u'header_focus_height': 262, u'title_font': u'Garamond Classic FS', u'header_image': u'http://static.tumblr.com/a50dd34705b42b1479c2535a15461b00/oevxq7m/g9cn5ivlx/tumblr_static_57wsbbc6rz0g0gk4ww8k884wk.png'}, u'name': u'waltzforluma'}, u'comment': u'<p>Or, your computer could overheat while you\u2019re listening to \u201cDeath Rag\u201d from Future Vision, and burst into flames!</p>', u'post': {u'id': u'111622677961'}}]}, u'player': u'<iframe src="https://w.soundcloud.com/player/?url=https%3A%2F%2Fapi.soundcloud.com%2Ftracks%2F192213990&visual=true&liking=false&sharing=false&auto_play=false&show_comments=false&continuous_play=false&origin=tumblr" frameborder="0" allowtransparency="true" class="soundcloud_audio_player" width="500" height="500"></iframe>', u'id': 113020390888L, u'post_url': u'http://doscoon.tumblr.com/post/113020390888/waltzforluma-or-your-computer-could-overheat', u'source_title': u'waltzforluma', u'format': u'html', u'highlighted': [], u'state': u'published', u'track_name': u'Steven Universe - Death Rag', u'short_url': u'http://tmblr.co/ZlYOqv1fGYate', u'type': u'audio', u'tags': [], u'timestamp': 1425776404, u'note_count': 1014, u'source_url': u'http://waltzforluma.tumblr.com/post/111622677961/or-your-computer-could-overheat-while-youre', u'date': u'2015-03-08 01:00:04 GMT', u'plays': 38933, u'slug': u'waltzforluma-or-your-computer-could-overheat', u'album_art': u'http://38.media.tumblr.com/tumblr_nk3re1A1Cf1qzqb72_1424489834_cover.jpg', u'blog_name': u'doscoon', u'is_external': True, u'audio_url': u'https://api.soundcloud.com/tracks/192213990/stream?client_id=3cQaPshpEeLqMsNFAUw1Q', u'caption': u'<p><a class="tumblr_blog" href="http://waltzforluma.tumblr.com/post/111622677961/or-your-computer-could-overheat-while-youre">waltzforluma</a>:</p><blockquote><p>Or, your computer could overheat while you\u2019re listening to \u201cDeath Rag\u201d from Future Vision, and burst into flames!</p></blockquote>', u'audio_type': u'soundcloud', u'audio_source_url': u'https://soundcloud.com/aivisura/steven-universe-death-rag', u'embed': u'<iframe src="https://w.soundcloud.com/player/?url=https%3A%2F%2Fapi.soundcloud.com%2Ftracks%2F192213990&visual=true&liking=false&sharing=false&auto_play=false&show_comments=false&continuous_play=false&origin=tumblr" frameborder="0" allowtransparency="true" class="soundcloud_audio_player" width="500" height="500"></iframe>'} #soundcloud_result_1 = handle_audio_posts(session,soundcloud_post_dict_1) # soundcloud soundcloud_post_dict_2 = {u'reblog_key': u'S6VWj0Cb', u'reblog': {u'comment': u'', u'tree_html': u'<p><a class="tumblr_blog" href="http://waltzforluma.tumblr.com/post/111622677961/or-your-computer-could-overheat-while-youre">waltzforluma</a>:</p><blockquote><p>Or, your computer could overheat while you\u2019re listening to \u201cDeath Rag\u201d from Future Vision, and burst into flames!</p></blockquote>', u'trail': [{u'blog': {u'theme': {u'title_font_weight': u'regular', u'header_full_height': 262, u'title_color': u'#444444', u'header_bounds': u'0,623,262,157', u'background_color': u'#FAFAFA', u'link_color': u'#529ECC', u'header_image_focused': u'http://static.tumblr.com/a50dd34705b42b1479c2535a15461b00/oevxq7m/ZrTn5ivly/tumblr_static_tumblr_static_57wsbbc6rz0g0gk4ww8k884wk_focused_v3.png', u'show_description': True, u'header_full_width': 898, u'avatar_shape': u'circle', u'header_focus_width': 466, u'show_header_image': True, u'body_font': u'Helvetica Neue', u'show_title': True, u'header_stretch': True, u'header_image_scaled': u'http://static.tumblr.com/a50dd34705b42b1479c2535a15461b00/oevxq7m/g9cn5ivlx/tumblr_static_57wsbbc6rz0g0gk4ww8k884wk_2048_v2.png', u'show_avatar': True, u'header_focus_height': 262, u'title_font': u'Garamond Classic FS', u'header_image': u'http://static.tumblr.com/a50dd34705b42b1479c2535a15461b00/oevxq7m/g9cn5ivlx/tumblr_static_57wsbbc6rz0g0gk4ww8k884wk.png'}, u'name': u'waltzforluma'}, u'comment': u'<p>Or, your computer could overheat while you\u2019re listening to \u201cDeath Rag\u201d from Future Vision, and burst into flames!</p>', u'post': {u'id': u'111622677961'}}]}, u'player': u'<iframe src="https://w.soundcloud.com/player/?url=https%3A%2F%2Fapi.soundcloud.com%2Ftracks%2F192213990&visual=true&liking=false&sharing=false&auto_play=false&show_comments=false&continuous_play=false&origin=tumblr" frameborder="0" allowtransparency="true" class="soundcloud_audio_player" width="500" height="500"></iframe>', u'id': 113020390888L, u'post_url': u'http://doscoon.tumblr.com/post/113020390888/waltzforluma-or-your-computer-could-overheat', u'source_title': u'waltzforluma', u'format': u'html', u'highlighted': [], u'state': u'published', u'track_name': u'Steven Universe - Death Rag', u'short_url': u'http://tmblr.co/ZlYOqv1fGYate', u'type': u'audio', u'tags': [], u'timestamp': 1425776404, u'note_count': 1014, u'source_url': u'http://waltzforluma.tumblr.com/post/111622677961/or-your-computer-could-overheat-while-youre', u'date': u'2015-03-08 01:00:04 GMT', u'plays': 38933, u'slug': u'waltzforluma-or-your-computer-could-overheat', u'album_art': u'http://38.media.tumblr.com/tumblr_nk3re1A1Cf1qzqb72_1424489834_cover.jpg', u'blog_name': u'doscoon', u'is_external': True, u'audio_url': u'https://api.soundcloud.com/tracks/192213990/stream?client_id=3cQaPshpEeLqMsNFAUw1Q', u'caption': u'<p><a class="tumblr_blog" href="http://waltzforluma.tumblr.com/post/111622677961/or-your-computer-could-overheat-while-youre">waltzforluma</a>:</p><blockquote><p>Or, your computer could overheat while you\u2019re listening to \u201cDeath Rag\u201d from Future Vision, and burst into flames!</p></blockquote>', u'audio_type': u'soundcloud', u'audio_source_url': u'https://soundcloud.com/aivisura/steven-universe-death-rag', u'embed': u'<iframe src="https://w.soundcloud.com/player/?url=https%3A%2F%2Fapi.soundcloud.com%2Ftracks%2F192213990&visual=true&liking=false&sharing=false&auto_play=false&show_comments=false&continuous_play=false&origin=tumblr" frameborder="0" allowtransparency="true" class="soundcloud_audio_player" width="500" height="500"></iframe>'} soundcloud_result_2 = handle_audio_posts(session,soundcloud_post_dict_2) logging.info("soundcloud_result_2:"+repr(soundcloud_result_2)) # Dead soundcloud soundcloud_post_dict_3 = {u'reblog_key': u'UygtqfHO', u'reblog': {u'comment': u'', u'tree_html': u''}, u'audio_url': None, u'id': 96589998856L, u'album': u'Pure Heroine - EP', u'highlighted': [], u'plays': 463291, u'format': u'html', u'post_url': u'http://tepidtwisted.tumblr.com/post/96589998856', u'recommended_source': None, u'state': u'published', u'track_name': u'Everybody Wants To Rule The World', u'short_url': u'http://tmblr.co/ZrDCJw1PzDci8', u'type': u'audio', u'tags': [], u'timestamp': 1409799977, u'note_count': 62060, u'source_url': u'http://audio-trip.tumblr.com/post/84453352328', u'trail': [], u'player': u'<iframe src="https://w.soundcloud.com/player/?url=&visual=true&liking=false&sharing=false&auto_play=false&show_comments=false&continuous_play=false&origin=tumblr" frameborder="0" allowtransparency="true" class="soundcloud_audio_player" width="500" height="500"></iframe>', u'date': u'2014-09-04 03:06:17 GMT', u'source_title': u'audio-trip', u'slug': u'', u'album_art': u'http://33.media.tumblr.com/tumblr_n4wy9ee6BY1rhgjr7o1_1398977906_cover.jpg', u'blog_name': u'tepidtwisted', u'is_external': True, u'artist': u'Lorde', u'caption': u'', u'audio_type': u'soundcloud', u'audio_source_url': None, u'embed': u'<iframe src="https://w.soundcloud.com/player/?url=&visual=true&liking=false&sharing=false&auto_play=false&show_comments=false&continuous_play=false&origin=tumblr" frameborder="0" allowtransparency="true" class="soundcloud_audio_player" width="500" height="500"></iframe>'} soundcloud_result_3 = handle_audio_posts(session,soundcloud_post_dict_3) logging.info("soundcloud_result_3:"+repr(soundcloud_result_3)) # bandcamp bandcamp_post_dict = {u'reblog_key': u'AbxPRz4A', u'short_url': u'http://tmblr.co/ZE5FbyhlEyOo', u'audio_url': u'http://popplers5.bandcamp.com/download/track?enc=mp3-128&fsig=ca33bbe78b527483940c050bd627a33d&id=2851867171&nl=1&stream=1&ts=1428792368.0', u'player': u'<iframe class="bandcamp_audio_player" width="500" height="120" src="http://bandcamp.com/EmbeddedPlayer/size=medium/bgcol=ffffff/linkcol=0687f5/notracklist=true/transparent=true/track=2851867171/" allowtransparency="true" frameborder="0"></iframe>', u'id': 46963344946, u'album': u'Hunter Hunted - EP', u'post_url': u'http://staff.tumblr.com/post/46963344946/listen-up-our-audio-player-just-got-super-rad', u'source_title': u'Bandcamp', u'tags': [u'hunter hunted', u'keep together', u'music on tumblr', u'bands on tumblr'], u'highlighted': [], u'state': u'published', u'track_name': u'Keep Together', u'type': u'audio', u'featured_in_tag': [u'Music', u'Design'], u'format': u'html', u'timestamp': 1364937324, u'note_count': 5105, u'source_url': u'http://hunterhuntedmusic.bandcamp.com/track/keep-together', u'date': u'2013-04-02 21:15:24 GMT', u'plays': 110209, u'slug': u'listen-up-our-audio-player-just-got-super-rad', u'album_art': u'http://31.media.tumblr.com/tumblr_mkn8sgrMDN1qz8q0h_dSDeaHscmUuv4sl8epf8mAC32IU_cover.jpg', u'blog_name': u'staff', u'is_external': True, u'artist': u'Hunter Hunted', u'caption': u'<p>Listen up: our audio player just got super rad!</p>\n<ul><li><span>Fancy new audio visualizer</span></li>\n<li><span>Bigger album art</span></li>\n<li><span>Click and drag to skip around</span></li>\n</ul><p>Hit play and happy listening!</p>', u'audio_type': u'bandcamp', u'audio_source_url': u'http://popplers5.bandcamp.com/download/track?enc=mp3-128&fsig=ca33bbe78b527483940c050bd627a33d&id=2851867171&nl=1&stream=1&ts=1428792368.0', u'embed': u'<iframe class="bandcamp_audio_player" width="100%" height="120" src="http://bandcamp.com/EmbeddedPlayer/size=medium/bgcol=ffffff/linkcol=0687f5/notracklist=true/transparent=true/track=2851867171/" allowtransparency="true" frameborder="0"></iframe>'} #bandcamp_result = handle_audio_posts(session,bandcamp_post_dict) # Spotify spotify_post_dict = {u'reblog_key': u'AbxPRz4A', u'short_url': u'http://tmblr.co/ZE5FbyhlEyOo', u'audio_url': u'http://popplers5.bandcamp.com/download/track?enc=mp3-128&fsig=ca33bbe78b527483940c050bd627a33d&id=2851867171&nl=1&stream=1&ts=1428792368.0', u'player': u'<iframe class="bandcamp_audio_player" width="500" height="120" src="http://bandcamp.com/EmbeddedPlayer/size=medium/bgcol=ffffff/linkcol=0687f5/notracklist=true/transparent=true/track=2851867171/" allowtransparency="true" frameborder="0"></iframe>', u'id': 46963344946, u'album': u'Hunter Hunted - EP', u'post_url': u'http://staff.tumblr.com/post/46963344946/listen-up-our-audio-player-just-got-super-rad', u'source_title': u'Bandcamp', u'tags': [u'hunter hunted', u'keep together', u'music on tumblr', u'bands on tumblr'], u'highlighted': [], u'state': u'published', u'track_name': u'Keep Together', u'type': u'audio', u'featured_in_tag': [u'Music', u'Design'], u'format': u'html', u'timestamp': 1364937324, u'note_count': 5105, u'source_url': u'http://hunterhuntedmusic.bandcamp.com/track/keep-together', u'date': u'2013-04-02 21:15:24 GMT', u'plays': 110209, u'slug': u'listen-up-our-audio-player-just-got-super-rad', u'album_art': u'http://31.media.tumblr.com/tumblr_mkn8sgrMDN1qz8q0h_dSDeaHscmUuv4sl8epf8mAC32IU_cover.jpg', u'blog_name': u'staff', u'is_external': True, u'artist': u'Hunter Hunted', u'caption': u'<p>Listen up: our audio player just got super rad!</p>\n<ul><li><span>Fancy new audio visualizer</span></li>\n<li><span>Bigger album art</span></li>\n<li><span>Click and drag to skip around</span></li>\n</ul><p>Hit play and happy listening!</p>', u'audio_type': u'bandcamp', u'audio_source_url': u'http://popplers5.bandcamp.com/download/track?enc=mp3-128&fsig=ca33bbe78b527483940c050bd627a33d&id=2851867171&nl=1&stream=1&ts=1428792368.0', u'embed': u'<iframe class="bandcamp_audio_player" width="100%" height="120" src="http://bandcamp.com/EmbeddedPlayer/size=medium/bgcol=ffffff/linkcol=0687f5/notracklist=true/transparent=true/track=2851867171/" allowtransparency="true" frameborder="0"></iframe>'} #spotify_result = handle_audio_posts(session,spotify_post_dict) return
def post_producer(post_queue, target_blog=None): """Keep a queue-like object filled with post dicts""" # Connect to the DB database_session = sql_functions.connect_to_db() # Keep the queue supplied with posts counter = 0 while True: if post_queue.qsize() < 100: logging.info( "Adding more posts to post queue. Posts added so far: " + repr(counter)) new_posts = list_new_posts(database_session=database_session, max_rows=1000, target_blog=target_blog) if len(new_posts) == 0: # Tell consumers to stop by flooding the queue with None objects #logging.error("No more posts to process, exiting.") for n in xrange(100): post_queue.put(None) return for new_post in new_posts: assert (post_queue.full() is False) counter += 1 post_queue.put(new_post) else: time.sleep(1)
def generate_stats_page(): logging.info("Generating stats page...") # Connect to the DB session = sql_functions.connect_to_db() page_html = "Rowcounts:" + "\r\n" # Get rowcounts raw_posts_rowcount = session.query( sqlalchemy.func.count(RawPosts.primary_key)).scalar() media_rowcount = session.query(sqlalchemy.func.count( Media.media_id)).scalar() posts_rowcount = session.query(sqlalchemy.func.count( posts.post_id)).scalar() blogs_rowcount = session.query(sqlalchemy.func.count( blogs.blog_id)).scalar() # Add values to page page_html += "raw_posts: " + splitthousands( str(raw_posts_rowcount)) + "\r\n" page_html += "posts: " + splitthousands(str(posts_rowcount)) + "\r\n" page_html += "blogs: " + splitthousands(str(blogs_rowcount)) + "\r\n" page_html += "Media: " + splitthousands(str(media_rowcount)) + "\r\n" page_html += "\r\n" page_html += "\r\n" page_html += "\r\n" page_html += "Ratios:" + "\r\n" page_html += "raw_posts/posts: " + repr( (raw_posts_rowcount / posts_rowcount)) + "\r\n" page_html += "raw_posts: " + splitthousands( str(raw_posts_rowcount)) + ", posts: " + splitthousands( str(posts_rowcount)) + "\r\n" logging.debug(page_html) save_file(file_path="stats.txt", data=page_html, force_save=True, allow_fail=False) logging.info("Generated stats page.")
def blog_consumer(blog_url_queue): "Process posts for a queue-like object containing blog URLs" logging.debug("Consumer function started.") # Connect to DB database_session = sql_functions.connect_to_db() # Process posts from the queue c = 0# Counter for number of posts processed while True: c += 1 if c%100 == 0: logging.info(repr(c)+" posts processed by this process") #suicide_timer = threading.Timer(1200, suicider_media)# Kill after 20 minutes (1200 seconds) #suicide_timer.start() blog_url = blog_url_queue.get(timeout=600) if blog_url is None:# Stop if None object is put into the queue logging.info("Post consumer recieved None object as exit signal") break# Stop doing work and exit thread/process save_blog(blog_url) #suicide_timer.cancel()# Remove suicide timer after each blog continue # Disconnect from DB database_session.close() logging.debug("Consumer function exiting.") return
def debug(): """For WIP, debug, ect function calls""" session = sql_functions.connect_to_db() # unusual url u'http://fallfeatherspony.tumblr.com/post/109567450555/askloona-\u1d1b-\u029c-\u1d07-\u028f-\u1d21-\u1d07-\u0280-\u1d07' strange_url_result_1 = run_yt_dl_single( session, download_url= u'http://fallfeatherspony.tumblr.com/post/109567450555/askloona-\u1d1b-\u029c-\u1d07-\u028f-\u1d21-\u1d07-\u0280-\u1d07', extractor_used="DEBUG:strange", audio_id=None, video_id=None) logging.info("strange_url_result_1: " + repr(strange_url_result_1)) #return # gfycat gfycat_result = run_yt_dl_single( session, download_url="http://gfycat.com/MerryAdolescentAlaskankleekai", extractor_used="DEBUG:linked_gfycat", audio_id=None, video_id=None) logging.debug("gfycat_result: " + repr(gfycat_result)) # youtube https://www.youtube.com/embed/JIQ_jjfDsw0 youtube_result = run_yt_dl_single( session, download_url="https://www.youtube.com/embed/JIQ_jjfDsw0", extractor_used="DEBUG:youtube", audio_id=None, video_id=None) return
def generate_stats_page(): logging.info("Generating stats page...") # Connect to the DB session = sql_functions.connect_to_db() page_html = "Rowcounts:"+"\r\n" # Get rowcounts raw_posts_rowcount = session.query(sqlalchemy.func.count(RawPosts.primary_key)).scalar() media_rowcount = session.query(sqlalchemy.func.count(Media.media_id)).scalar() posts_rowcount = session.query(sqlalchemy.func.count(posts.post_id)).scalar() blogs_rowcount = session.query(sqlalchemy.func.count(blogs.blog_id)).scalar() # Add values to page page_html += "raw_posts: "+splitthousands(str(raw_posts_rowcount))+"\r\n" page_html += "posts: "+splitthousands(str(posts_rowcount))+"\r\n" page_html += "blogs: "+splitthousands(str(blogs_rowcount))+"\r\n" page_html += "Media: "+splitthousands(str(media_rowcount))+"\r\n" page_html += "\r\n" page_html += "\r\n" page_html += "\r\n" page_html += "Ratios:"+"\r\n" page_html += "raw_posts/posts: "+repr((raw_posts_rowcount/posts_rowcount))+"\r\n" page_html += "raw_posts: "+splitthousands(str(raw_posts_rowcount))+", posts: "+splitthousands(str(posts_rowcount))+"\r\n" logging.debug(page_html) save_file( file_path="stats.txt", data=page_html, force_save=True, allow_fail=False ) logging.info("Generated stats page.")
def debug(): """For WIP, debug, ect function calls""" session = sql_functions.connect_to_db() # unusual url u'http://fallfeatherspony.tumblr.com/post/109567450555/askloona-\u1d1b-\u029c-\u1d07-\u028f-\u1d21-\u1d07-\u0280-\u1d07' strange_url_result_1 = run_yt_dl_single( session, download_url = u'http://fallfeatherspony.tumblr.com/post/109567450555/askloona-\u1d1b-\u029c-\u1d07-\u028f-\u1d21-\u1d07-\u0280-\u1d07', extractor_used = "DEBUG:strange", audio_id=None, video_id=None ) logging.info("strange_url_result_1: "+repr(strange_url_result_1)) #return # gfycat gfycat_result = run_yt_dl_single( session, download_url = "http://gfycat.com/MerryAdolescentAlaskankleekai", extractor_used = "DEBUG:linked_gfycat", audio_id=None, video_id=None ) logging.debug("gfycat_result: "+repr(gfycat_result)) # youtube https://www.youtube.com/embed/JIQ_jjfDsw0 youtube_result = run_yt_dl_single( session, download_url = "https://www.youtube.com/embed/JIQ_jjfDsw0", extractor_used = "DEBUG:youtube", audio_id=None, video_id=None ) return
def save_blog(blog_url): """Save one tumblr blog""" logging.info("Saving blog: "+repr(blog_url)) try: # Connect to DB session = sql_functions.connect_to_db() # Instantiate blog, collect blog metadata blog = tumblr_blog(session, consumer_key = config.consumer_key, blog_url=blog_url) # Ensure blog actually exists if blog.blog_exists is False: logging.error("Blog does not exist! "+repr(blog_url)) appendlist( blog_url, list_file_path="tumblr_failed_list.txt", initial_text="# List of failed items.\n" ) return # Add new posts for the blog posts = blog.save_new_posts(config.max_pages_to_check) logging.info("Finished saving blog: "+repr(blog_url)) appendlist(blog_url,list_file_path=config.done_list_path,initial_text="# List of completed items.\n") except Exception, e:# Log exceptions and pass them on logging.critical("Unhandled exception in save_blog()!") logging.exception(e) raise
def post_producer(post_queue,target_blog=None): """Keep a queue-like object filled with post dicts""" # Connect to the DB database_session = sql_functions.connect_to_db() # Keep the queue supplied with posts counter = 0 while True: if post_queue.qsize() < 100: logging.info("Adding more posts to post queue. Posts added so far: "+repr(counter)) new_posts = list_new_posts( database_session=database_session, max_rows=1000, target_blog=target_blog ) if len(new_posts) == 0: # Tell consumers to stop by flooding the queue with None objects #logging.error("No more posts to process, exiting.") for n in xrange(100): post_queue.put(None) return for new_post in new_posts: assert(post_queue.full() is False) counter += 1 post_queue.put(new_post) else: time.sleep(1)
def debug(): """For WIP, debug, ect function calls""" session = sql_functions.connect_to_db() print "foo" # Album album_media_id_list = save_album( session, album_link = "http://imgur.com/a/AMibi#0" ) logging.debug("album_media_id_list:"+repr(album_media_id_list)) # Multiple image link multiple_image_link_media_id_list = save_imgur( session, link = "http://imgur.com/UQ1j8OT,2PvFmxV#1" ) logging.debug("multiple_image_link_media_id_list:"+repr(multiple_image_link_media_id_list)) # Single image link single_image_link_media_id_list = save_imgur( session, link = "http://imgur.com/UQ1j8OT" ) logging.debug("single_image_link_media_id_list:"+repr(single_image_link_media_id_list)) logging.debug(repr(locals())) return
def debug(): """Code for debugging during programming goes here so everything is logged to file""" session = sql_functions.connect_to_db() # Debug video DB check #sql_functions.check_if_video_in_db(connection,media_url="https://www.youtube.com/embed/lGIEmH3BoyA",youtube_id=None,sha512base64_hash=None,post_id=None) #return # Debug images logging.debug("Debug images") image_post_dict = {u'highlighted': [], u'reblog_key': u'RSNOnudd', u'format': u'html', u'timestamp': 1401396780, u'note_count': 429, u'tags': [u'p**n', u'furry', u'anthro', u'art', u'fantasy', u'compilation', u'myart', u'futa', u'female', u'nude', u'werewolf'], 'link_to_hash_dict': {}, u'photos': [{u'caption': u'My character Gwen, the hermaphrodite Unicorn. Short for I Guinevere.', u'original_size': {u'url': u'http://41.media.tumblr.com/51dc06d26888063e978967b9effdd79d/tumblr_n6csptiJ5u1rzato1o1_1280.jpg', u'width': 1280, u'height': 1739}, u'alt_sizes': [{u'url': u'http://41.media.tumblr.com/51dc06d26888063e978967b9effdd79d/tumblr_n6csptiJ5u1rzato1o1_1280.jpg', u'width': 1280, u'height': 1739}, {u'url': u'http://41.media.tumblr.com/51dc06d26888063e978967b9effdd79d/tumblr_n6csptiJ5u1rzato1o1_500.jpg', u'width': 500, u'height': 679}, {u'url': u'http://41.media.tumblr.com/51dc06d26888063e978967b9effdd79d/tumblr_n6csptiJ5u1rzato1o1_400.jpg', u'width': 400, u'height': 543}, {u'url': u'http://40.media.tumblr.com/51dc06d26888063e978967b9effdd79d/tumblr_n6csptiJ5u1rzato1o1_250.jpg', u'width': 250, u'height': 340}, {u'url': u'http://41.media.tumblr.com/51dc06d26888063e978967b9effdd79d/tumblr_n6csptiJ5u1rzato1o1_100.jpg', u'width': 100, u'height': 136}, {u'url': u'http://41.media.tumblr.com/51dc06d26888063e978967b9effdd79d/tumblr_n6csptiJ5u1rzato1o1_75sq.jpg', u'width': 75, u'height': 75}]}, {u'caption': u'A young man and one of his harem concubines.', u'original_size': {u'url': u'http://40.media.tumblr.com/df5d6e743955acef44262810e7e68196/tumblr_n6csptiJ5u1rzato1o2_1280.jpg', u'width': 1280, u'height': 1037}, u'alt_sizes': [{u'url': u'http://40.media.tumblr.com/df5d6e743955acef44262810e7e68196/tumblr_n6csptiJ5u1rzato1o2_1280.jpg', u'width': 1280, u'height': 1037}, {u'url': u'http://40.media.tumblr.com/df5d6e743955acef44262810e7e68196/tumblr_n6csptiJ5u1rzato1o2_500.jpg', u'width': 500, u'height': 405}, {u'url': u'http://41.media.tumblr.com/df5d6e743955acef44262810e7e68196/tumblr_n6csptiJ5u1rzato1o2_400.jpg', u'width': 400, u'height': 324}, {u'url': u'http://40.media.tumblr.com/df5d6e743955acef44262810e7e68196/tumblr_n6csptiJ5u1rzato1o2_250.jpg', u'width': 250, u'height': 203}, {u'url': u'http://41.media.tumblr.com/df5d6e743955acef44262810e7e68196/tumblr_n6csptiJ5u1rzato1o2_100.jpg', u'width': 100, u'height': 81}, {u'url': u'http://40.media.tumblr.com/df5d6e743955acef44262810e7e68196/tumblr_n6csptiJ5u1rzato1o2_75sq.jpg', u'width': 75, u'height': 75}]}, {u'caption': u'Gift-art for Robotjoe at FA.', u'original_size': {u'url': u'http://40.media.tumblr.com/027e4e40a7b6dd7437ba19bb0bf66394/tumblr_n6csptiJ5u1rzato1o3_1280.jpg', u'width': 1280, u'height': 1280}, u'alt_sizes': [{u'url': u'http://40.media.tumblr.com/027e4e40a7b6dd7437ba19bb0bf66394/tumblr_n6csptiJ5u1rzato1o3_1280.jpg', u'width': 1280, u'height': 1280}, {u'url': u'http://41.media.tumblr.com/027e4e40a7b6dd7437ba19bb0bf66394/tumblr_n6csptiJ5u1rzato1o3_500.jpg', u'width': 500, u'height': 500}, {u'url': u'http://41.media.tumblr.com/027e4e40a7b6dd7437ba19bb0bf66394/tumblr_n6csptiJ5u1rzato1o3_400.jpg', u'width': 400, u'height': 400}, {u'url': u'http://41.media.tumblr.com/027e4e40a7b6dd7437ba19bb0bf66394/tumblr_n6csptiJ5u1rzato1o3_250.jpg', u'width': 250, u'height': 250}, {u'url': u'http://40.media.tumblr.com/027e4e40a7b6dd7437ba19bb0bf66394/tumblr_n6csptiJ5u1rzato1o3_100.jpg', u'width': 100, u'height': 100}, {u'url': u'http://40.media.tumblr.com/027e4e40a7b6dd7437ba19bb0bf66394/tumblr_n6csptiJ5u1rzato1o3_75sq.jpg', u'width': 75, u'height': 75}]}, {u'caption': u'Giftart for Ritts at FA.', u'original_size': {u'url': u'http://41.media.tumblr.com/b04099342f13a3aaad3ef8d7f9f3080f/tumblr_n6csptiJ5u1rzato1o4_1280.jpg', u'width': 1280, u'height': 1152}, u'alt_sizes': [{u'url': u'http://41.media.tumblr.com/b04099342f13a3aaad3ef8d7f9f3080f/tumblr_n6csptiJ5u1rzato1o4_1280.jpg', u'width': 1280, u'height': 1152}, {u'url': u'http://40.media.tumblr.com/b04099342f13a3aaad3ef8d7f9f3080f/tumblr_n6csptiJ5u1rzato1o4_500.jpg', u'width': 500, u'height': 450}, {u'url': u'http://40.media.tumblr.com/b04099342f13a3aaad3ef8d7f9f3080f/tumblr_n6csptiJ5u1rzato1o4_400.jpg', u'width': 400, u'height': 360}, {u'url': u'http://40.media.tumblr.com/b04099342f13a3aaad3ef8d7f9f3080f/tumblr_n6csptiJ5u1rzato1o4_250.jpg', u'width': 250, u'height': 225}, {u'url': u'http://40.media.tumblr.com/b04099342f13a3aaad3ef8d7f9f3080f/tumblr_n6csptiJ5u1rzato1o4_100.jpg', u'width': 100, u'height': 90}, {u'url': u'http://41.media.tumblr.com/b04099342f13a3aaad3ef8d7f9f3080f/tumblr_n6csptiJ5u1rzato1o4_75sq.jpg', u'width': 75, u'height': 75}]}, {u'caption': u'', u'original_size': {u'url': u'http://41.media.tumblr.com/96a2f5867ff3269def55cba3ddb42282/tumblr_n6csptiJ5u1rzato1o5_1280.jpg', u'width': 1153, u'height': 1920}, u'alt_sizes': [{u'url': u'http://41.media.tumblr.com/96a2f5867ff3269def55cba3ddb42282/tumblr_n6csptiJ5u1rzato1o5_1280.jpg', u'width': 1153, u'height': 1920}, {u'url': u'http://40.media.tumblr.com/96a2f5867ff3269def55cba3ddb42282/tumblr_n6csptiJ5u1rzato1o5_500.jpg', u'width': 450, u'height': 750}, {u'url': u'http://40.media.tumblr.com/96a2f5867ff3269def55cba3ddb42282/tumblr_n6csptiJ5u1rzato1o5_400.jpg', u'width': 360, u'height': 600}, {u'url': u'http://40.media.tumblr.com/96a2f5867ff3269def55cba3ddb42282/tumblr_n6csptiJ5u1rzato1o5_250.jpg', u'width': 240, u'height': 400}, {u'url': u'http://41.media.tumblr.com/96a2f5867ff3269def55cba3ddb42282/tumblr_n6csptiJ5u1rzato1o5_100.jpg', u'width': 100, u'height': 167}, {u'url': u'http://36.media.tumblr.com/96a2f5867ff3269def55cba3ddb42282/tumblr_n6csptiJ5u1rzato1o5_75sq.jpg', u'width': 75, u'height': 75}]}, {u'caption': u'She shot herself in the face, or did others? Up to you.', u'original_size': {u'url': u'http://41.media.tumblr.com/879c064933cf138ae169a152dbd717a4/tumblr_n6csptiJ5u1rzato1o6_1280.jpg', u'width': 841, u'height': 1400}, u'alt_sizes': [{u'url': u'http://41.media.tumblr.com/879c064933cf138ae169a152dbd717a4/tumblr_n6csptiJ5u1rzato1o6_1280.jpg', u'width': 841, u'height': 1400}, {u'url': u'http://40.media.tumblr.com/879c064933cf138ae169a152dbd717a4/tumblr_n6csptiJ5u1rzato1o6_500.jpg', u'width': 451, u'height': 750}, {u'url': u'http://41.media.tumblr.com/879c064933cf138ae169a152dbd717a4/tumblr_n6csptiJ5u1rzato1o6_400.jpg', u'width': 360, u'height': 600}, {u'url': u'http://40.media.tumblr.com/879c064933cf138ae169a152dbd717a4/tumblr_n6csptiJ5u1rzato1o6_250.jpg', u'width': 240, u'height': 400}, {u'url': u'http://40.media.tumblr.com/879c064933cf138ae169a152dbd717a4/tumblr_n6csptiJ5u1rzato1o6_100.jpg', u'width': 100, u'height': 166}, {u'url': u'http://36.media.tumblr.com/879c064933cf138ae169a152dbd717a4/tumblr_n6csptiJ5u1rzato1o6_75sq.jpg', u'width': 75, u'height': 75}]}, {u'caption': u"They're now twins.", u'original_size': {u'url': u'http://36.media.tumblr.com/56df999c75b2ea6e10d9e9e3a4248db6/tumblr_n6csptiJ5u1rzato1o7_1280.jpg', u'width': 1153, u'height': 1920}, u'alt_sizes': [{u'url': u'http://36.media.tumblr.com/56df999c75b2ea6e10d9e9e3a4248db6/tumblr_n6csptiJ5u1rzato1o7_1280.jpg', u'width': 1153, u'height': 1920}, {u'url': u'http://36.media.tumblr.com/56df999c75b2ea6e10d9e9e3a4248db6/tumblr_n6csptiJ5u1rzato1o7_500.jpg', u'width': 450, u'height': 750}, {u'url': u'http://41.media.tumblr.com/56df999c75b2ea6e10d9e9e3a4248db6/tumblr_n6csptiJ5u1rzato1o7_400.jpg', u'width': 360, u'height': 600}, {u'url': u'http://40.media.tumblr.com/56df999c75b2ea6e10d9e9e3a4248db6/tumblr_n6csptiJ5u1rzato1o7_250.jpg', u'width': 240, u'height': 400}, {u'url': u'http://41.media.tumblr.com/56df999c75b2ea6e10d9e9e3a4248db6/tumblr_n6csptiJ5u1rzato1o7_100.jpg', u'width': 100, u'height': 167}, {u'url': u'http://41.media.tumblr.com/56df999c75b2ea6e10d9e9e3a4248db6/tumblr_n6csptiJ5u1rzato1o7_75sq.jpg', u'width': 75, u'height': 75}]}, {u'caption': u'This is knot a funny joke.', u'original_size': {u'url': u'http://36.media.tumblr.com/dd7b3f0723d26d0cf9a5daff5cb82e8a/tumblr_n6csptiJ5u1rzato1o8_1280.jpg', u'width': 1000, u'height': 1000}, u'alt_sizes': [{u'url': u'http://36.media.tumblr.com/dd7b3f0723d26d0cf9a5daff5cb82e8a/tumblr_n6csptiJ5u1rzato1o8_1280.jpg', u'width': 1000, u'height': 1000}, {u'url': u'http://40.media.tumblr.com/dd7b3f0723d26d0cf9a5daff5cb82e8a/tumblr_n6csptiJ5u1rzato1o8_500.jpg', u'width': 500, u'height': 500}, {u'url': u'http://40.media.tumblr.com/dd7b3f0723d26d0cf9a5daff5cb82e8a/tumblr_n6csptiJ5u1rzato1o8_400.jpg', u'width': 400, u'height': 400}, {u'url': u'http://36.media.tumblr.com/dd7b3f0723d26d0cf9a5daff5cb82e8a/tumblr_n6csptiJ5u1rzato1o8_250.jpg', u'width': 250, u'height': 250}, {u'url': u'http://40.media.tumblr.com/dd7b3f0723d26d0cf9a5daff5cb82e8a/tumblr_n6csptiJ5u1rzato1o8_100.jpg', u'width': 100, u'height': 100}, {u'url': u'http://40.media.tumblr.com/dd7b3f0723d26d0cf9a5daff5cb82e8a/tumblr_n6csptiJ5u1rzato1o8_75sq.jpg', u'width': 75, u'height': 75}]}, {u'caption': u'Gift-art for Quillu at FA.', u'original_size': {u'url': u'http://41.media.tumblr.com/e02229ba14b4fb3be2866595e371aaa7/tumblr_n6csptiJ5u1rzato1o9_1280.jpg', u'width': 800, u'height': 1410}, u'alt_sizes': [{u'url': u'http://41.media.tumblr.com/e02229ba14b4fb3be2866595e371aaa7/tumblr_n6csptiJ5u1rzato1o9_1280.jpg', u'width': 800, u'height': 1410}, {u'url': u'http://40.media.tumblr.com/e02229ba14b4fb3be2866595e371aaa7/tumblr_n6csptiJ5u1rzato1o9_500.jpg', u'width': 426, u'height': 750}, {u'url': u'http://41.media.tumblr.com/e02229ba14b4fb3be2866595e371aaa7/tumblr_n6csptiJ5u1rzato1o9_400.jpg', u'width': 340, u'height': 600}, {u'url': u'http://41.media.tumblr.com/e02229ba14b4fb3be2866595e371aaa7/tumblr_n6csptiJ5u1rzato1o9_250.jpg', u'width': 227, u'height': 400}, {u'url': u'http://40.media.tumblr.com/e02229ba14b4fb3be2866595e371aaa7/tumblr_n6csptiJ5u1rzato1o9_100.jpg', u'width': 100, u'height': 176}, {u'url': u'http://41.media.tumblr.com/e02229ba14b4fb3be2866595e371aaa7/tumblr_n6csptiJ5u1rzato1o9_75sq.jpg', u'width': 75, u'height': 75}]}, {u'caption': u'Werewolf herm, in heat. Watch out!', u'original_size': {u'url': u'http://36.media.tumblr.com/3a23e75b564f5d790bb440ba2ba6140c/tumblr_n6csptiJ5u1rzato1o10_1280.jpg', u'width': 1280, u'height': 962}, u'alt_sizes': [{u'url': u'http://36.media.tumblr.com/3a23e75b564f5d790bb440ba2ba6140c/tumblr_n6csptiJ5u1rzato1o10_1280.jpg', u'width': 1280, u'height': 962}, {u'url': u'http://41.media.tumblr.com/3a23e75b564f5d790bb440ba2ba6140c/tumblr_n6csptiJ5u1rzato1o10_500.jpg', u'width': 500, u'height': 376}, {u'url': u'http://41.media.tumblr.com/3a23e75b564f5d790bb440ba2ba6140c/tumblr_n6csptiJ5u1rzato1o10_400.jpg', u'width': 400, u'height': 301}, {u'url': u'http://36.media.tumblr.com/3a23e75b564f5d790bb440ba2ba6140c/tumblr_n6csptiJ5u1rzato1o10_250.jpg', u'width': 250, u'height': 188}, {u'url': u'http://36.media.tumblr.com/3a23e75b564f5d790bb440ba2ba6140c/tumblr_n6csptiJ5u1rzato1o10_100.jpg', u'width': 100, u'height': 75}, {u'url': u'http://40.media.tumblr.com/3a23e75b564f5d790bb440ba2ba6140c/tumblr_n6csptiJ5u1rzato1o10_75sq.jpg', u'width': 75, u'height': 75}]}], u'id': 87231460597L, u'post_url': u'http://zaggatar.tumblr.com/post/87231460597/i-thought-i-would-upload-some-of-what-i-think-is', u'caption': u'<p><span>I thought I would upload s</span>ome of what I think is best of my older stuff.</p>\n<p>As you can see, I am guilty for liking horsegirls with big dicks.</p>\n<p>Enjoy.</p>', u'state': u'published', u'short_url': u'http://tmblr.co/Zlxuxu1HFPdJr', u'date': u'2014-05-29 20:53:00 GMT', u'type': u'photo', u'slug': u'i-thought-i-would-upload-some-of-what-i-think-is', u'photoset_layout': u'1111111111', u'blog_name': u'zaggatar'} #print flatten(image_post_dict) #new_post_dict = save_media(session,image_post_dict) #download_image_link(session,"https://derpicdn.net/spns/W1siZiIsIjIwMTQvMDEvMTAvMDJfNDBfMjhfNjUyX2RlcnBpYm9vcnVfYmFubmVyLnBuZyJdXQ.png") # Debug audio #logging.debug("Debug audio") #soundcloud_post_dict = {u'reblog_key': u'S6VWj0Cb', u'reblog': {u'comment': u'', u'tree_html': u'<p><a class="tumblr_blog" href="http://waltzforluma.tumblr.com/post/111622677961/or-your-computer-could-overheat-while-youre">waltzforluma</a>:</p><blockquote><p>Or, your computer could overheat while you\u2019re listening to \u201cDeath Rag\u201d from Future Vision, and burst into flames!</p></blockquote>', u'trail': [{u'blog': {u'theme': {u'title_font_weight': u'regular', u'header_full_height': 262, u'title_color': u'#444444', u'header_bounds': u'0,623,262,157', u'background_color': u'#FAFAFA', u'link_color': u'#529ECC', u'header_image_focused': u'http://static.tumblr.com/a50dd34705b42b1479c2535a15461b00/oevxq7m/ZrTn5ivly/tumblr_static_tumblr_static_57wsbbc6rz0g0gk4ww8k884wk_focused_v3.png', u'show_description': True, u'header_full_width': 898, u'avatar_shape': u'circle', u'header_focus_width': 466, u'show_header_image': True, u'body_font': u'Helvetica Neue', u'show_title': True, u'header_stretch': True, u'header_image_scaled': u'http://static.tumblr.com/a50dd34705b42b1479c2535a15461b00/oevxq7m/g9cn5ivlx/tumblr_static_57wsbbc6rz0g0gk4ww8k884wk_2048_v2.png', u'show_avatar': True, u'header_focus_height': 262, u'title_font': u'Garamond Classic FS', u'header_image': u'http://static.tumblr.com/a50dd34705b42b1479c2535a15461b00/oevxq7m/g9cn5ivlx/tumblr_static_57wsbbc6rz0g0gk4ww8k884wk.png'}, u'name': u'waltzforluma'}, u'comment': u'<p>Or, your computer could overheat while you\u2019re listening to \u201cDeath Rag\u201d from Future Vision, and burst into flames!</p>', u'post': {u'id': u'111622677961'}}]}, u'player': u'<iframe src="https://w.soundcloud.com/player/?url=https%3A%2F%2Fapi.soundcloud.com%2Ftracks%2F192213990&visual=true&liking=false&sharing=false&auto_play=false&show_comments=false&continuous_play=false&origin=tumblr" frameborder="0" allowtransparency="true" class="soundcloud_audio_player" width="500" height="500"></iframe>', u'id': 113020390888L, u'post_url': u'http://doscoon.tumblr.com/post/113020390888/waltzforluma-or-your-computer-could-overheat', u'source_title': u'waltzforluma', u'format': u'html', u'highlighted': [], u'state': u'published', u'track_name': u'Steven Universe - Death Rag', u'short_url': u'http://tmblr.co/ZlYOqv1fGYate', u'type': u'audio', u'tags': [], u'timestamp': 1425776404, u'note_count': 1014, u'source_url': u'http://waltzforluma.tumblr.com/post/111622677961/or-your-computer-could-overheat-while-youre', u'date': u'2015-03-08 01:00:04 GMT', u'plays': 38933, u'slug': u'waltzforluma-or-your-computer-could-overheat', u'album_art': u'http://38.media.tumblr.com/tumblr_nk3re1A1Cf1qzqb72_1424489834_cover.jpg', u'blog_name': u'doscoon', u'is_external': True, u'audio_url': u'https://api.soundcloud.com/tracks/192213990/stream?client_id=3cQaPshpEeLqMsNFAUw1Q', u'caption': u'<p><a class="tumblr_blog" href="http://waltzforluma.tumblr.com/post/111622677961/or-your-computer-could-overheat-while-youre">waltzforluma</a>:</p><blockquote><p>Or, your computer could overheat while you\u2019re listening to \u201cDeath Rag\u201d from Future Vision, and burst into flames!</p></blockquote>', u'audio_type': u'soundcloud', u'audio_source_url': u'https://soundcloud.com/aivisura/steven-universe-death-rag', u'embed': u'<iframe src="https://w.soundcloud.com/player/?url=https%3A%2F%2Fapi.soundcloud.com%2Ftracks%2F192213990&visual=true&liking=false&sharing=false&auto_play=false&show_comments=false&continuous_play=false&origin=tumblr" frameborder="0" allowtransparency="true" class="soundcloud_audio_player" width="500" height="500"></iframe>'} #audio_handlers.handle_soundcloud_audio(session,soundcloud_post_dict) # Debug video logging.debug("Debug video") # Tumblr video #tumblr_video_post_dict = {u'reblog_key': u'3bqfxHgy', u'short_url': u'http://tmblr.co/Z_sLQw1eYTSqS', u'thumbnail_width': 480, u'player': [{u'width': 250, u'embed_code': u'<iframe width="250" height="140" id="youtube_iframe" src="https://www.youtube.com/embed/tT5pifkZzEk?feature=oembed&enablejsapi=1&origin=http://safe.txmblr.com&wmode=opaque" frameborder="0" allowfullscreen></iframe>'}, {u'width': 400, u'embed_code': u'<iframe width="400" height="225" id="youtube_iframe" src="https://www.youtube.com/embed/tT5pifkZzEk?feature=oembed&enablejsapi=1&origin=http://safe.txmblr.com&wmode=opaque" frameborder="0" allowfullscreen></iframe>'}, {u'width': 500, u'embed_code': u'<iframe width="500" height="281" id="youtube_iframe" src="https://www.youtube.com/embed/tT5pifkZzEk?feature=oembed&enablejsapi=1&origin=http://safe.txmblr.com&wmode=opaque" frameborder="0" allowfullscreen></iframe>'}], u'id': 112247295260L, u'post_url': u'http://tsitra360.tumblr.com/post/112247295260/my-latest-art-timelapse-is-up-see-how-i-drew', u'tags': [], u'highlighted': [], u'state': u'published', u'html5_capable': True, u'type': u'video', u'format': u'html', u'timestamp': 1425068852, u'note_count': 79, u'video_type': u'youtube', u'date': u'2015-02-27 20:27:32 GMT', u'thumbnail_height': 360, u'permalink_url': u'https://www.youtube.com/watch?v=tT5pifkZzEk', u'slug': u'my-latest-art-timelapse-is-up-see-how-i-drew', u'blog_name': u'tsitra360', u'caption': u'<p>My latest art timelapse is up! See how I drew Berry Swirl on my youtube channel.</p>', u'thumbnail_url': u'https://i.ytimg.com/vi/tT5pifkZzEk/hqdefault.jpg'} #save_media(session,tumblr_video_post_dict) # Youtube youtube_video_post_dict = {u'reblog_key': u'HfjckfH7', u'short_url': u'http://tmblr.co/ZUGffq1cfuHuJ', u'thumbnail_width': 480, u'player': [{u'width': 250, u'embed_code': u'<iframe width="250" height="140" id="youtube_iframe" src="https://www.youtube.com/embed/lGIEmH3BoyA?feature=oembed&enablejsapi=1&origin=http://safe.txmblr.com&wmode=opaque" frameborder="0" allowfullscreen></iframe>'}, {u'width': 400, u'embed_code': u'<iframe width="400" height="225" id="youtube_iframe" src="https://www.youtube.com/embed/lGIEmH3BoyA?feature=oembed&enablejsapi=1&origin=http://safe.txmblr.com&wmode=opaque" frameborder="0" allowfullscreen></iframe>'}, {u'width': 500, u'embed_code': u'<iframe width="500" height="281" id="youtube_iframe" src="https://www.youtube.com/embed/lGIEmH3BoyA?feature=oembed&enablejsapi=1&origin=http://safe.txmblr.com&wmode=opaque" frameborder="0" allowfullscreen></iframe>'}], u'id': 110224285203L, u'post_url': u'http://askbuttonsmom.tumblr.com/post/110224285203/throwback-can-you-believe-its-been-almost-2yrs', u'tags': [u"button's mom", u'hardcopy', u'song', u'shadyvox'], u'highlighted': [], u'state': u'published', u'html5_capable': True, u'type': u'video', u'format': u'html', u'timestamp': 1423197599, u'note_count': 145, u'video_type': u'youtube', u'date': u'2015-02-06 04:39:59 GMT', u'thumbnail_height': 360, u'permalink_url': u'https://www.youtube.com/watch?v=lGIEmH3BoyA', u'slug': u'throwback-can-you-believe-its-been-almost-2yrs', u'blog_name': u'askbuttonsmom', u'caption': u'<p>Throwback! Can you believe it’s been almost 2yrs since this came out? Mommy’s getting old…</p>', u'thumbnail_url': u'https://i.ytimg.com/vi/lGIEmH3BoyA/hqdefault.jpg'} youtube_dict_two = {u'highlighted': [], u'reblog_key': u'qO3JnfS7', u'player': [{u'width': 250, u'embed_code': False}, {u'width': 400, u'embed_code': False}, {u'width': 500, u'embed_code': False}], u'format': u'html', u'timestamp': 1390412461, u'note_count': 4282, u'tags': [], u'video_type': u'youtube', u'id': 74184911379L, u'post_url': u'http://askbuttonsmom.tumblr.com/post/74184911379/ask-thecrusaders-bar-buddies-dont-worry', u'caption': u'<p><a class="tumblr_blog" href="http://ask-thecrusaders.tumblr.com/post/74162414750/bar-buddies-dont-worry-neon-you-will-have-your">ask-thecrusaders</a>:</p>\n<blockquote>\n<p><strong>"Bar Buddies"</strong><br/><br/>Dont\u2019 worry Neon, you will have your music video soon enough.</p>\n</blockquote>\n<p>Honestly, that Neon Lights is a TERRIBLE influence!! No son of mine will grow up to be a drunken drug-shooting bass dropping hipster! :C</p>', u'state': u'published', u'html5_capable': False, u'reblog': {u'comment': u'<p>Honestly, that Neon Lights is a TERRIBLE influence!! No son of mine will grow up to be a drunken drug-shooting bass dropping hipster! :C</p>', u'tree_html': u'<p><a class="tumblr_blog" href="http://ask-thecrusaders.tumblr.com/post/74162414750/bar-buddies-dont-worry-neon-you-will-have-your">ask-thecrusaders</a>:</p><blockquote>\n<p><strong>"Bar Buddies"</strong><br/><br/>Dont\u2019 worry Neon, you will have your music video soon enough.</p>\n</blockquote>', u'trail': [{u'blog': {u'theme': {u'title_font_weight': u'bold', u'title_color': u'#444444', u'header_bounds': 0, u'title_font': u'Helvetica Neue', u'link_color': u'#529ECC', u'header_image_focused': u'http://assets.tumblr.com/images/default_header/optica_pattern_04.png?_v=7c4e5e82cf797042596e2e64af1c383f', u'show_description': True, u'show_header_image': True, u'header_stretch': True, u'body_font': u'Helvetica Neue', u'show_title': True, u'header_image_scaled': u'http://assets.tumblr.com/images/default_header/optica_pattern_04.png?_v=7c4e5e82cf797042596e2e64af1c383f', u'avatar_shape': u'circle', u'show_avatar': True, u'background_color': u'#F6F6F6', u'header_image': u'http://assets.tumblr.com/images/default_header/optica_pattern_04.png?_v=7c4e5e82cf797042596e2e64af1c383f'}, u'name': u'ask-thecrusaders'}, u'comment': u'<p><strong>"Bar Buddies"</strong><br><br>Dont\u2019 worry Neon, you will have your music video soon enough.</p>', u'post': {u'id': u'74162414750'}}]}, u'short_url': u'http://tmblr.co/ZUGffq155m_eJ', u'date': u'2014-01-22 17:41:01 GMT', u'type': u'video', u'slug': u'ask-thecrusaders-bar-buddies-dont-worry', u'blog_name': u'askbuttonsmom'} #yt_result_1 = save_media(session,youtube_video_post_dict) #save_media(session,youtube_dict_two) # Vine vine_dict = {u'reblog_key': u'A5DhHt28', u'reblog': {u'comment': u'<p>Have a nice weekend, Tumblr. </p>', u'tree_html': u'', u'trail': []}, u'placement_id': u'{"i":"mF4avY6GyshXjaQmfk0v","v":4,"t":1427193020,"c":{"p":"113540981790","b":"staff"},"d":{"v":{"e":"hjWIUFOYD31"}},"h":"3291f1aa07"}', u'thumbnail_width': 480, u'player': [{u'width': 250, u'embed_code': u'<iframe class="vine-embed" src="https://vine.co/v/hjWIUFOYD31/embed/simple"width="250" height="250" frameborder="0"></iframe><script async src="//platform.vine.co/static/scripts/embed.js" charset="utf-8"></script>'}, {u'width': 400, u'embed_code': u'<iframe class="vine-embed" src="https://vine.co/v/hjWIUFOYD31/embed/simple"width="400" height="400" frameborder="0"></iframe><script async src="//platform.vine.co/static/scripts/embed.js" charset="utf-8"></script>'}, {u'width': 500, u'embed_code': u'<iframe class="vine-embed" src="https://vine.co/v/hjWIUFOYD31/embed/simple"width="500" height="500" frameborder="0"></iframe><script async src="//platform.vine.co/static/scripts/embed.js" charset="utf-8"></script>'}], u'id': 113540981790L, u'post_url': u'http://staff.tumblr.com/post/113540981790/have-a-nice-weekend-tumblr', u'source_title': u'weloveshortvideos.com', u'tags': [], u'highlighted': [], u'state': u'published', u'short_url': u'http://tmblr.co/ZE5Fby1flaUGU', u'html5_capable': True, u'type': u'video', u'format': u'html', u'timestamp': 1426282797, u'note_count': 48309, u'video_type': u'vine', u'source_url': u'http://weloveshortvideos.com', u'date': u'2015-03-13 21:39:57 GMT', u'thumbnail_height': 480, u'permalink_url': u'https://vine.co/v/hjWIUFOYD31', u'slug': u'have-a-nice-weekend-tumblr', u'blog_name': u'staff', u'caption': u'<p>Have a nice weekend, Tumblr.\xa0</p>', u'thumbnail_url': u'http://v.cdn.vine.co/r/thumbs/FE4C8DC8781008139866036658176_1c16044fdd3.3.4.mp4_l_pAXVyCckNVnk2OzdadqNB_6bq4mYoBHpBFRIF8Hi3OdOW1vmjP1TR075G1ZegT.jpg?versionId=abawWSw4Y_QFv2TKPWz6j8N5y7.6LOGq'} #vine_restult = save_media(session,vine_dict) # Vimeo vimeo_dict = {u'reblog_key': u'3BuzwM1q', u'reblog': {u'comment': u'', u'tree_html': u'<p><a href="http://robscorner.tumblr.com/post/110250942998/a-hyperfast-preview-video-for-the-kind-of-content" class="tumblr_blog">robscorner</a>:</p><blockquote><p>A hyperfast preview video for the kind of content I\u2019m featuring on Patreon (patreon.com/robaato)! Slower version will be available for my supporters!<br/>MUSIC: The End (T.E.I.N. Pt. 2) | 12th Planet<br/></p><p>Support for high-resolution art, PSDs, process videos, tutorials, character requests, and more!<br/></p></blockquote>', u'trail': [{u'blog': {u'theme': {u'title_font_weight': u'bold', u'header_full_height': 1071, u'title_color': u'#FFFFFF', u'header_bounds': u'92,1581,978,3', u'title_font': u'Gibson', u'link_color': u'#529ECC', u'header_image_focused': u'http://static.tumblr.com/a5a733e78671519e8eb9cf3700ccfb70/ybimlef/1eon5zyi0/tumblr_static_tumblr_static_2df9bnxrqh1c4c8sgk8448s80_focused_v3.jpg', u'show_description': False, u'header_full_width': 1600, u'header_focus_width': 1578, u'header_stretch': True, u'show_header_image': True, u'body_font': u'Helvetica Neue', u'show_title': True, u'header_image_scaled': u'http://static.tumblr.com/cfa3addece89b58093ea0a8a87082653/ybimlef/FWyn5zyhv/tumblr_static_2df9bnxrqh1c4c8sgk8448s80_2048_v2.png', u'avatar_shape': u'square', u'show_avatar': True, u'header_focus_height': 886, u'background_color': u'#337db1', u'header_image': u'http://static.tumblr.com/cfa3addece89b58093ea0a8a87082653/ybimlef/FWyn5zyhv/tumblr_static_2df9bnxrqh1c4c8sgk8448s80.png'}, u'name': u'robscorner'}, u'comment': u'<p>A hyperfast preview video for the kind of content I\u2019m featuring on Patreon (patreon.com/robaato)! Slower version will be available for my supporters!<br>MUSIC: The End (T.E.I.N. Pt. 2) | 12th Planet<br></p><p>Support for high-resolution art, PSDs, process videos, tutorials, character requests, and more!<br></p>', u'post': {u'id': u'110250942998'}}]}, u'thumbnail_width': 295, u'player': [{u'width': 250, u'embed_code': u'<iframe src="https://player.vimeo.com/video/118912193?title=0&byline=0&portrait=0" width="250" height="156" frameborder="0" title="Hyperfast Preview - Mai (Patreon Process Videos)" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>'}, {u'width': 400, u'embed_code': u'<iframe src="https://player.vimeo.com/video/118912193?title=0&byline=0&portrait=0" width="400" height="250" frameborder="0" title="Hyperfast Preview - Mai (Patreon Process Videos)" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>'}, {u'width': 500, u'embed_code': u'<iframe src="https://player.vimeo.com/video/118912193?title=0&byline=0&portrait=0" width="500" height="312" frameborder="0" title="Hyperfast Preview - Mai (Patreon Process Videos)" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>'}], u'id': 110255840681, u'post_url': u'http://nsfw.kevinsano.com/post/110255840681/robscorner-a-hyperfast-preview-video-for-the-kind', u'source_title': u'robscorner', u'tags': [u'reblog', u'erohua'], u'highlighted': [], u'state': u'published', u'short_url': u'http://tmblr.co/Zo9zBq1chmfsf', u'html5_capable': True, u'type': u'video', u'format': u'html', u'timestamp': 1423238010, u'note_count': 415, u'video_type': u'vimeo', u'source_url': u'http://robscorner.tumblr.com/post/110250942998/a-hyperfast-preview-video-for-the-kind-of-content', u'date': u'2015-02-06 15:53:30 GMT', u'thumbnail_height': 184, u'permalink_url': u'https://vimeo.com/118912193', u'slug': u'robscorner-a-hyperfast-preview-video-for-the-kind', u'blog_name': u'nsfwkevinsano', u'caption': u'<p><a href="http://robscorner.tumblr.com/post/110250942998/a-hyperfast-preview-video-for-the-kind-of-content" class="tumblr_blog">robscorner</a>:</p><blockquote><p>A hyperfast preview video for the kind of content I\u2019m featuring on Patreon (patreon.com/robaato)! Slower version will be available for my supporters!<br/>MUSIC: The End (T.E.I.N. Pt. 2) | 12th Planet<br/></p><p>Support for high-resolution art, PSDs, process videos, tutorials, character requests, and more!<br/></p></blockquote>', u'thumbnail_url': u'https://i.vimeocdn.com/video/506047324_295x166.jpg'} #save_media(session,vimeo_dict) #video_handlers.handle_video_posts(session,vimeo_dict) # Imgur video imgur_post_dict = {u'highlighted': [], u'reblog_key': u'qX0EtplN', u'player': [{u'width': 250, u'embed_code': u'<iframe class="imgur-embed" width="100%" height="720" frameborder="0" src="http://i.imgur.com/wSBlRyv.gifv#embed"></iframe>'}, {u'width': 400, u'embed_code': u'<iframe class="imgur-embed" width="100%" height="720" frameborder="0" src="http://i.imgur.com/wSBlRyv.gifv#embed"></iframe>'}, {u'width': 500, u'embed_code': u'<iframe class="imgur-embed" width="100%" height="720" frameborder="0" src="http://i.imgur.com/wSBlRyv.gifv#embed"></iframe>'}], u'format': u'html', u'timestamp': 1415466120, u'note_count': 109, u'tags': [], u'thumbnail_width': 0, u'id': 102102282191, u'post_url': u'http://jessicaanner.tumblr.com/post/102102282191/front-view-clothed-large-version-gif-back', u'caption': u'<p><em><strong><a href="http://jessicaanner.tumblr.com/post/101601852991/front-view-clothed-large-version-gif-back">Front View (Clothed)</a> <a href="http://i.imgur.com/fDixfAC.gifv"><span class="auto_link" title="">(Large version)</span></a><a href="http://d.facdn.net/art/benezia/1414952655.benezia_front_armored_optimized.gif"><span class="auto_link" title=""> (GIF)</span></a></strong></em><br/><em><strong><a href="http://jessicaanner.tumblr.com/post/101666148721/front-view-clothed-large-version-gif-back">Back View (Clothed)</a> <a href="http://i.imgur.com/QYfRNeQ.gifv" title="">(Large version)</a> <a href="http://d.facdn.net/art/benezia/1415012804.benezia_back_armored_optimized.gif">(GIF)</a></strong></em><br/><em><strong><a href="http://jessicaanner.tumblr.com/post/101768307896/front-view-clothed-large-version-gif-back">Front View (Nude)</a> <a href="http://i.imgur.com/0N7ir7o.gifv">(Large version)</a> <a href="http://d.facdn.net/art/benezia/1415120393.benezia_front_nude_optimized.gif" title="">(GIF)</a></strong></em><br/><em><strong><a href="http://jessicaanner.tumblr.com/post/101852253284/front-view-clothed-large-version-gif-back">Back View (Nude)</a> <a href="http://i.imgur.com/sP5h9ux.gifv" title="">(Large version)</a> <a href="http://d.facdn.net/art/benezia/1415120590.benezia_back_nude_optimized.gif" title="">(GIF)</a></strong></em><br/><strong><em><a href="http://jessicaanner.tumblr.com/post/101934955336/front-view-clothed-large-version-gif-back">Buttocks Closeup View</a> <a href="http://i.imgur.com/BXMYuxk.gifv" title="">(Large version)</a> <a href="http://i.imgur.com/3bhzRP2.gif">(GIF)</a></em></strong><br/><em><strong><a href="http://jessicaanner.tumblr.com/post/102102282191/front-view-clothed-large-version-gif-back">Crotch Closeup View</a> <a href="http://i.imgur.com/wSBlRyv.gifv">(Large version)</a> <a href="http://i.imgur.com/UiDU1XB.gif">(GIF)</a></strong></em><br/><em><strong><a href="http://jessicaanner.tumblr.com/post/102017653601/front-view-clothed-large-version-gif-back">Bust Closeup View</a> <a href="http://i.imgur.com/S5M6PID.gifv">(Large version)</a> <a href="http://i.imgur.com/BlMYohP.gif">(GIF)</a></strong></em></p>', u'state': u'published', u'html5_capable': False, u'video_type': u'unknown', u'short_url': u'http://tmblr.co/ZLO7Om1V5nI-F', u'date': u'2014-11-08 17:02:00 GMT', u'thumbnail_height': 0, u'thumbnail_url': u'', u'type': u'video', u'slug': u'front-view-clothed-large-version-gif-back', u'blog_name': u'jessicaanner'} #imgur_result = save_media(session,imgur_post_dict) # Livestream livestream_post_dict ={u'reblog_key': u'oapXWQlr', u'reblog': {u'comment': u'<p><span>To reiterate: this an </span><strong>only</strong><span> and an </span><strong>exclusive </strong><span>and it </span><strong>starts in just a few minutes</strong><span>. Hurry on over. </span></p>', u'tree_html': u'<p><a class="tumblr_blog" href="http://92y.tumblr.com/post/101031505431/watch-the-92y-livestream-of-game-of-thrones">92y</a>:</p><blockquote>\n<p>Watch the 92Y Livestream of <strong>Game of Thrones</strong> creator <strong>George R.R. Martin, TONIGHT at 8\xa0pm ET</strong>, in his <strong>only</strong> public U.S. appearance for the release of <a href="http://www.amazon.com/gp/product/B00EGMGGVK/ref=as_li_tl?ie=UTF8&camp=1789&creative=390957&creativeASIN=B00EGMGGVK&linkCode=as2&tag=92y-20&linkId=V3MMY57QIQ7QVFNK"><em>The World of Ice and Fire: The Untold History of Westeros and the Game of Thrones</em></a>. Exclusively on Tumblr!</p>\n</blockquote>', u'trail': [{u'blog': {u'theme': {u'title_font_weight': u'bold', u'title_color': u'#444444', u'header_bounds': u'', u'title_font': u'Gibson', u'link_color': u'#529ECC', u'header_image_focused': u'http://assets.tumblr.com/images/default_header/optica_pattern_13_focused_v3.png?_v=2f4063be1dd2ee91e4eca54332e25191', u'show_description': True, u'show_header_image': True, u'header_stretch': True, u'body_font': u'Helvetica Neue', u'show_title': True, u'header_image_scaled': u'http://assets.tumblr.com/images/default_header/optica_pattern_13_focused_v3.png?_v=2f4063be1dd2ee91e4eca54332e25191', u'avatar_shape': u'square', u'show_avatar': True, u'background_color': u'#FAFAFA', u'header_image': u'http://assets.tumblr.com/images/default_header/optica_pattern_13.png?_v=2f4063be1dd2ee91e4eca54332e25191'}, u'name': u'92y'}, u'comment': u'<p>Watch the 92Y Livestream of <strong>Game of Thrones</strong> creator <strong>George R.R. Martin, TONIGHT at 8\xa0pm ET</strong>, in his <strong>only</strong> public U.S. appearance for the release of <a href="http://www.amazon.com/gp/product/B00EGMGGVK/ref=as_li_tl?ie=UTF8&camp=1789&creative=390957&creativeASIN=B00EGMGGVK&linkCode=as2&tag=92y-20&linkId=V3MMY57QIQ7QVFNK"><em>The World of Ice and Fire: The Untold History of Westeros and the Game of Thrones</em></a>. Exclusively on Tumblr!</p>', u'post': {u'id': u'101031505431'}}]}, u'thumbnail_width': 0, u'player': [{u'width': 250, u'embed_code': u'<iframe src="http://new.livestream.com/accounts/1249127/events/3464519/player?width=560&height=315&autoPlay=true&mute=false" width="250" height="140" frameborder="0" scrolling="no"> </iframe>'}, {u'width': 400, u'embed_code': u'<iframe src="http://new.livestream.com/accounts/1249127/events/3464519/player?width=560&height=315&autoPlay=true&mute=false" width="400" height="225" frameborder="0" scrolling="no"> </iframe>'}, {u'width': 500, u'embed_code': u'<iframe src="http://new.livestream.com/accounts/1249127/events/3464519/player?width=560&height=315&autoPlay=true&mute=false" width="500" height="281" frameborder="0" scrolling="no"> </iframe>'}], u'id': 101038462325, u'post_url': u'http://staff.tumblr.com/post/101038462325/92y-watch-the-92y-livestream-of-game-of-thrones', u'tags': [], u'highlighted': [], u'state': u'published', u'short_url': u'http://tmblr.co/ZE5Fby1U6N9Lr', u'html5_capable': False, u'type': u'video', u'format': u'html', u'timestamp': 1414366397, u'note_count': 917, u'video_type': u'unknown', u'date': u'2014-10-26 23:33:17 GMT', u'thumbnail_height': 0, u'slug': u'92y-watch-the-92y-livestream-of-game-of-thrones', u'blog_name': u'staff', u'caption': u'<p><a class="tumblr_blog" href="http://92y.tumblr.com/post/101031505431/watch-the-92y-livestream-of-game-of-thrones">92y</a>:</p>\n<blockquote>\n<p>Watch the 92Y Livestream of <strong>Game of Thrones</strong> creator <strong>George R.R. Martin, TONIGHT at 8\xa0pm ET</strong>, in his <strong>only</strong> public U.S. appearance for the release of <a href="http://www.amazon.com/gp/product/B00EGMGGVK/ref=as_li_tl?ie=UTF8&camp=1789&creative=390957&creativeASIN=B00EGMGGVK&linkCode=as2&tag=92y-20&linkId=V3MMY57QIQ7QVFNK"><em>The World of Ice and Fire: The Untold History of Westeros and the Game of Thrones</em></a>. Exclusively on Tumblr!</p>\n</blockquote>\n<p><span>To reiterate: this an\xa0</span><strong>only</strong><span>\xa0and an\xa0</span><strong>exclusive\xa0</strong><span>and it\xa0</span><strong>starts in just a few minutes</strong><span>. Hurry on over. \xa0</span></p>', u'thumbnail_url': u''} #livestream_result = save_media(session,livestream_post_dict) # Yahoo video yahoo_post_dict = {u'reblog_key': u'GGWw7A77', u'reblog': {u'comment': u'<p>It’s really happening!</p>', u'tree_html': u'<p><a class="tumblr_blog" href="http://whitehouse.tumblr.com/post/88396016693/obamairl">whitehouse</a>:</p><blockquote>\n<p>President Obama is answering your questions on education and college affordability in his first-ever Tumblr Q&A today.</p>\n<p>Tune in right here at 4 p.m. ET, and make sure to follow us @<a class="tumblelog" href="http://tmblr.co/mWgXp6TEB4GEsC_jKXfrSvw">whitehouse</a>.</p>\n</blockquote>', u'trail': [{u'blog': {u'theme': {u'title_font_weight': u'bold', u'header_full_height': 1056, u'title_color': u'#444444', u'header_bounds': u'43,1500,887,0', u'title_font': u'Gibson', u'link_color': u'#529ECC', u'header_image_focused': u'http://static.tumblr.com/861cd9f032b93a7ace681b4fcb7d05e4/mjqkjev/pEEn56435/tumblr_static_tumblr_static_17trsnvc8xes0og8kgk88coc0_focused_v3.jpg', u'show_description': True, u'header_full_width': 1500, u'header_focus_width': 1500, u'header_stretch': True, u'show_header_image': True, u'body_font': u'Helvetica Neue', u'show_title': True, u'header_image_scaled': u'http://static.tumblr.com/861cd9f032b93a7ace681b4fcb7d05e4/mjqkjev/sgln56432/tumblr_static_17trsnvc8xes0og8kgk88coc0_2048_v2.jpg', u'avatar_shape': u'square', u'show_avatar': True, u'header_focus_height': 844, u'background_color': u'#FAFAFA', u'header_image': u'http://static.tumblr.com/861cd9f032b93a7ace681b4fcb7d05e4/mjqkjev/sgln56432/tumblr_static_17trsnvc8xes0og8kgk88coc0.jpg'}, u'name': u'whitehouse'}, u'comment': u'<p>President Obama is answering your questions on education and college affordability in his first-ever Tumblr Q&A today.</p>\n<p>Tune in right here at 4 p.m. ET, and make sure to follow us @<a class="tumblelog" href="http://tmblr.co/mWgXp6TEB4GEsC_jKXfrSvw">whitehouse</a>.</p>', u'post': {u'id': u'88396016693'}}]}, u'thumbnail_width': 320, u'player': [{u'width': 250, u'embed_code': u'<iframe width="250" height="140" src="https://news.yahoo.com/video/tumblr-goes-white-house-190000218.html?format=embed" frameborder="0" allowfullscreen></iframe>'}, {u'width': 400, u'embed_code': u'<iframe width="400" height="225" src="https://news.yahoo.com/video/tumblr-goes-white-house-190000218.html?format=embed" frameborder="0" allowfullscreen></iframe>'}, {u'width': 500, u'embed_code': u'<iframe width="500" height="281" src="https://news.yahoo.com/video/tumblr-goes-white-house-190000218.html?format=embed" frameborder="0" allowfullscreen></iframe>'}], u'id': 88400573880, u'post_url': u'http://staff.tumblr.com/post/88400573880/whitehouse-president-obama-is-answering-your', u'tags': [u'ObamaIRL'], u'highlighted': [], u'state': u'published', u'short_url': u'http://tmblr.co/ZE5Fby1IL5RMu', u'html5_capable': True, u'type': u'video', u'format': u'html', u'timestamp': 1402430040, u'note_count': 9899, u'video_type': u'yahoo', u'date': u'2014-06-10 19:54:00 GMT', u'thumbnail_height': 180, u'permalink_url': u'https://news.yahoo.com/video/tumblr-goes-white-house-190000218.html', u'slug': u'whitehouse-president-obama-is-answering-your', u'blog_name': u'staff', u'caption': u'<p><a class="tumblr_blog" href="http://whitehouse.tumblr.com/post/88396016693/obamairl">whitehouse</a>:</p>\n<blockquote>\n<p>President Obama is answering your questions on education and college affordability in his first-ever Tumblr Q&A today.</p>\n<p>Tune in right here at 4 p.m. ET, and make sure to follow us @<a class="tumblelog" href="http://tmblr.co/mWgXp6TEB4GEsC_jKXfrSvw">whitehouse</a>.</p>\n</blockquote>\n<p>It\u2019s really happening!</p>', u'thumbnail_url': u'https://s1.yimg.com/uu/api/res/1.2/JW58D_.UFfRLkBOrIemIXw--/dz0zMjA7c209MTtmaT1maWxsO3B5b2ZmPTA7aD0xODA7YXBwaWQ9eXRhY2h5b24-/http://l.yimg.com/os/publish-images/ivy/2014-06-10/912811c0-f0c6-11e3-bb53-bd3ad1c7b3ec_06102014_tumblr_white_house.jpg'} yahoo_result = save_media(session,yahoo_post_dict) logging.debug("Closing DB session") session.commit() return
def debug(): """Temp code for debug""" session = sql_functions.connect_to_db() read_post( session, post_url= "http://askbuttonsmom.tumblr.com/post/118625666843/happy-mothers-day")
def reset_rows(): """Reset processed flag to False""" session = sql_functions.connect_to_db() update_statement = sqlalchemy.update(RawPosts).where(RawPosts.media_processed == True).\ values(media_processed = False) session.execute(update_statement) session.commit() return
def debug(): """For WIP, debug, ect function calls""" session = sql_functions.connect_to_db() result = download_image_links(session, media_urls = ["http://blog.crooz.jp/svc/userFrontArticle/ShowFiles/?no=1538&blog_id=53800&file_str=5380015381862e669e0073d13d4175ecae9d5a34b8ff05fe3.jpg&guid=on&vga_flg=0&y=2014&m=02&d=19&wid=480&hei=640", "http://static.tumblr.com/f6539f27dff5045834f7722e61c02e21/w5cnjnh/fVJnpuk9d/tumblr_static_digrsyxj0eg400so4g4kgsc4k.jpg", "https://dl.dropboxusercontent.com/u/27379736/NSFWSFM/SWF/TwilightSoloLightParticles.swf" ] ) logging.debug(result)
def delete_broken_posts(): """Remove posts that do not have the processed flag set to true in the rawposts table""" session = sql_functions.connect_to_db() # -Posts with status mismatch- # Each post row must have the same timestamp, blog url, and origin id as a rawposts row # The rawposts row must have media_processed == True # Choose a blog blog_statement = sqlalchemy.select([blogs]).\ where(blogs.blog_id < 100) blog_rows = session.execute(blog_statement) blog_row = blog_rows.fetchone() logging.debug(repr(blog_row)) blog_id = 1 # For each post from this blog select_posts_statement = sqlalchemy.select([posts]).\ where(posts.blog_id == blog_id).\ limit(100) post_rows = session.execute(select_posts_statement) for post_row in post_rows: logging.debug("post_row:" + repr(post_row)) post_id = post_row["post_id"] # Grab the matching row from rawposts compare_statement = sqlalchemy.select([RawPosts]).\ where(post_row.source_url == RawPosts.all_posts_post_url).\ where(post_row.timestamp == RawPosts.all_posts_timestamp).\ limit(10) compare_rows = session.execute(compare_statement) compare_row = compare_rows.fetchone() logging.debug("compare_row:" + repr(compare_row)) # If post row exists but processed flag is not true, delete post if compare_row["media_processed"] == False: logging.debug("Bad") # Delete post delete_post_statement = sqlalchemy.select([posts]).\ where(posts.post_id == post_id) session.execute(delete_post_statement) continue else: logging.debug("Good") continue # Delete child rows # Delete post row # Commit changes session.commit()
def main(): try: setup_logging(log_file_path=os.path.join("debug", "display_post-log.txt"), concise_log_file_path=os.path.join( "debug", "short-display_post-log.txt")) # Program session = sql_functions.connect_to_db() build_all_posts(session) ## # Generate lists of what we have ## list_domain_posts( ## session, ## blog_domain = "askbuttonsmom.tumblr.com", ## output_path="debug\\list_posts.txt" ## ) ## ## list_blogs( ## session, ## output_path="debug\\blog_list.txt" ## ) ## ## list_all_media( ## session, ## output_path="debug\\media_list.txt" ## ) ## ## # Test generating posts for display ## ## display_post( ## session, ## source_id = 117088487878L, ## output_path="debug\\post_117088487878L.txt", ## ) ## ## display_post( ## session, ## source_id = 118863575131L, ## output_path="debug\\post_118863575131L.txt", ## ) ## ## display_post( ## session, ## source_id = 118863575131, ## output_path="debug\\post_118863575131.txt", ## ) # /Program logging.info("Finished, exiting.") return except Exception, e: # Log fatal exceptions logging.critical("Unhandled exception!") logging.exception(e)
def debug(): """For WIP, debug, ect function calls""" session = sql_functions.connect_to_db() result = download_image_links( session, media_urls=[ "http://blog.crooz.jp/svc/userFrontArticle/ShowFiles/?no=1538&blog_id=53800&file_str=5380015381862e669e0073d13d4175ecae9d5a34b8ff05fe3.jpg&guid=on&vga_flg=0&y=2014&m=02&d=19&wid=480&hei=640", "http://static.tumblr.com/f6539f27dff5045834f7722e61c02e21/w5cnjnh/fVJnpuk9d/tumblr_static_digrsyxj0eg400so4g4kgsc4k.jpg", "https://dl.dropboxusercontent.com/u/27379736/NSFWSFM/SWF/TwilightSoloLightParticles.swf" ]) logging.debug(result)
def add_blogs(list_file_path="tumblr_todo_list.txt"): """Save tumblr blogs from a list""" logging.info("Adding blogs from list file: " + repr(list_file_path)) # Get the list of blogs we want to add blog_url_list = import_blog_list(list_file_path) # Connect to DB session = sql_functions.connect_to_db() # Process each URL for blog_url in blog_url_list: add_blog(session, blog_url) logging.info("Finished downloading blogs list") return
def debug(): """Temp code for debug""" session = sql_functions.connect_to_db() blog_urls = [ #"staff.tumblr.com", #"testsetsts2.tumblr.com", "lunarshinestore.tumblr.com", "atryl.tumblr.com", ] for blog_url in blog_urls: dummy_blog_id = sql_functions.add_blog(session,blog_url) logging.debug("dummy_blog_id: "+repr(dummy_blog_id)) update_blog_theme(session,blog_url) continue return
def add_blogs(list_file_path="tumblr_todo_list.txt"): """Save tumblr blogs from a list""" logging.info("Adding blogs from list file: "+repr(list_file_path)) # Get the list of blogs we want to add blog_url_list = import_blog_list(list_file_path) # Connect to DB session = sql_functions.connect_to_db() # Process each URL for blog_url in blog_url_list: add_blog(session,blog_url) logging.info("Finished downloading blogs list") return
def debug(): """Temp code for debug""" session = sql_functions.connect_to_db() blog_urls = [ #"staff.tumblr.com", #"testsetsts2.tumblr.com", "lunarshinestore.tumblr.com", "atryl.tumblr.com", ] for blog_url in blog_urls: dummy_blog_id = sql_functions.add_blog(session, blog_url) logging.debug("dummy_blog_id: " + repr(dummy_blog_id)) update_blog_theme(session, blog_url) continue return
def main(): try: setup_logging(log_file_path=os.path.join("debug","asagi_vocaroo_grabber-log.txt")) session = sql_functions.connect_to_db() scan_db( session=session, output_dir = config.output_dir, start_id = config.start_id, stop_id = config.stop_id, step_number = config.step_number ) except Exception, e:# Log fatal exceptions logging.critical("Unhandled exception!") logging.exception(e)
def process_posts(blog_domain): """Find all posts for a given blog""" # Connect to the DB session = sql_functions.connect_to_db() # Grab the posts posts_query = sqlalchemy.select([RawPosts]).\ where(RawPosts.blog_domain == blog_domain) posts_rows = session.execute(posts_query) found_blog_links = [] # Process each post counter = 0 for posts_row in posts_rows: counter += 1 logging.debug("Processing post "+repr(counter)) # Grab post data from DB post_dict = posts_row["raw_post_json"] # Stringify post data post_string = repr(post_dict) # Parse out links blog_links_in_post = find_blog_links(post_string) logging.debug("blog_links_in_post: "+repr(blog_links_in_post)) # Append found blogs to list found_blog_links += blog_links_in_post # Clean up links logging.debug("found_blog_links: "+repr(found_blog_links)) kept_blog_links = remove_unwanted_links(found_blog_links) logging.debug("kept_blog_links: "+repr(kept_blog_links)) output_list = uniquify(kept_blog_links) logging.debug("output_list: "+repr(output_list)) timestamp_string = datetime.datetime.utcnow().strftime("%Y-%m-%d %H.%M.%S%Z") # Save found links to file appendlist( output_list, list_file_path="linked_blogs_"+blog_domain+"_"+timestamp_string+".txt", initial_text="# List of found blog_links.\n" ) return
def post_consumer(post_queue): "Process posts for a queue-like object containing post dicts" logging.debug("Consumer function started.") # Connect to DB database_session = sql_functions.connect_to_db() # Process posts from the queue c = 0 # Counter for number of posts processed while True: c += 1 if c % 100 == 0: logging.info(repr(c) + " posts processed by this process") post_row = post_queue.get(timeout=600) if post_row is None: # Stop if None object is put into the queue logging.info("Post consumer recieved None object as exit signal") break # Stop doing work and exit thread/process process_one_new_posts_media(database_session, post_row) continue # Disconnect from DB database_session.close() logging.debug("Consumer function exiting.") return
def post_consumer(post_queue): "Process posts for a queue-like object containing post dicts" logging.debug("Consumer function started.") # Connect to DB database_session = sql_functions.connect_to_db() # Process posts from the queue c = 0# Counter for number of posts processed while True: c += 1 if c%100 == 0: logging.info(repr(c)+" posts processed by this process") post_row = post_queue.get(timeout=600) if post_row is None:# Stop if None object is put into the queue logging.info("Post consumer recieved None object as exit signal") break# Stop doing work and exit thread/process process_one_new_posts_media(database_session,post_row) continue # Disconnect from DB database_session.close() logging.debug("Consumer function exiting.") return
def list_blogs(): """Return a list of up to maximum_blogs blogids""" # Connect to DB session = sql_functions.connect_to_db() # Load new blogs # date_posts_last_saved NULL select_null_query = sqlalchemy.select([blogs]).\ where(blogs.date_posts_last_saved == sqlalchemy.null()).\ order_by(blogs.date_posts_last_saved.asc()) null_blogs_rows = session.execute(select_null_query) # Grab blog URLs null_blog_urls = [] for null_blogs_row in null_blogs_rows: null_blog_url = null_blogs_row["blog_url"] null_blog_urls += [null_blog_url] logging.debug("len(null_blog_urls):"+repr(len(null_blog_urls))) # date_posts_last_saved NOT NULL select_query = sqlalchemy.select([blogs]).\ where(blogs.date_posts_last_saved != sqlalchemy.null()).\ order_by(blogs.date_posts_last_saved.asc()) not_null_blogs_rows = session.execute(select_query) # Grab blog URLs not_null_blog_urls = [] for not_null_blogs_row in not_null_blogs_rows: not_null_blog_url = not_null_blogs_row["blog_url"] not_null_blog_urls += [not_null_blog_url] logging.debug("len(not_null_blog_urls):"+repr(len(not_null_blog_urls))) # Disconnect from DB session.close() blog_urls = null_blog_urls+not_null_blog_urls return blog_urls
def debug(): """where stuff is called to debug and test""" ## save_vocaroo_link( ## vocaroo_link="http://vocaroo.com/i/s0xtktsit8rE", ## output_dir=os.path.join("debug", "output"), ## ) session = sql_functions.connect_to_db() ## find_vocaroo_links_in_db( ## session=session, ## start_id=0, ## stop_id=1000, ## output_dir=os.path.join("debug", "output") ## ) scan_db( session=session, output_dir=os.path.join("output"), start_id=0, stop_id=None, step_number=1000 ) return
def debug(): """Temp code for debug""" session = sql_functions.connect_to_db() read_post(session,post_url="http://askbuttonsmom.tumblr.com/post/118625666843/happy-mothers-day")
def drop_tables(): """Drop all tables except for media and rawposts""" session = sql_functions.connect_to_db() Base.metadata.drop_all()
def delete_broken_posts(): """Remove posts that do not have the processed flag set to true in the rawposts table""" session = sql_functions.connect_to_db() # -Posts with status mismatch- # Each post row must have the same timestamp, blog url, and origin id as a rawposts row # The rawposts row must have media_processed == True # Choose a blog blog_statement = sqlalchemy.select([blogs]).\ where(blogs.blog_id < 100) blog_rows = session.execute(blog_statement) blog_row = blog_rows.fetchone() logging.debug(repr(blog_row)) blog_id = 1 # For each post from this blog select_posts_statement = sqlalchemy.select([posts]).\ where(posts.blog_id == blog_id).\ limit(100) post_rows = session.execute(select_posts_statement) for post_row in post_rows: logging.debug("post_row:"+repr(post_row)) post_id = post_row["post_id"] # Grab the matching row from rawposts compare_statement = sqlalchemy.select([RawPosts]).\ where(post_row.source_url == RawPosts.all_posts_post_url).\ where(post_row.timestamp == RawPosts.all_posts_timestamp).\ limit(10) compare_rows = session.execute(compare_statement) compare_row = compare_rows.fetchone() logging.debug("compare_row:"+repr(compare_row)) # If post row exists but processed flag is not true, delete post if compare_row["media_processed"] == False: logging.debug("Bad") # Delete post delete_post_statement = sqlalchemy.select([posts]).\ where(posts.post_id == post_id) session.execute(delete_post_statement) continue else: logging.debug("Good") continue # Delete child rows # Delete post row # Commit changes session.commit()
def main(): session = sql_functions.connect_to_db() dump_posts( session, output_file_path=os.path.join("debug","raw_posts_dump.txt") )
def debug(): """For WIP, debug, ect function calls""" session = sql_functions.connect_to_db() postimg_result = save_postimg_gallery( session, link="http://postimg.org/gallery/1g80elqce/") logging.debug("postimg_result") return #handle_generic_link(session,link="media.tumblr.com/80638dc40978c286b5d3f18b2bfcf3d6/tumblr_inline_mfb0pxs7lf1qfi5b1.gif") #fastswf_result = handle_fastswf_link(session,"http://www.fastswf.com/5MH9MZw") #return ## result = handle_links( ## session, ## post_dict = {u'highlighted': [], u'asking_url': None, u'reblog_key': u'O3UPhQYC', u'format': u'html', u'asking_name': u'Anonymous', u'timestamp': 1430518955, u'note_count': 43253, u'tags': [], u'question': u'lauren will you tell us a story', u'trail': [{u'blog': {u'theme': {u'title_font_weight': u'bold', u'header_full_height': 1226, u'title_color': u'#8f1e03', u'header_bounds': u'87,647,433,31', u'title_font': u'Courier New', u'link_color': u'#ffdd00', u'header_image_focused': u'http://static.tumblr.com/cafca27bb4f69629bf936818efb44e6f/88apzkd/fEXnhmx2i/tumblr_static_tumblr_static_983gvr4ppg4cgkgcs8c8owk4o_focused_v3.png', u'show_description': True, u'header_full_width': 677, u'header_focus_width': 616, u'header_stretch': True, u'show_header_image': True, u'body_font': u'Helvetica Neue', u'show_title': True, u'header_image_scaled': u'http://static.tumblr.com/cafca27bb4f69629bf936818efb44e6f/88apzkd/h29nhmx2f/tumblr_static_983gvr4ppg4cgkgcs8c8owk4o_2048_v2.png', u'avatar_shape': u'circle', u'show_avatar': True, u'header_focus_height': 346, u'background_color': u'#f38321', u'header_image': u'http://static.tumblr.com/cafca27bb4f69629bf936818efb44e6f/88apzkd/h29nhmx2f/tumblr_static_983gvr4ppg4cgkgcs8c8owk4o.png'}, u'name': u'iguanamouth'}, u'content': u'<p>ok heres something</p><p>around a year ago someone asked me to draw <a href="http://iguanamouth.tumblr.com/post/82902281208/draw-danny-devito-as-a-kitty">danny devito as a kitty</a>, spawning this terrible terrible image\xa0</p><figure data-orig-width="500" data-orig-height="406" class="tmblr-full"><img src="https://41.media.tumblr.com/d7549754fd8c427d3393eb702939652f/tumblr_inline_nnld4fWtJH1qmoaae_540.png" alt="image" data-orig-width="500" data-orig-height="406"></figure><p>time passes. a lot of time passes. then two months ago i get an email from a group of people called <a href="https://www.facebook.com/fpoafm">FPOAFM</a> doing a pottery installation event, and theyre going around gathering artwork from artists to put onto cups and dishes to sell,, in exchange for a few pieces with the artists work on them</p><p>and i said SURE you can use some of my stuff \u2026 . but in exchange.\u2026 <b>\xa0i want something with kitty devito on it. </b>i dont care if you put it on anything else, but one item that i get in return has to have this cat man abomination</p><p>i give them my address and a few images and months pass. i forget about it. THEN literally two days ago i get this big package on my doorstep, and INSIDE OF IT\u2026. is the holy grail</p><p>in addition to <a href="https://40.media.tumblr.com/cdd10d787d5bd9c4ab797fca37671cb0/tumblr_nneimxvAoS1rhtthso2_r3_1280.png">two</a> <a href="https://40.media.tumblr.com/7e01bf6e04497f0c9da80ed0b83cc3df/tumblr_nneimxvAoS1rhtthso3_r2_540.png">plates</a> is this incredible porcelain cup with the fabled kitty devito on it, proudly grinning his terrible cat grin</p><figure data-orig-width="800" data-orig-height="885" class="tmblr-full"><img src="https://41.media.tumblr.com/aca0ef3ca89612a513ef9143a439b48d/tumblr_inline_nnldwbZNR91qmoaae_540.png" alt="image" data-orig-width="800" data-orig-height="885"></figure><p>the thing that pushes this cup into the Far Reaches of Awful isnt just the image stamped on it. its that it is one hundred percent made from a mold of a styrofoam cup</p><figure data-orig-width="800" data-orig-height="600" class="tmblr-full"><img src="https://40.media.tumblr.com/7ce202e1b1877d34c0211876cdbb575e/tumblr_inline_nnleakjIpO1qmoaae_540.png" alt="image" data-orig-width="800" data-orig-height="600"></figure><p>its finger presses on the rim, those little lines going around</p><figure data-orig-width="800" data-orig-height="600" class="tmblr-full"><img src="https://36.media.tumblr.com/75f6a2b06915bac6aec356d18f226a47/tumblr_inline_nnleefM9Ty1qmoaae_540.png" alt="image" data-orig-width="800" data-orig-height="600"></figure><p>and all this jargon on the bottom, right under the glaze. the amount of effort that went into reproducing this styrofoam cup is incredible and i can stick it in my shelf and drink soup from it at four in the morning with danny devitos smug cat face looking out over everything i do, forever. follow your dreams</p>', u'post': {u'id': u'117728731307'}, u'is_root_item': True}, {u'blog': {u'theme': {u'title_font_weight': u'bold', u'title_color': u'#FFFFFF', u'header_bounds': 0, u'title_font': u'Gibson', u'link_color': u'#56BC8A', u'header_image_focused': u'http://static.tumblr.com/10e607f9b9b4c588d1cbd6c9f8b564d9/3hpyv0p/nFBnlgttl/tumblr_static_1sssiwavcjs0gs4cggwsok040_2048_v2.gif', u'show_description': True, u'show_header_image': True, u'header_stretch': True, u'body_font': u'Helvetica Neue', u'show_title': True, u'header_image_scaled': u'http://static.tumblr.com/10e607f9b9b4c588d1cbd6c9f8b564d9/3hpyv0p/nFBnlgttl/tumblr_static_1sssiwavcjs0gs4cggwsok040_2048_v2.gif', u'avatar_shape': u'square', u'show_avatar': False, u'background_color': u'#37475c', u'header_image': u'http://static.tumblr.com/10e607f9b9b4c588d1cbd6c9f8b564d9/3hpyv0p/nFBnlgttl/tumblr_static_1sssiwavcjs0gs4cggwsok040.gif'}, u'name': u'staff'}, u'content': u'<p>Have a follow-your-dreams weekend, Tumblr.</p>', u'post': {u'id': u'117887933455'}, u'is_current_item': True}], u'id': 117887933455L, u'post_url': u'http://staff.tumblr.com/post/117887933455/lauren-will-you-tell-us-a-story', u'answer': u'<p><a class="tumblr_blog" href="http://iguanamouth.tumblr.com/post/117728731307/lauren-will-you-tell-us-a-story">iguanamouth</a>:</p>\n\n<blockquote><p>ok heres something</p><p>around a year ago someone asked me to draw <a href="http://iguanamouth.tumblr.com/post/82902281208/draw-danny-devito-as-a-kitty">danny devito as a kitty</a>, spawning this terrible terrible image\xa0</p><figure data-orig-width="500" data-orig-height="406" class="tmblr-full"><img src="https://41.media.tumblr.com/d7549754fd8c427d3393eb702939652f/tumblr_inline_nnld4fWtJH1qmoaae_540.png" alt="image" data-orig-width="500" data-orig-height="406"/></figure><p>time passes. a lot of time passes. then two months ago i get an email from a group of people called <a href="https://www.facebook.com/fpoafm">FPOAFM</a> doing a pottery installation event, and theyre going around gathering artwork from artists to put onto cups and dishes to sell,, in exchange for a few pieces with the artists work on them</p><p>and i said SURE you can use some of my stuff \u2026 . but in exchange.\u2026 <b>\xa0i want something with kitty devito on it. </b>i dont care if you put it on anything else, but one item that i get in return has to have this cat man abomination</p><p>i give them my address and a few images and months pass. i forget about it. THEN literally two days ago i get this big package on my doorstep, and INSIDE OF IT\u2026. is the holy grail</p><p>in addition to <a href="https://40.media.tumblr.com/cdd10d787d5bd9c4ab797fca37671cb0/tumblr_nneimxvAoS1rhtthso2_r3_1280.png">two</a> <a href="https://40.media.tumblr.com/7e01bf6e04497f0c9da80ed0b83cc3df/tumblr_nneimxvAoS1rhtthso3_r2_540.png">plates</a> is this incredible porcelain cup with the fabled kitty devito on it, proudly grinning his terrible cat grin</p><figure data-orig-width="800" data-orig-height="885" class="tmblr-full"><img src="https://41.media.tumblr.com/aca0ef3ca89612a513ef9143a439b48d/tumblr_inline_nnldwbZNR91qmoaae_540.png" alt="image" data-orig-width="800" data-orig-height="885"/></figure><p>the thing that pushes this cup into the Far Reaches of Awful isnt just the image stamped on it. its that it is one hundred percent made from a mold of a styrofoam cup</p><figure data-orig-width="800" data-orig-height="600" class="tmblr-full"><img src="https://40.media.tumblr.com/7ce202e1b1877d34c0211876cdbb575e/tumblr_inline_nnleakjIpO1qmoaae_540.png" alt="image" data-orig-width="800" data-orig-height="600"/></figure><p>its finger presses on the rim, those little lines going around</p><figure data-orig-width="800" data-orig-height="600" class="tmblr-full"><img src="https://36.media.tumblr.com/75f6a2b06915bac6aec356d18f226a47/tumblr_inline_nnleefM9Ty1qmoaae_540.png" alt="image" data-orig-width="800" data-orig-height="600"/></figure><p>and all this jargon on the bottom, right under the glaze. the amount of effort that went into reproducing this styrofoam cup is incredible and i can stick it in my shelf and drink soup from it at four in the morning with danny devitos smug cat face looking out over everything i do, forever. follow your dreams</p></blockquote><p><p>Have a follow-your-dreams weekend, Tumblr.</p></p>', u'state': u'published', u'reblog': {u'tree_html': u'<p><a class="tumblr_blog" href="http://iguanamouth.tumblr.com/post/117728731307/lauren-will-you-tell-us-a-story">iguanamouth</a>:</p><blockquote><p>ok heres something</p><p>around a year ago someone asked me to draw <a href="http://iguanamouth.tumblr.com/post/82902281208/draw-danny-devito-as-a-kitty">danny devito as a kitty</a>, spawning this terrible terrible image\xa0</p><figure data-orig-width="500" data-orig-height="406" class="tmblr-full"><img src="https://41.media.tumblr.com/d7549754fd8c427d3393eb702939652f/tumblr_inline_nnld4fWtJH1qmoaae_540.png" alt="image" data-orig-width="500" data-orig-height="406"/></figure><p>time passes. a lot of time passes. then two months ago i get an email from a group of people called <a href="https://www.facebook.com/fpoafm">FPOAFM</a> doing a pottery installation event, and theyre going around gathering artwork from artists to put onto cups and dishes to sell,, in exchange for a few pieces with the artists work on them</p><p>and i said SURE you can use some of my stuff \u2026 . but in exchange.\u2026 <b>\xa0i want something with kitty devito on it. </b>i dont care if you put it on anything else, but one item that i get in return has to have this cat man abomination</p><p>i give them my address and a few images and months pass. i forget about it. THEN literally two days ago i get this big package on my doorstep, and INSIDE OF IT\u2026. is the holy grail</p><p>in addition to <a href="https://40.media.tumblr.com/cdd10d787d5bd9c4ab797fca37671cb0/tumblr_nneimxvAoS1rhtthso2_r3_1280.png">two</a> <a href="https://40.media.tumblr.com/7e01bf6e04497f0c9da80ed0b83cc3df/tumblr_nneimxvAoS1rhtthso3_r2_540.png">plates</a> is this incredible porcelain cup with the fabled kitty devito on it, proudly grinning his terrible cat grin</p><figure data-orig-width="800" data-orig-height="885" class="tmblr-full"><img src="https://41.media.tumblr.com/aca0ef3ca89612a513ef9143a439b48d/tumblr_inline_nnldwbZNR91qmoaae_540.png" alt="image" data-orig-width="800" data-orig-height="885"/></figure><p>the thing that pushes this cup into the Far Reaches of Awful isnt just the image stamped on it. its that it is one hundred percent made from a mold of a styrofoam cup</p><figure data-orig-width="800" data-orig-height="600" class="tmblr-full"><img src="https://40.media.tumblr.com/7ce202e1b1877d34c0211876cdbb575e/tumblr_inline_nnleakjIpO1qmoaae_540.png" alt="image" data-orig-width="800" data-orig-height="600"/></figure><p>its finger presses on the rim, those little lines going around</p><figure data-orig-width="800" data-orig-height="600" class="tmblr-full"><img src="https://36.media.tumblr.com/75f6a2b06915bac6aec356d18f226a47/tumblr_inline_nnleefM9Ty1qmoaae_540.png" alt="image" data-orig-width="800" data-orig-height="600"/></figure><p>and all this jargon on the bottom, right under the glaze. the amount of effort that went into reproducing this styrofoam cup is incredible and i can stick it in my shelf and drink soup from it at four in the morning with danny devitos smug cat face looking out over everything i do, forever. follow your dreams</p></blockquote>'}, u'short_url': u'http://tmblr.co/ZE5Fby1jognmF', u'date': u'2015-05-01 22:22:35 GMT', u'type': u'answer', u'slug': u'lauren-will-you-tell-us-a-story', u'blog_name': u'staff'} ## ) ## logging.debug("result:"+repr(result)) # Dropbox error case dropboxusercontent_post_dict = { u'reblog_key': u'w7hVL658', u'reblog': { u'comment': u'', u'tree_html': u'<p><a href="http://verysaltyonions.tumblr.com/post/82259171463/fruitymilkstuff-im-terrible-at-being-lazy-so" class="tumblr_blog" target="_blank">verysaltyonions</a>:</p><blockquote><p><a href="http://fruitymilk.co.uk/post/81236468626/im-terrible-at-being-lazy-so-much-for-my-no" class="tumblr_blog" target="_blank">fruitymilkstuff</a>:</p>\n\n<blockquote><p><em><strong>I\u2019m terrible at being lazy.</strong></em>\xa0So much for my \u2018no content\u2019 weekend.</p>\n<p>I used some time last night to experiment with particles and lights coming off said particles. Magic in this case. Nothing spectacular though!</p>\n<p><em>I was testing out particles some more, trying to figure out how to make them loop and stuff. Turns out I can manage it! That and lighting to match them.\xa0</em></p>\n<p><strong><em>As usual click-through for a MUCH higher quality SWF file!</em></strong></p></blockquote>\n\n<p>Unusual futa-c**k</p></blockquote>' }, u'id': 82654349960L, u'post_url': u'http://aegissp.tumblr.com/post/82654349960/verysaltyonions-fruitymilkstuff-im-terrible', u'source_title': u'fruitymilkstuff', u'image_permalink': u'http://aegissp.tumblr.com/image/82654349960', u'tags': [ u'NSFW', u'nsfw:twilight sparkle', u'futa', u'futanari', u'rule 34', u'clop', u'media' ], u'highlighted': [], u'recommended_source': None, u'state': u'published', u'short_url': u'http://tmblr.co/ZeuQFu1C_bKg8', u'type': u'photo', u'format': u'html', u'timestamp': 1397446549, u'note_count': 348, u'source_url': u'http://fruitymilkstuff.tumblr.com/post/81236468626/im-terrible-at-being-lazy-so-much-for-my-no', u'photos': [{ u'caption': u'', u'original_size': { u'url': u'http://33.media.tumblr.com/894426e6f4f1b90b330e986abc26bb00/tumblr_n39x44yu4P1tpeknfo1_500.gif', u'width': 445, u'height': 250 }, u'alt_sizes': [{ u'url': u'http://33.media.tumblr.com/894426e6f4f1b90b330e986abc26bb00/tumblr_n39x44yu4P1tpeknfo1_500.gif', u'width': 445, u'height': 250 }, { u'url': u'http://33.media.tumblr.com/894426e6f4f1b90b330e986abc26bb00/tumblr_n39x44yu4P1tpeknfo1_400.gif', u'width': 400, u'height': 225 }, { u'url': u'http://38.media.tumblr.com/894426e6f4f1b90b330e986abc26bb00/tumblr_n39x44yu4P1tpeknfo1_250.gif', u'width': 250, u'height': 140 }, { u'url': u'http://33.media.tumblr.com/894426e6f4f1b90b330e986abc26bb00/tumblr_n39x44yu4P1tpeknfo1_100.gif', u'width': 100, u'height': 56 }, { u'url': u'http://33.media.tumblr.com/894426e6f4f1b90b330e986abc26bb00/tumblr_n39x44yu4P1tpeknfo1_75sq.gif', u'width': 75, u'height': 75 }] }], u'date': u'2014-04-14 03:35:49 GMT', u'slug': u'verysaltyonions-fruitymilkstuff-im-terrible', u'blog_name': u'aegissp', u'trail': [{ u'blog': { u'active': True, u'theme': { u'title_font_weight': u'bold', u'title_color': u'#FFFFFF', u'header_bounds': 0, u'title_font': u'Streetscript', u'link_color': u'#529ECC', u'header_image_focused': u'http://static.tumblr.com/ef1069245069af066df2ac8411f241c9/4it4dnn/Ipwnk5eq8/tumblr_static_8cqn4964iow0ok8gw80c4cwg8_2048_v2.png', u'show_description': True, u'show_header_image': True, u'body_font': u'Helvetica Neue', u'show_title': True, u'header_stretch': False, u'avatar_shape': u'circle', u'show_avatar': True, u'background_color': u'#b01e1e', u'header_image': u'http://static.tumblr.com/ef1069245069af066df2ac8411f241c9/4it4dnn/Ipwnk5eq8/tumblr_static_8cqn4964iow0ok8gw80c4cwg8.png', u'header_image_scaled': u'http://static.tumblr.com/ef1069245069af066df2ac8411f241c9/4it4dnn/Ipwnk5eq8/tumblr_static_8cqn4964iow0ok8gw80c4cwg8_2048_v2.png' }, u'name': u'fruitymilkstuff' }, u'content': u'<p><em><strong>I\u2019m terrible at being lazy.</strong></em>\xa0So much for my \u2018no content\u2019 weekend.</p>\n<p>I used some time last night to experiment with particles and lights coming off said particles. Magic in this case. Nothing spectacular though!</p>\n<p><em>I was testing out particles some more, trying to figure out how to make them loop and stuff. Turns out I can manage it! That and lighting to match them.\xa0</em></p>\n<p><strong><em>As usual click-through for a MUCH higher quality SWF file!</em></strong></p>', u'post': { u'id': u'81236468626' }, u'is_root_item': True }, { u'blog': { u'active': True, u'theme': { u'title_font_weight': u'bold', u'title_color': u'#444444', u'header_bounds': u'', u'title_font': u'Helvetica Neue', u'link_color': u'#FFA200', u'header_image_focused': u'http://assets.tumblr.com/images/default_header/optica_pattern_08_focused_v3.png?_v=f0f055039bb6136b9661cf2227b535c2', u'show_description': True, u'show_header_image': True, u'body_font': u'Helvetica Neue', u'show_title': True, u'header_stretch': True, u'avatar_shape': u'circle', u'show_avatar': True, u'background_color': u'#ffffff', u'header_image': u'http://assets.tumblr.com/images/default_header/optica_pattern_08.png?_v=f0f055039bb6136b9661cf2227b535c2', u'header_image_scaled': u'http://assets.tumblr.com/images/default_header/optica_pattern_08_focused_v3.png?_v=f0f055039bb6136b9661cf2227b535c2' }, u'name': u'verysaltyonions' }, u'content': u'<p>Unusual futa-c**k</p>', u'post': { u'id': u'82259171463' } }], u'link_url': u'https://dl.dropboxusercontent.com/u/27379736/NSFWSFM/SWF/TwilightSoloLightParticles.swf', u'caption': u'<p><a href="http://verysaltyonions.tumblr.com/post/82259171463/fruitymilkstuff-im-terrible-at-being-lazy-so" class="tumblr_blog" target="_blank">verysaltyonions</a>:</p>\n\n<blockquote><p><a href="http://fruitymilk.co.uk/post/81236468626/im-terrible-at-being-lazy-so-much-for-my-no" class="tumblr_blog" target="_blank">fruitymilkstuff</a>:</p>\n\n<blockquote><p><em><strong>I\u2019m terrible at being lazy.</strong></em>\xa0So much for my \u2018no content\u2019 weekend.</p>\n<p>I used some time last night to experiment with particles and lights coming off said particles. Magic in this case. Nothing spectacular though!</p>\n<p><em>I was testing out particles some more, trying to figure out how to make them loop and stuff. Turns out I can manage it! That and lighting to match them.\xa0</em></p>\n<p><strong><em>As usual click-through for a MUCH higher quality SWF file!</em></strong></p></blockquote>\n\n<p>Unusual futa-c**k</p></blockquote>' } dropboxusercontent_result = handle_links( session, post_dict=dropboxusercontent_post_dict) logging.debug("dropboxusercontent_result:" + repr(dropboxusercontent_result)) # malformed link case malformed_link_post_dict = { u'reblog_key': u'wlQEgKpb', u'reblog': { u'comment': u'', u'tree_html': u'<p><a href="http://taboolicious.tumblr.com/post/118601139146/happy-milf-day-many-thanks-for-all-those-that" class="tumblr_blog">taboolicious</a>:</p><blockquote><p>happy milf day! many thanks for all those that where present in the raffle and suggested such hot milfs :D</p><p>here is the wallpaper: 1920x1200</p><p><a href="https://dl.dropboxusercontent.com/u/1834642/milf%20paper.jpg">https://dl.dropboxusercontent.com/u/1834642/milf%20paper.jpg</a></p><p>from left to right:</p><p>bananacreamcakecomic: Jenna OC</p><p>jacquesthebird: Kate OC</p><p>damabiath: android 18</p><p>bellend08: Molly Cosset OC</p><p>vic79: Mystique</p><p>ravenlordmercenary: Grace pokemon X/Y</p><p>Th3A1chemist: Vuong OC</p><p>Evenanthy: Harley Quinn</p><p><br/></p><p>if any of the winners wants a wallpaper with just his/her requested girl, pm me :)</p></blockquote>' }, u'id': 118615261815L, u'post_url': u'http://dclzexonask.tumblr.com/post/118615261815/taboolicious-happy-milf-day-many-thanks-for', u'source_title': u'taboolicious', u'image_permalink': u'http://dclzexonask.tumblr.com/image/118615261815', u'tags': [], u'highlighted': [], u'recommended_source': None, u'state': u'published', u'short_url': u'http://tmblr.co/Z7jrQx1kU1K9t', u'type': u'photo', u'format': u'html', u'timestamp': 1431273428, u'note_count': 510, u'source_url': u'http://taboolicious.tumblr.com/post/118601139146/happy-milf-day-many-thanks-for-all-those-that', u'photos': [{ u'caption': u'', u'original_size': { u'url': u'http://40.media.tumblr.com/4f98bc55f60c925f6755fa1a272ec6d7/tumblr_no4u7pEOCl1s36m6bo1_1280.jpg', u'width': 1280, u'height': 800 }, u'alt_sizes': [{ u'url': u'http://40.media.tumblr.com/4f98bc55f60c925f6755fa1a272ec6d7/tumblr_no4u7pEOCl1s36m6bo1_1280.jpg', u'width': 1280, u'height': 800 }, { u'url': u'http://40.media.tumblr.com/4f98bc55f60c925f6755fa1a272ec6d7/tumblr_no4u7pEOCl1s36m6bo1_500.jpg', u'width': 500, u'height': 313 }, { u'url': u'http://36.media.tumblr.com/4f98bc55f60c925f6755fa1a272ec6d7/tumblr_no4u7pEOCl1s36m6bo1_400.jpg', u'width': 400, u'height': 250 }, { u'url': u'http://40.media.tumblr.com/4f98bc55f60c925f6755fa1a272ec6d7/tumblr_no4u7pEOCl1s36m6bo1_250.jpg', u'width': 250, u'height': 156 }, { u'url': u'http://40.media.tumblr.com/4f98bc55f60c925f6755fa1a272ec6d7/tumblr_no4u7pEOCl1s36m6bo1_100.jpg', u'width': 100, u'height': 63 }, { u'url': u'http://40.media.tumblr.com/4f98bc55f60c925f6755fa1a272ec6d7/tumblr_no4u7pEOCl1s36m6bo1_75sq.jpg', u'width': 75, u'height': 75 }] }], u'date': u'2015-05-10 15:57:08 GMT', u'slug': u'taboolicious-happy-milf-day-many-thanks-for', u'blog_name': u'dclzexonask', u'trail': [{ u'blog': { u'active': True, u'theme': { u'title_font_weight': u'bold', u'title_color': u'#ff0062', u'header_bounds': 0, u'title_font': u'Calluna Sans', u'link_color': u'#ff002b', u'header_image_focused': u'http://static.tumblr.com/d48ceb5a7ccc9b9eeffdcf3aa9972391/3jhlkdq/1DXn0hvh0/tumblr_static_splash.jpg', u'show_description': True, u'show_header_image': True, u'body_font': u'Lucida Sans', u'show_title': True, u'header_stretch': True, u'avatar_shape': u'circle', u'show_avatar': True, u'background_color': u'#f6f6f6', u'header_image': u'http://static.tumblr.com/d48ceb5a7ccc9b9eeffdcf3aa9972391/3jhlkdq/1DXn0hvh0/tumblr_static_splash.jpg', u'header_image_scaled': u'http://static.tumblr.com/d48ceb5a7ccc9b9eeffdcf3aa9972391/3jhlkdq/1DXn0hvh0/tumblr_static_splash.jpg' }, u'name': u'taboolicious' }, u'content': u'<p>happy milf day! many thanks for all those that where present in the raffle and suggested such hot milfs :D</p><p>here is the wallpaper: 1920x1200</p><p><a href="https://dl.dropboxusercontent.com/u/1834642/milf%20paper.jpg">https://dl.dropboxusercontent.com/u/1834642/milf%20paper.jpg</a></p><p>from left to right:</p><p>bananacreamcakecomic: Jenna OC</p><p>jacquesthebird: Kate OC</p><p>damabiath: android 18</p><p>bellend08: Molly Cosset OC</p><p>vic79: Mystique</p><p>ravenlordmercenary: Grace pokemon X/Y</p><p>Th3A1chemist: Vuong OC</p><p>Evenanthy: Harley Quinn</p><p><br></p><p>if any of the winners wants a wallpaper with just his/her requested girl, pm me :)</p>', u'post': { u'id': u'118601139146' }, u'is_root_item': True }], u'caption': u'<p><a href="http://taboolicious.tumblr.com/post/118601139146/happy-milf-day-many-thanks-for-all-those-that" class="tumblr_blog">taboolicious</a>:</p>\n\n<blockquote><p>happy milf day! many thanks for all those that where present in the raffle and suggested such hot milfs :D</p><p>here is the wallpaper: 1920x1200</p><p><a href="https://dl.dropboxusercontent.com/u/1834642/milf%20paper.jpg">https://dl.dropboxusercontent.com/u/1834642/milf%20paper.jpg</a></p><p>from left to right:</p><p>bananacreamcakecomic: Jenna OC</p><p>jacquesthebird: Kate OC</p><p>damabiath: android 18</p><p>bellend08: Molly Cosset OC</p><p>vic79: Mystique</p><p>ravenlordmercenary: Grace pokemon X/Y</p><p>Th3A1chemist: Vuong OC</p><p>Evenanthy: Harley Quinn</p><p><br/></p><p>if any of the winners wants a wallpaper with just his/her requested girl, pm me :)</p></blockquote>' } malformed_link_result = handle_links(session, post_dict=malformed_link_post_dict) logging.debug("malformed_link_result:" + repr(malformed_link_result))
def main(): session = sql_functions.connect_to_db() add_skip(session,primary_key=1548538L) confirm_skip(session)
def main(): session = sql_functions.connect_to_db() add_skip(session, primary_key=1548538L) confirm_skip(session)