def edit_submission(text, submission_url): if submission_url == "": log("EDIT: Submission url is empty") return submission = Submission(reddit_auth, url=submission_url) submission.edit(text) log("Submission edited")
def check_deleted_posts(db_feeds, website_feeds): # Compare posts from db against the rss posts to make sure there are no deleted posts in the db i = 0 for db_feed in db_feeds: i += 1 is_deleted = True for website_feed in website_feeds: if (db_feed.title == website_feed.title) and ( db_feed.date == website_feed.date) and (db_feed.link == website_feed.link): is_deleted = False # We found the equivalent, break the loop break if is_deleted: # There was no equivalent on pietsmiet.de, means it was probably deleted # => Remove it from the database log("Feed with title '" + db_feed.title + "' was in db but not on pietsmiet.de. Deleting from database!") if not debug: delete_feed(db_feed) remove_image_from_gcloud(db_feed) # Only compare db posts against the same size of pietsmiet.de posts # because there are more db posts loaded than pietsmiet.de posts if i >= len(website_feeds): break
def send_fcm(feed, debug=False): global firebase_fcm message = feed.desc title = feed.title # game = None if feed.scope == "uploadplan": message = get_uploadplan_from_desc(feed.desc) if len(re.findall(".*?<br.*?>", message)) > 6: message = "<i>Diese Vorschau ist möglicherweise unvollständig:</i><br/>" + message elif feed.scope == "video": # Only send the title of the video (as message) title = "Neues Video (pietsmiet.de)" message = feed.title # game = get_game_from_video(message) elif feed.scope == "news": # Only send the title of the news item (as message) title = "News (pietsmiet.de)" message = feed.title if debug: title = "DEBUG: " + title data_message = { "title": title, "topic": feed.scope, "message": message, "link": feed.link, # "game": game } topic = feed.scope low_priority = True if debug is True: topic = "test_neu" low_priority = False retry_count = 1 while retry_count <= 3: log( "Info", "Sending fcm for " + feed.scope + " to topic/" + topic + " with content: " + message) try: firebase_fcm.notify_topic_subscribers(data_message=data_message, topic_name=topic, time_to_live=86400, low_priority=low_priority) return True except pyfcm.errors.FCMServerError as e: retry_time = pow(4, retry_count) log( "Warning", "Firebase servers are asleep, new try in " + str(retry_time) + " seconds." + "\n Exception is: " + format(e)) time.sleep(retry_time) firebase_fcm = FCMNotification(api_key=fcm_key) retry_count += 1 return False
def remove_image_from_gcloud(feed): try: # Delete the image from the firebase storage bucket = client.get_bucket(cloud_bucket_name) blob = bucket.blob(get_storage_path(feed)) blob.delete() log("Debug", "Image deleted") except Exception as e: log("Warning", "Deleting image failed: " + format(e))
def get_uploadplan_from_desc(desc): # Only send the actual uploadplan match = re.search( "(?:<p>)?(?:<strong>)?Upload-Plan am \d\d?\..*?(?:</strong>)?(?:<p>|<br ?/?>)(.{120,}?)(?:<br ?/?>)*</p>", desc, re.DOTALL) if match is not None: return match.group(1) else: log("Error", "No Uploadplan found in desc! Uploadplan was:\n " + desc) return "In der App ansehen..."
def update_desc(feed): name = get_id_of_feed(feed) try: result = firebase_db.put("/new/uploadplan/" + name, "desc", data=feed.desc) except Exception as e: log("Warning", "Failed to update feed desc: " + format(e)) result = None return result
def is_enabled(): # Master switch to disable bot remotely try: result = firebase_db.get("/", "active") if (result is None) or (not isinstance(result, bool)) or (result is False): return False else: return True except Exception as e: log('Error getting active status from fb db' + format(e)) return False
def check_uploadplan_edited(old_feed, new_feed): # Check if uploadplan desc changed new_feed.desc = scrape_site(new_feed.link) if new_feed.desc != old_feed.desc: if debug: log("Desc has changed, not putting into db because of debug") return if old_feed.reddit_url is not None: edit_submission(format_text(new_feed), old_feed.reddit_url) else: # Inform about missing reddit url and still store the new desc to avoid spam of this log("Warning", "No reddit url provided") # Put the updated desc back into db update_desc(new_feed)
def fetch_and_store(scope, count): website_feeds = parse_feed(scope, count) log("Loading " + str(len(website_feeds)) + " items in " + scope) for feed in website_feeds: if (scope == SCOPE_UPLOADPLAN) or (scope == SCOPE_NEWS): feed.desc = scrape_site(feed.link) time.sleep(1) if scope == SCOPE_NEWS: feed.desc = smart_truncate(feed) if (scope == SCOPE_VIDEO) and (feed.image_url is not None): feed.image_url = store_image_in_gcloud(feed.image_url, feed) if debug: log("Not posting to firebase because of debug") else: post_feed(feed) time.sleep(1)
def get_reddit_url(): try: result = firebase_db.get("/new/uploadplan", None, params={ 'orderBy': '"date"', 'limitToLast': '1' }) for key, feed in result.items(): if 'reddit_url' in feed: return feed['reddit_url'] else: log("No reddit url in result") except Exception as e: log('Error getting reddit url feed from fb db' + format(e)) return None
def post_feed(feed): """ Stores the content of the Feed object in firebase database :param feed: the feed object :return: None """ scope = feed.scope try: name = get_id_of_feed(feed) data = { "desc": feed.desc, "link": feed.link, "title": feed.title, "date": feed.date, "scope": feed.scope, "reddit_url": feed.reddit_url, "image_url": feed.image_url } firebase_db.put(url="/new/" + scope, name=name, data=data) except Exception as e: log('Error putting feed into fb db' + format(e))
def submit_to_reddit(title, text, debug=False): """ Posts a link to the given subreddit :param debug: Submit to test subreddit if ture :param title: Title of the reddit post :param text: Text to add to the reddit self post """ if debug is True: subreddit = "l3d00m" else: subreddit = "pietsmiet" if (text == '') or (title == ''): log("Warning: Not submitting to reddit, null text or title") return # Submit the post submission_url = reddit_auth.subreddit(subreddit).submit(title, selftext=text, resubmit=False, send_replies=False).shortlink log("Debug", submission_url) return submission_url
def store_image_in_gcloud(url, feed): try: # Get image with timeout r = requests.get(url, stream=True, timeout=10) if r.status_code == 200: # Check that the file is not bigger than max size content = r.raw.read(max_image_size + 1, decode_content=True) if len(content) > max_image_size: raise ValueError( 'File is too big (maybe it\'s the video instead of the thumb), not downloading!' ) else: raise ValueError("Wrong status code at download") # Resize the image to save bandwidth and storage im = Image.open(BytesIO(content)) im.thumbnail(size) im.save(image_path, format="JPEG") # Upload the image to the firebase storage bucket = client.get_bucket(cloud_bucket_name) blob = bucket.blob(get_storage_path(feed)) blob.upload_from_filename(image_path, content_type="image/jpg") # Cleanup ressources log("Debug", "Uploaded image") del content del im os.remove(image_path) # Return the download url of the image return "https://storage.googleapis.com/" + cloud_bucket_name + "/" + get_storage_path( feed) except Exception as e: log("Warning", "Couldn't up- / download image" + format(e)) return None
def get_last_feeds(scope, limit): try: result = firebase_db.get("/new/" + scope, None, params={ 'orderBy': '"date"', 'limitToLast': limit }) except Exception as e: log('Error getting feed from fb db' + format(e)) return None if result is None: return False old_feeds = [] for key, feed in result.items(): title = None desc = None link = None date = None reddit_url = None if 'title' in feed: title = feed['title'] else: log("Warning: Title not in db!") if 'desc' in feed: desc = feed['desc'] elif scope != "video": log("Warning: Desc not in db!") if 'link' in feed: link = feed['link'] else: log("Warning: Link not in db!") if 'date' in feed: date = feed['date'] if 'reddit_url' in feed: reddit_url = feed['reddit_url'] old_feeds.append( Feed(scope=scope, title=title, desc=desc, link=link, date=date, reddit_url=reddit_url)) old_feeds.sort(key=lambda x: x.date, reverse=True) return old_feeds
def process_new_item(feed, scope, i): # Submit to firebase FCM & DB and if uploadplan to reddit log("Debug", "New item in " + feed.scope + " with title: " + feed.title) if (scope == SCOPE_UPLOADPLAN) or (scope == SCOPE_NEWS): # Scrape site for the feed description feed.desc = scrape_site(feed.link) if scope == SCOPE_NEWS: # Truncate the news description feed.desc = smart_truncate(feed) if scope == SCOPE_VIDEO: if feed.image_url is not None: # Store thumb in gcloud and send fcm feed.image_url = store_image_in_gcloud(feed.image_url, feed) if nofcm is True: fcm_success = True else: fcm_success = send_fcm(feed, debug) else: # Don't send FCM as videos without thumbs are usually bad uploads and will be reuploaded # Still store it in the DB if it just doesn't have a thumb for another reason log( "Warning", "No thumbnail found, means it's probably a bad upload. Not sending FCM!" + "Title is \"" + feed.title + "\"") fcm_success = True else: if nofcm is True: fcm_success = True else: fcm_success = send_fcm(feed, debug) if not fcm_success: log("Error", "Could not send FCM, aborting!") return if (scope == SCOPE_UPLOADPLAN) and (i == 0): # Don't submit old uploadplan: Only if it's the first new_feed and new, submit it log("Submitting uploadplan to reddit") time.sleep(1) r_url = submit_to_reddit(feed.title, format_text(feed), debug=debug) feed.reddit_url = r_url if not debug: post_feed(feed)
def delete_submission(submission_url): if submission_url == "": log("Warning", "DELETE: Submission url is empty") return submission = Submission(reddit_auth, url=submission_url) # Only remove the submission if there are less than 4 comments, otherwise unsticky comment_count = len(submission.comments.list()) if comment_count < 5: submission.mod.remove() submission.mod.lock() log("Submission removed") else: submission.mod.sticky(False) log("Submission unstickied")
def check_for_update(scope): global limit # Check if master switch in db is off and abort if true if not is_enabled(): if debug: log("Info", "Master switch is off, ignoring (debug)") else: log("Warning", "Master switch is off, aborting") return # Set limit to default if not set manually if limit is -1: if scope is "video": limit = default_video_limit else: limit = default_post_limit log("Checking for " + scope) website_feeds = parse_feed(scope, limit) if len(website_feeds) == 0: log( "Error", "Pietsmiet.de feeds are empty, bad network or site down? Aborting") return # Load more db items than new ones to compare better (e.g. if there are deleted items in the db) db_feed_limit = limit + 5 db_feeds = get_last_feeds(scope, db_feed_limit) # Check that loading of db posts was successful if db_feeds is None: log("Error", "Cannot retrieve old feeds! Aborting") return # Check that there are posts in db, otherwise reload posts if db_feeds is False: log("Warning", "No feeds in db, loading all posts in db") fetch_and_store(scope, 25) return # Check that all posts were loaded, otherwise reload posts if len(db_feeds) is not db_feed_limit: log( "Error", "Loaded " + str(len(db_feeds)) + " feeds from db, should be " + str(db_feed_limit)) fetch_and_store(scope, 25) return # Iterate through every website feed and check if it is new (its title or link does _not_ match # one of the old feeds) new_feeds = {} i = 0 for website_feed in website_feeds: # Compare pietsmiet.de feed against all feeds from db if (find_feed_in_array(website_feed, db_feeds) is False) or force: new_feeds[i] = website_feed i += 1 if (len(new_feeds) >= limit) and not force: # All feeds changed, means there was probably a gap inbetween => Reload all posts into db # This should only happen if the script wasn't running for a few days log("Posts in db too old, loading all posts in db") fetch_and_store(scope, 25) elif len(new_feeds) == 0: # No new posts found => Database should be the same as pietsmiet.de now, # so we can check if there are invalid posts in db log("Info", "No new posts found for scope " + scope) check_deleted_posts(db_feeds, website_feeds) if scope == SCOPE_UPLOADPLAN: # Also check if the uploadplan was edited website_feed = find_feed_in_array(db_feeds[0], website_feeds) if website_feed is not False: check_uploadplan_edited(db_feeds[0], website_feed) else: # Iterate through all new feeds and process them for i, new_feed in new_feeds.items(): # New item found process_new_item(new_feed, scope, i) time.sleep(1)
def delete_feed(feed): try: firebase_db.delete('/new/' + feed.scope, get_id_of_feed(feed)) log('Feed deleted') except Exception as e: log('Error deleting feed from fb db' + format(e))
required=False, type=int, help= "(Re)loads the specified amount of posts in all scopes into the database. " + "Note: Limit for uploadplan, pietcast and news is always 8") parser.add_argument( "-l", "--limit", required=False, type=int, choices=range(2, 20), help="Set a custom limit how many posts should be compared.") args = parser.parse_args() if args.debug: log("Debug enabled.") debug = True if args.force: log("Debug and force enabled.") force = True debug = True if args.nofcm: log("No FCM mode active.") nofcm = True if args.limit: if args.loadall: log("Limit ignored because it's specified in the --loadall parameter")