def edit_submission(text, submission_url):
if submission_url == "":
log("EDIT: Submission url is empty")
return
submission = Submission(reddit_auth, url=submission_url)
submission.edit(text)
log("Submission edited")
def check_deleted_posts(db_feeds, website_feeds):
# Compare posts from db against the rss posts to make sure there are no deleted posts in the db
i = 0
for db_feed in db_feeds:
i += 1
is_deleted = True
for website_feed in website_feeds:
if (db_feed.title == website_feed.title) and (
db_feed.date
== website_feed.date) and (db_feed.link
== website_feed.link):
is_deleted = False
# We found the equivalent, break the loop
break
if is_deleted:
# There was no equivalent on pietsmiet.de, means it was probably deleted
# => Remove it from the database
log("Feed with title '" + db_feed.title +
"' was in db but not on pietsmiet.de. Deleting from database!")
if not debug:
delete_feed(db_feed)
remove_image_from_gcloud(db_feed)
# Only compare db posts against the same size of pietsmiet.de posts
# because there are more db posts loaded than pietsmiet.de posts
if i >= len(website_feeds):
break
def send_fcm(feed, debug=False):
global firebase_fcm
message = feed.desc
title = feed.title
# game = None
if feed.scope == "uploadplan":
message = get_uploadplan_from_desc(feed.desc)
if len(re.findall(".*?<br.*?>", message)) > 6:
message = "<i>Diese Vorschau ist möglicherweise unvollständig:</i><br/>" + message
elif feed.scope == "video":
# Only send the title of the video (as message)
title = "Neues Video (pietsmiet.de)"
message = feed.title
# game = get_game_from_video(message)
elif feed.scope == "news":
# Only send the title of the news item (as message)
title = "News (pietsmiet.de)"
message = feed.title
if debug:
title = "DEBUG: " + title
data_message = {
"title": title,
"topic": feed.scope,
"message": message,
"link": feed.link,
# "game": game
}
topic = feed.scope
low_priority = True
if debug is True:
topic = "test_neu"
low_priority = False
retry_count = 1
while retry_count <= 3:
log(
"Info", "Sending fcm for " + feed.scope + " to topic/" + topic +
" with content: " + message)
try:
firebase_fcm.notify_topic_subscribers(data_message=data_message,
topic_name=topic,
time_to_live=86400,
low_priority=low_priority)
return True
except pyfcm.errors.FCMServerError as e:
retry_time = pow(4, retry_count)
log(
"Warning",
"Firebase servers are asleep, new try in " + str(retry_time) +
" seconds." + "\n Exception is: " + format(e))
time.sleep(retry_time)
firebase_fcm = FCMNotification(api_key=fcm_key)
retry_count += 1
return False
def remove_image_from_gcloud(feed):
try:
# Delete the image from the firebase storage
bucket = client.get_bucket(cloud_bucket_name)
blob = bucket.blob(get_storage_path(feed))
blob.delete()
log("Debug", "Image deleted")
except Exception as e:
log("Warning", "Deleting image failed: " + format(e))
def get_uploadplan_from_desc(desc):
# Only send the actual uploadplan
match = re.search(
"(?:<p>)?(?:<strong>)?Upload-Plan am \d\d?\..*?(?:</strong>)?(?:<p>|<br ?/?>)(.{120,}?)(?:<br ?/?>)*</p>",
desc, re.DOTALL)
if match is not None:
return match.group(1)
else:
log("Error", "No Uploadplan found in desc! Uploadplan was:\n " + desc)
return "In der App ansehen..."
def update_desc(feed):
name = get_id_of_feed(feed)
try:
result = firebase_db.put("/new/uploadplan/" + name,
"desc",
data=feed.desc)
except Exception as e:
log("Warning", "Failed to update feed desc: " + format(e))
result = None
return result
def is_enabled():
# Master switch to disable bot remotely
try:
result = firebase_db.get("/", "active")
if (result is None) or (not isinstance(result,
bool)) or (result is False):
return False
else:
return True
except Exception as e:
log('Error getting active status from fb db' + format(e))
return False
def check_uploadplan_edited(old_feed, new_feed):
# Check if uploadplan desc changed
new_feed.desc = scrape_site(new_feed.link)
if new_feed.desc != old_feed.desc:
if debug:
log("Desc has changed, not putting into db because of debug")
return
if old_feed.reddit_url is not None:
edit_submission(format_text(new_feed), old_feed.reddit_url)
else:
# Inform about missing reddit url and still store the new desc to avoid spam of this
log("Warning", "No reddit url provided")
# Put the updated desc back into db
update_desc(new_feed)
def fetch_and_store(scope, count):
website_feeds = parse_feed(scope, count)
log("Loading " + str(len(website_feeds)) + " items in " + scope)
for feed in website_feeds:
if (scope == SCOPE_UPLOADPLAN) or (scope == SCOPE_NEWS):
feed.desc = scrape_site(feed.link)
time.sleep(1)
if scope == SCOPE_NEWS:
feed.desc = smart_truncate(feed)
if (scope == SCOPE_VIDEO) and (feed.image_url is not None):
feed.image_url = store_image_in_gcloud(feed.image_url, feed)
if debug:
log("Not posting to firebase because of debug")
else:
post_feed(feed)
time.sleep(1)
def get_reddit_url():
try:
result = firebase_db.get("/new/uploadplan",
None,
params={
'orderBy': '"date"',
'limitToLast': '1'
})
for key, feed in result.items():
if 'reddit_url' in feed:
return feed['reddit_url']
else:
log("No reddit url in result")
except Exception as e:
log('Error getting reddit url feed from fb db' + format(e))
return None
def post_feed(feed):
"""
Stores the content of the Feed object in firebase database
:param feed: the feed object
:return: None
"""
scope = feed.scope
try:
name = get_id_of_feed(feed)
data = {
"desc": feed.desc,
"link": feed.link,
"title": feed.title,
"date": feed.date,
"scope": feed.scope,
"reddit_url": feed.reddit_url,
"image_url": feed.image_url
}
firebase_db.put(url="/new/" + scope, name=name, data=data)
except Exception as e:
log('Error putting feed into fb db' + format(e))
def submit_to_reddit(title, text, debug=False):
"""
Posts a link to the given subreddit
:param debug: Submit to test subreddit if ture
:param title: Title of the reddit post
:param text: Text to add to the reddit self post
"""
if debug is True:
subreddit = "l3d00m"
else:
subreddit = "pietsmiet"
if (text == '') or (title == ''):
log("Warning: Not submitting to reddit, null text or title")
return
# Submit the post
submission_url = reddit_auth.subreddit(subreddit).submit(title, selftext=text, resubmit=False,
send_replies=False).shortlink
log("Debug", submission_url)
return submission_url
def store_image_in_gcloud(url, feed):
try:
# Get image with timeout
r = requests.get(url, stream=True, timeout=10)
if r.status_code == 200:
# Check that the file is not bigger than max size
content = r.raw.read(max_image_size + 1, decode_content=True)
if len(content) > max_image_size:
raise ValueError(
'File is too big (maybe it\'s the video instead of the thumb), not downloading!'
)
else:
raise ValueError("Wrong status code at download")
# Resize the image to save bandwidth and storage
im = Image.open(BytesIO(content))
im.thumbnail(size)
im.save(image_path, format="JPEG")
# Upload the image to the firebase storage
bucket = client.get_bucket(cloud_bucket_name)
blob = bucket.blob(get_storage_path(feed))
blob.upload_from_filename(image_path, content_type="image/jpg")
# Cleanup ressources
log("Debug", "Uploaded image")
del content
del im
os.remove(image_path)
# Return the download url of the image
return "https://storage.googleapis.com/" + cloud_bucket_name + "/" + get_storage_path(
feed)
except Exception as e:
log("Warning", "Couldn't up- / download image" + format(e))
return None
def get_last_feeds(scope, limit):
try:
result = firebase_db.get("/new/" + scope,
None,
params={
'orderBy': '"date"',
'limitToLast': limit
})
except Exception as e:
log('Error getting feed from fb db' + format(e))
return None
if result is None:
return False
old_feeds = []
for key, feed in result.items():
title = None
desc = None
link = None
date = None
reddit_url = None
if 'title' in feed:
title = feed['title']
else:
log("Warning: Title not in db!")
if 'desc' in feed:
desc = feed['desc']
elif scope != "video":
log("Warning: Desc not in db!")
if 'link' in feed:
link = feed['link']
else:
log("Warning: Link not in db!")
if 'date' in feed:
date = feed['date']
if 'reddit_url' in feed:
reddit_url = feed['reddit_url']
old_feeds.append(
Feed(scope=scope,
title=title,
desc=desc,
link=link,
date=date,
reddit_url=reddit_url))
old_feeds.sort(key=lambda x: x.date, reverse=True)
return old_feeds
def process_new_item(feed, scope, i):
# Submit to firebase FCM & DB and if uploadplan to reddit
log("Debug", "New item in " + feed.scope + " with title: " + feed.title)
if (scope == SCOPE_UPLOADPLAN) or (scope == SCOPE_NEWS):
# Scrape site for the feed description
feed.desc = scrape_site(feed.link)
if scope == SCOPE_NEWS:
# Truncate the news description
feed.desc = smart_truncate(feed)
if scope == SCOPE_VIDEO:
if feed.image_url is not None:
# Store thumb in gcloud and send fcm
feed.image_url = store_image_in_gcloud(feed.image_url, feed)
if nofcm is True:
fcm_success = True
else:
fcm_success = send_fcm(feed, debug)
else:
# Don't send FCM as videos without thumbs are usually bad uploads and will be reuploaded
# Still store it in the DB if it just doesn't have a thumb for another reason
log(
"Warning",
"No thumbnail found, means it's probably a bad upload. Not sending FCM!"
+ "Title is \"" + feed.title + "\"")
fcm_success = True
else:
if nofcm is True:
fcm_success = True
else:
fcm_success = send_fcm(feed, debug)
if not fcm_success:
log("Error", "Could not send FCM, aborting!")
return
if (scope == SCOPE_UPLOADPLAN) and (i == 0):
# Don't submit old uploadplan: Only if it's the first new_feed and new, submit it
log("Submitting uploadplan to reddit")
time.sleep(1)
r_url = submit_to_reddit(feed.title, format_text(feed), debug=debug)
feed.reddit_url = r_url
if not debug:
post_feed(feed)
def delete_submission(submission_url):
if submission_url == "":
log("Warning", "DELETE: Submission url is empty")
return
submission = Submission(reddit_auth, url=submission_url)
# Only remove the submission if there are less than 4 comments, otherwise unsticky
comment_count = len(submission.comments.list())
if comment_count < 5:
submission.mod.remove()
submission.mod.lock()
log("Submission removed")
else:
submission.mod.sticky(False)
log("Submission unstickied")
def check_for_update(scope):
global limit
# Check if master switch in db is off and abort if true
if not is_enabled():
if debug:
log("Info", "Master switch is off, ignoring (debug)")
else:
log("Warning", "Master switch is off, aborting")
return
# Set limit to default if not set manually
if limit is -1:
if scope is "video":
limit = default_video_limit
else:
limit = default_post_limit
log("Checking for " + scope)
website_feeds = parse_feed(scope, limit)
if len(website_feeds) == 0:
log(
"Error",
"Pietsmiet.de feeds are empty, bad network or site down? Aborting")
return
# Load more db items than new ones to compare better (e.g. if there are deleted items in the db)
db_feed_limit = limit + 5
db_feeds = get_last_feeds(scope, db_feed_limit)
# Check that loading of db posts was successful
if db_feeds is None:
log("Error", "Cannot retrieve old feeds! Aborting")
return
# Check that there are posts in db, otherwise reload posts
if db_feeds is False:
log("Warning", "No feeds in db, loading all posts in db")
fetch_and_store(scope, 25)
return
# Check that all posts were loaded, otherwise reload posts
if len(db_feeds) is not db_feed_limit:
log(
"Error", "Loaded " + str(len(db_feeds)) +
" feeds from db, should be " + str(db_feed_limit))
fetch_and_store(scope, 25)
return
# Iterate through every website feed and check if it is new (its title or link does _not_ match
# one of the old feeds)
new_feeds = {}
i = 0
for website_feed in website_feeds:
# Compare pietsmiet.de feed against all feeds from db
if (find_feed_in_array(website_feed, db_feeds) is False) or force:
new_feeds[i] = website_feed
i += 1
if (len(new_feeds) >= limit) and not force:
# All feeds changed, means there was probably a gap inbetween => Reload all posts into db
# This should only happen if the script wasn't running for a few days
log("Posts in db too old, loading all posts in db")
fetch_and_store(scope, 25)
elif len(new_feeds) == 0:
# No new posts found => Database should be the same as pietsmiet.de now,
# so we can check if there are invalid posts in db
log("Info", "No new posts found for scope " + scope)
check_deleted_posts(db_feeds, website_feeds)
if scope == SCOPE_UPLOADPLAN:
# Also check if the uploadplan was edited
website_feed = find_feed_in_array(db_feeds[0], website_feeds)
if website_feed is not False:
check_uploadplan_edited(db_feeds[0], website_feed)
else:
# Iterate through all new feeds and process them
for i, new_feed in new_feeds.items():
# New item found
process_new_item(new_feed, scope, i)
time.sleep(1)
def delete_feed(feed):
try:
firebase_db.delete('/new/' + feed.scope, get_id_of_feed(feed))
log('Feed deleted')
except Exception as e:
log('Error deleting feed from fb db' + format(e))
required=False,
type=int,
help=
"(Re)loads the specified amount of posts in all scopes into the database. "
+ "Note: Limit for uploadplan, pietcast and news is always 8")
parser.add_argument(
"-l",
"--limit",
required=False,
type=int,
choices=range(2, 20),
help="Set a custom limit how many posts should be compared.")
args = parser.parse_args()
if args.debug:
log("Debug enabled.")
debug = True
if args.force:
log("Debug and force enabled.")
force = True
debug = True
if args.nofcm:
log("No FCM mode active.")
nofcm = True
if args.limit:
if args.loadall:
log("Limit ignored because it's specified in the --loadall parameter")