def post(self): task_key = ndb.Key(urlsafe=self.request.get('task')) task = task_key.get() task.update('Starting migration...', status='inprogress') logging.info('Starting migration ...') try: images = [i for i in UserImage.query() if i.filename != i.original_size_key] task.update('Migrating...', total_images=len(images)) logging.info('Migrating %s images' % len(images)) for img in images: img.migrate_to_gcs() task.migrated_images += 1 if task.migrated_images % 3 == 0: task.update('Migrated %s/%s images' % (task.migrated_images, task.total_images)) logging.info(task.message) task.put() task.update('Finished migrating images. Have a nice day :)', status='finished') logging.info(task.message) except Exception, ex: task.update('Failed to migrate: %s' % ex, status='failed') log_error('Failed migrate images', traceback.format_exc(6))
def post(self): task_key = ndb.Key(urlsafe=self.request.get('task')) task = task_key.get() task.update('Starting migration...', status='inprogress') logging.info('Starting migration ...') try: images = [ i for i in UserImage.query() if i.filename != i.original_size_key ] task.update('Migrating...', total_images=len(images)) logging.info('Migrating %s images' % len(images)) for img in images: img.migrate_to_gcs() task.migrated_images += 1 if task.migrated_images % 3 == 0: task.update('Migrated %s/%s images' % (task.migrated_images, task.total_images)) logging.info(task.message) task.put() task.update('Finished migrating images. Have a nice day :)', status='finished') logging.info(task.message) except Exception, ex: task.update('Failed to migrate: %s' % ex, status='failed') log_error('Failed migrate images', traceback.format_exc(6))
def check_if_mail_already_sent(self, date): #Check if we've already sent an email existing_slug = Slug.query(Slug.date == date).get() if existing_slug: msg = 'Tried to send another email on %s, already sent %s' % (date, existing_slug.slug) log_error('Tried to send email again', msg) raise Exception(msg)
def get_id(self, mail_message): to_str = ''.join(mail_message.to) match = re.search('post\+([0-9abcdef]{12})@', to_str) if not match: log_error('Unexpected mail received', 'Got mail with recipient %s', to_str) return None return match.group(1)
def receive(self, mail_message): try: id = self.get_id(mail_message) if not id: return slug = Slug.query(Slug.slug == id).get() if not slug: log_error('Invalid slug', 'Found no slug for id %s', id) return body_text, body_html = self.get_bodies(mail_message) raw_mail = RawMail( subject=mail_message.subject, sender=mail_message.sender, slug=id, date=slug.date, text=body_text, html=body_html ) raw_mail.put() post = Post.query(Post.date == slug.date).get() is_new_post = post is None if is_new_post: post = Post( date=slug.date, source='email', has_images=False ) #Now let's try parsing it into a good post... if body_html: post_text = strip_html(body_html) #Prefer html because then we don't get linebreak issues logging.info('Parsing post from html') else: post_text = body_text logging.info('Parsing post from plain text') if not post_text: raise Exception('No plain text body in email, html body can\'t be parsed yet!') try: email_index = post_text.index('post+%s@' % id) post_text = post_text[:email_index] newline_index = post_text.rstrip().rindex('\n') post_text = post_text[:newline_index].strip() except: logging.info('Failed to remove all crap from post') #Strip 'Sent from my iPhone' if it's there. There are probably endless other Sent from #we could handle, but hey, I have an iPhone so that's the one I care about... post_text = re.sub('\s*Sent from my iPhone\s*$', '', post_text) post_text = post_text.rstrip() if post.text: post.text = post.text + '\r\n\r\n' + Post.seperator + '\r\n\r\n' + post_text else: post.text = post_text self.process_attachments(mail_message, post) post.put() if is_new_post: counter = PostCounter.get() counter.increment(post.date.year, post.date.month) except: log_error('Failed to parse incoming email', traceback.format_exc(6))
def send(self, is_intro_email=False): try: now = datetime.datetime.now() settings = Settings.get() if is_intro_email: current_time = now logging.info('Sending intro email to %s' % settings.email_address) else: current_time, id, name, offset = self.get_time_in_timezone(settings) if current_time.hour != settings.email_hour: logging.info('Current time for %s is %s, not sending email now, will send at %02d:00' % (name, current_time, settings.email_hour)) return today = current_time.date() if self.check_if_intro_email_sent_today(today): logging.info('Already sent the intro email today, skipping this email for now') return self.check_if_mail_already_sent(today) slug_id = self.get_slug() slug = Slug(slug=slug_id, date=today) slug.put() subject = "It's %s, %s %s - How did your day go?" % (today.strftime('%A'), today.strftime("%b"), today.day) app_id = app_identity.get_application_id() sender = "MyLife <post+%s@%s.appspotmail.com>" % (slug.slug, app_id) message = mail.EmailMessage(sender=sender, subject=subject) message.to = settings.email_address if not settings.email_address: log_error('Missing To in daily email', 'There is no configured email address in your settings. Please visit your settings page to configure it so we can send you your daily email.') return message.body = """ Just reply to this email with your entry. OLD_POST You can see your past entries here: https://APP_ID.appspot.com/past """.replace('APP_ID', app_id) message.html = """ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.= w3.org/TR/html4/loose.dtd"> <html> <head> <title></title> </head> <body> Just reply to this email with your entry. <br> <br> OLD_POST <a href="https://APP_ID.appspot.com/past">Past entries</a> </body> </html> """.replace('APP_ID', app_id) if is_intro_email: intro_msg = "Welcome to MyLife. We've sent you this email immediately so you can try the system out. In the future we will email you once a day and include an old post in each email. You can configure when you get your email and which email address we should use on the settings page." message.html = message.html.replace('OLD_POST', intro_msg + '<br><br>') message.body = message.body.replace('OLD_POST', intro_msg + '\r\n\r\n') else: #Lets try to put in an old post... old_post, old_type = self.get_old_post(today) if old_post and settings.include_old_post_in_entry: old_post_text = 'Remember this? One %s ago you wrote:\r\n\r\n' % old_type old_post_text += old_post.text.rstrip() + '\r\n\r\n' message.body = re.sub(r'OLD_POST\r?\n', old_post_text, message.body) old_post_text = re.sub(r'\r?\n', '<br>', old_post_text) message.html = re.sub(r'OLD_POST\r?\n', old_post_text, message.html) else: message.body = re.sub('OLD_POST\r?\n', '', message.body) message.html = re.sub('OLD_POST\r?\n', '', message.html) message.send() if is_intro_email: logging.info('Sent intro email') else: if old_post: logging.info('Sent daily email to %s, using old post from %s' % (message.to, old_post.date)) else: logging.info('Sent daily email to %s, could not find old post' % message.to) return 'Email sent' except: log_error('Failed to send daily email', traceback.format_exc(6)) return 'Failed sending email: %s' % traceback.format_exc(6)
def receive(self, mail_message): try: id = self.get_id(mail_message) if not id: return slug = Slug.query(Slug.slug == id).get() if not slug: log_error('Invalid slug', 'Found no slug for id %s', id) return body_text, body_html = self.get_bodies(mail_message) raw_mail = RawMail(subject=mail_message.subject, sender=mail_message.sender, slug=id, date=slug.date, text=body_text, html=body_html) raw_mail.put() post = Post.query(Post.date == slug.date).get() is_new_post = post is None if is_new_post: post = Post(date=slug.date, source='email', has_images=False) #Now let's try parsing it into a good post... if body_html: post_text = strip_html( body_html ) #Prefer html because then we don't get linebreak issues logging.info('Parsing post from html') else: post_text = body_text logging.info('Parsing post from plain text') if not post_text: raise Exception( 'No plain text body in email, html body can\'t be parsed yet!' ) try: email_index = post_text.index('post+%s@' % id) post_text = post_text[:email_index] newline_index = post_text.rstrip().rindex('\n') post_text = post_text[:newline_index].strip() except: logging.info('Failed to remove all crap from post') #Strip 'Sent from my iPhone' if it's there. There are probably endless other Sent from #we could handle, but hey, I have an iPhone so that's the one I care about... post_text = re.sub('\s*Sent from my iPhone\s*$', '', post_text) post_text = post_text.rstrip() if post.text: post.text = post.text + '\r\n\r\n' + Post.seperator + '\r\n\r\n' + post_text else: post.text = post_text self.process_attachments(mail_message, post) post.put() if is_new_post: counter = PostCounter.get() counter.increment(post.date.year, post.date.month) except: log_error('Failed to parse incoming email', traceback.format_exc(6))
def post(self): import_task_key = ndb.Key(urlsafe=self.request.get('task')) import_task = import_task_key.get() import_task.update('Unpacking zip file...', status='inprogress') logging.info('Starting import ...') counter = PostCounter.get() try: posts, images = self.read_zip_file(import_task.uploaded_file) import_task.update('Importing...', total_photos=len(images), total_posts=len(posts)) logging.info('Importing %s posts, %s images' % (len(posts), len(images))) posts = self.filter_posts(posts) for date, text in posts: str_date = date.strftime('%Y-%m-%d') p = Post(date=date, source='ohlife', text=text.decode('utf-8')) p.images = [] p.has_images = False post_images = [(k, images[k]) for k in images.keys() if str_date in k] if len(post_images): logging.info('Importing %s images for date %s' % (len(post_images), str_date)) p.images = [] p.has_images = True for name, bytes in post_images: user_image = UserImage() img_name = name.replace('img_', '').replace('.jpeg', '.jpg') user_image.import_image(img_name, name, bytes, date) p.images.append(img_name) import_task.imported_photos += 1 user_image.put() p.put() counter.increment(p.date.year, p.date.month, False) import_task.imported_posts += 1 if import_task.imported_posts % 10 == 0: import_task.update( 'Imported %s/%s post, %s/%s photos...' % (import_task.imported_posts, import_task.total_posts, import_task.imported_photos, import_task.total_photos)) logging.info(import_task.message) counter.put() counter.put() skipped_posts = import_task.total_posts - import_task.imported_posts skipped_photos = import_task.total_photos - import_task.imported_photos msg = 'Imported %s posts and %s photos.' % ( import_task.imported_posts, import_task.imported_photos) if skipped_posts or skipped_photos: msg += ' %s posts and %s photos already existed and were skipped.' % ( skipped_posts, skipped_photos) import_task.update(msg, status='finished') logging.info(import_task.message) filestore.delete(import_task.uploaded_file) except Exception, ex: try: filestore.delete(import_task.uploaded_file) except: pass try: counter.put() except: pass import_task.update('Failed to import: %s' % ex, status='failed') log_error('Failed import', traceback.format_exc(6))
def get(self): images_total = 0 images_backed_up = 0 try: self.response.headers['Content-Type'] = 'text/plain' settings = Settings.get() if not settings.dropbox_access_token: self.log('No access token available, no backup will be performed.') return posts = [p for p in Post.query().order(Post.date).fetch()] self.log('Backing up %s posts to Dropbox' % len(posts)) post_text = StringIO() for p in posts: post_text.write(p.date.strftime('%Y-%m-%d')) post_text.write('\r\n\r\n') post_text.write(p.text.replace('\r\n', '\n').replace('\n', '\r\n').rstrip()) post_text.write('\r\n\r\n') result = self.put_file(settings.dropbox_access_token, 'MyLife.txt', post_text.getvalue().encode('utf-8')) post_text.close() self.log('Backed up posts. Revision: %s' % result['rev']) self.log('Fetching Dropbox file list') files_in_dropbox = self.get_dropbox_filelist(settings.dropbox_access_token) self.log('Got %s files from Dropbox' % len(files_in_dropbox)) self.log('Fetching images...') images = [i for i in UserImage.query().order(UserImage.date).fetch()] self.log('Total images in MyLife: %s' % len(images)) not_backed_up = [i for i in images if not i.backed_up_in_dropbox] not_in_dropbox = [i for i in images if not i.filename in files_in_dropbox] self.log('\nFiles not backed up: \n\n' + '\n'.join([i.filename for i in not_backed_up])) self.log('\nFiles marked as backed up, but not in Dropbox: \n\n' + '\n'.join([i.filename for i in not_in_dropbox])) images = not_backed_up + not_in_dropbox images_total = len(images) self.log('Found %s images that need to be backed up in Dropbox' % images_total) for img in images: self.log('Backing up %s' % img.filename) bytes = filestore.read(img.original_size_key) result = self.put_file(settings.dropbox_access_token, img.filename, bytes) self.log('Backed up %s. Revision: %s' % (img.filename, result['rev'])) img.backed_up_in_dropbox = True img.put() images_backed_up += 1 settings.dropbox_last_backup = datetime.datetime.now() settings.put() self.log('Finished backup successfully') except apiproxy_errors.OverQuotaError, ex: self.log(ex) log_error('Error backing up to Dropbox, quota exceeded', 'The backup operation did not complete because it ran out of quota. ' + 'The next time it runs it will continue backing up your posts and images.' + '%s images out of %s were backed up before failing' % (images_backed_up, images_total))
def get(self): images_total = 0 images_backed_up = 0 try: self.response.headers['Content-Type'] = 'text/plain' settings = Settings.get() if not settings.dropbox_access_token: self.log( 'No access token available, no backup will be performed.') return posts = [p for p in Post.query().order(Post.date).fetch()] self.log('Backing up %s posts to Dropbox' % len(posts)) post_text = StringIO() for p in posts: post_text.write(p.date.strftime('%Y-%m-%d')) post_text.write('\r\n\r\n') post_text.write( p.text.replace('\r\n', '\n').replace('\n', '\r\n').rstrip()) post_text.write('\r\n\r\n') result = self.put_file(settings.dropbox_access_token, 'MyLife.txt', post_text.getvalue().encode('utf-8')) post_text.close() self.log('Backed up posts. Revision: %s' % result['rev']) self.log('Fetching Dropbox file list') files_in_dropbox = self.get_dropbox_filelist( settings.dropbox_access_token) self.log('Got %s files from Dropbox' % len(files_in_dropbox)) self.log('Fetching images...') images = [ i for i in UserImage.query().order(UserImage.date).fetch() ] self.log('Total images in MyLife: %s' % len(images)) not_backed_up = [i for i in images if not i.backed_up_in_dropbox] not_in_dropbox = [ i for i in images if not i.filename in files_in_dropbox ] self.log('\nFiles not backed up: \n\n' + '\n'.join([i.filename for i in not_backed_up])) self.log('\nFiles marked as backed up, but not in Dropbox: \n\n' + '\n'.join([i.filename for i in not_in_dropbox])) images = not_backed_up + not_in_dropbox images_total = len(images) self.log('Found %s images that need to be backed up in Dropbox' % images_total) for img in images: self.log('Backing up %s' % img.filename) bytes = filestore.read(img.original_size_key) result = self.put_file(settings.dropbox_access_token, img.filename, bytes) self.log('Backed up %s. Revision: %s' % (img.filename, result['rev'])) img.backed_up_in_dropbox = True img.put() images_backed_up += 1 settings.dropbox_last_backup = datetime.datetime.now() settings.put() self.log('Finished backup successfully') except apiproxy_errors.OverQuotaError, ex: self.log(ex) log_error( 'Error backing up to Dropbox, quota exceeded', 'The backup operation did not complete because it ran out of quota. ' + 'The next time it runs it will continue backing up your posts and images.' + '%s images out of %s were backed up before failing' % (images_backed_up, images_total))
class DropboxBackupHandler(webapp2.RequestHandler): def get(self): images_total = 0 images_backed_up = 0 try: self.response.headers['Content-Type'] = 'text/plain' settings = Settings.get() if not settings.dropbox_access_token: self.log( 'No access token available, no backup will be performed.') return headers = { 'Content-Type': 'application/json', 'Authorization': 'Bearer ' + settings.dropbox_access_token } posts = [p for p in Post.query().order(Post.date).fetch()] self.log('Backing up %s posts to Dropbox' % len(posts)) post_text = StringIO() for p in posts: post_text.write(p.date.strftime('%Y-%m-%d')) post_text.write('\r\n\r\n') post_text.write( p.text.replace('\r\n', '\n').replace('\n', '\r\n').rstrip()) post_text.write('\r\n\r\n') result = self.put_file(headers, 'MyLife.txt', post_text.getvalue().encode('utf-8')) post_text.close() self.log('Backed up posts. Revision: %s' % result['revision']) result = self.get_dropbox_filelist(settings.dropbox_access_token, headers) self.log('Fetching Dropbox file list') files_in_dropbox = [m['path'][1:] for m in result['contents']] self.log('Fetching images...') images = [ i for i in UserImage.query().order(UserImage.date).fetch() ] images = [ i for i in images if not i.filename in files_in_dropbox or not i.backed_up_in_dropbox ] images_total = len(images) self.log('Found %s images that need to be backed up in Dropbox' % images_total) for img in images: self.log('Backing up %s' % img.filename) bytes = filestore.read(img.original_size_key) result = self.put_file(headers, img.filename, bytes) self.log('Backed up %s. Revision: %s' % (img.filename, result['revision'])) img.backed_up_in_dropbox = True img.put() images_backed_up += 1 settings.dropbox_last_backup = datetime.datetime.now() settings.put() self.log('Finished backup successfully') except apiproxy_errors.OverQuotaError, ex: self.log(ex) log_error( 'Error backing up to Dropbox, quota exceeded', 'The backup operation did not complete because it ran out of quota. ' + 'The next time it runs it will continue backing up your posts and images.' + '%s images out of %s were backed up before failing' % (images_backed_up, images_total)) except Exception, ex: self.log('ERROR: %s' % ex) log_error( 'Error backing up to Dropbox', 'Failed to backup posts and images to dropbox: %s' % traceback.format_exc(6))
def post(self): import_task_key = ndb.Key(urlsafe=self.request.get('task')) import_task = import_task_key.get() import_task.update('Unpacking zip file...', status='inprogress') logging.info('Starting import ...') counter = PostCounter.get() try: posts, images = self.read_zip_file(import_task.uploaded_file) import_task.update('Importing...', total_photos=len(images), total_posts=len(posts)) logging.info('Importing %s posts, %s images' % (len(posts), len(images))) posts = self.filter_posts(posts) for date, text in posts: str_date = date.strftime('%Y-%m-%d') p = Post( date=date, source='ohlife', text=text.decode('utf-8') ) p.images = [] p.has_images = False post_images = [(k,images[k]) for k in images.keys() if str_date in k] if len(post_images): logging.info('Importing %s images for date %s' % (len(post_images), str_date)) p.images = [] p.has_images = True for name, bytes in post_images: user_image = UserImage() img_name = name.replace('img_', '').replace('.jpeg', '.jpg') user_image.import_image(img_name, name, bytes, date) p.images.append(img_name) import_task.imported_photos += 1 user_image.put() p.put() counter.increment(p.date.year, p.date.month, False) import_task.imported_posts += 1 if import_task.imported_posts % 10 == 0: import_task.update('Imported %s/%s post, %s/%s photos...' % (import_task.imported_posts, import_task.total_posts,import_task.imported_photos, import_task.total_photos)) logging.info(import_task.message) counter.put() counter.put() skipped_posts = import_task.total_posts - import_task.imported_posts skipped_photos = import_task.total_photos - import_task.imported_photos msg = 'Imported %s posts and %s photos.' % (import_task.imported_posts, import_task.imported_photos) if skipped_posts or skipped_photos: msg += ' %s posts and %s photos already existed and were skipped.' % (skipped_posts, skipped_photos) import_task.update(msg, status='finished') logging.info(import_task.message) filestore.delete(import_task.uploaded_file) except Exception, ex: try: filestore.delete(import_task.uploaded_file) except: pass try: counter.put() except: pass import_task.update('Failed to import: %s' % ex, status='failed') log_error('Failed import', traceback.format_exc(6))
def send(self, is_intro_email=False, force_send=False, date=None): try: now = datetime.datetime.now() settings = Settings.get() if is_intro_email: current_time = now logging.info('Sending intro email to %s' % settings.email_address) else: current_time, id, name, offset = self.get_time_in_timezone(settings) if current_time.hour != settings.email_hour and not force_send: logging.info('Current time for %s is %s, not sending email now, will send at %02d:00' % (name, current_time, settings.email_hour)) return if date and force_send: today = date #Allow overriding this stuff else: today = current_time.date() if self.check_if_intro_email_sent_today(today) and not force_send: logging.info('Already sent the intro email today, skipping this email for now') return #Check if we've already sent an email slug = Slug.query(Slug.date == today).get() if slug and not force_send: msg = 'Tried to send another email on %s, already sent %s' % (date, slug.slug) log_error('Tried to send email again', msg) raise Exception(msg) if not slug: slug_id = self.get_slug() slug = Slug(slug=slug_id, date=today) slug.put() subject = "It's %s, %s %s - How did your day go?" % (today.strftime('%A'), today.strftime("%b"), today.day) app_id = app_identity.get_application_id() sender = "MyLife <post+%s@%s.appspotmail.com>" % (slug.slug, app_id) message = mail.EmailMessage(sender=sender, subject=subject) message.to = settings.email_address if not settings.email_address: log_error('Missing To in daily email', 'There is no configured email address in your settings. Please visit your settings page to configure it so we can send you your daily email.') return message.body = """ Just reply to this email with your entry. OLD_POST """.replace('APP_ID', app_id) message.html = """ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.= w3.org/TR/html4/loose.dtd"> <html> <head> <title></title> </head> <body> Just reply to this email with your entry. <br> <br> OLD_POST </body> </html> """.replace('APP_ID', app_id) if is_intro_email: intro_msg = "Welcome to MyLife. We've sent you this email immediately so you can try the system out. In the future we will email you once a day and include an old post in each email. You can configure when you get your email and which email address we should use on the settings page." message.html = message.html.replace('OLD_POST', intro_msg + '<br><br>') message.body = message.body.replace('OLD_POST', intro_msg + '\r\n\r\n') else: #Lets try to put in an old post... old_post, old_type = self.get_old_post(today) if old_post and settings.include_old_post_in_entry: logging.info('Going to use old post %s because %s' % (old_post, settings.include_old_post_in_entry)) if (old_type == 'random'): old_post_text = 'Remember this? On <b>%s, %s %s, %s</b> you wrote:\r\n\r\n' % (old_post.date.strftime('%A'), old_post.date.strftime("%b"), old_post.date.day, old_post.date.strftime("%Y")) else: old_post_text = 'Remember this? One %s ago you wrote:\r\n\r\n' % old_type old_post_text += old_post.text.rstrip() + '\r\n\r\n' message.body = re.sub(r'OLD_POST\r?\n', old_post_text, message.body) old_post_text = re.sub(r'\r?\n', '<br>', old_post_text) message.html = re.sub(r'OLD_POST\r?\n', old_post_text, message.html) else: logging.info('Not using Old post %s because %s' % (old_post, settings.include_old_post_in_entry)) message.body = re.sub('OLD_POST\r?\n', '', message.body) message.html = re.sub('OLD_POST\r?\n', '', message.html) message.send() if is_intro_email: logging.info('Sent intro email') else: if old_post: logging.info('Sent daily email to %s, using old post from %s' % (message.to, old_post.date)) else: logging.info('Sent daily email to %s, could not find old post' % message.to) return 'Email sent' except: log_error('Failed to send daily email', traceback.format_exc(6)) return 'Failed sending email: %s' % traceback.format_exc(6)