def post(self): file_info = self.get_file_infos()[0] self.response.headers['Content-Type'] = "application/json" year = self.request.get('year') month = self.request.get('month') day = self.request.get('day') date = datetime.datetime(int(year), int(month), int(day)) if file_info.content_type.lower() not in ('image/jpeg', 'image/jpg', 'image/png', 'image/gif', 'image/bmp'): return self.response.write(json.dumps({"status" : "error", "message" : "Unsupported content type: " + file_info.content_type})) bytes = filestore.read(file_info.gs_object_name) existing_images = [u.filename for u in UserImage.query(UserImage.date == date).fetch()] filename = UserImage.create_image_name(file_info.filename, date, existing_images) img = UserImage() img.import_image(filename, file_info.filename, bytes, date, None) img.put() filestore.delete(file_info.gs_object_name) #If there's a post here we should add the image... post = Post.query(Post.date == date).get() if post: post.has_images = True if post.images is None: post.images = [] post.images.append(filename) post.put() self.response.write(json.dumps({"status" : "ok", "filename" : filename}))
def post(self): task_key = ndb.Key(urlsafe=self.request.get('task')) task = task_key.get() task.update('Starting migration...', status='inprogress') logging.info('Starting migration ...') try: images = [i for i in UserImage.query() if i.filename != i.original_size_key] task.update('Migrating...', total_images=len(images)) logging.info('Migrating %s images' % len(images)) for img in images: img.migrate_to_gcs() task.migrated_images += 1 if task.migrated_images % 3 == 0: task.update('Migrated %s/%s images' % (task.migrated_images, task.total_images)) logging.info(task.message) task.put() task.update('Finished migrating images. Have a nice day :)', status='finished') logging.info(task.message) except Exception, ex: task.update('Failed to migrate: %s' % ex, status='failed') log_error('Failed migrate images', traceback.format_exc(6))
def post(self, filename): self.response.headers['Content-Type'] = "application/json" img = UserImage.query(UserImage.filename == filename).get() if not img: return self.response.write(json.dumps({"status" : "error", "message" : "Image does not exit"})) post = Post.query(Post.date == img.date).get() #Remove it from the post if post: try: post.images.remove(filename) post.text = post.text.replace('$IMG:' + filename, '').replace('\n\n\n\n', '\n\n') except: pass if len(post.images) == 0: post.has_images = False post.put() filestore.delete(img.serving_size_key) filestore.delete(img.original_size_key) img.key.delete() self.response.write(json.dumps({"status" : "ok"}))
def post(self, filename): self.response.headers['Content-Type'] = "application/json" img = UserImage.query(UserImage.filename == filename).get() if not img: return self.response.write(json.dumps({"status" : "error", "message" : "Image does not exit"})) post = Post.query(Post.date == img.date).get() #Remove it from the post if post: try: post.images.remove(filename) except: pass if len(post.images) == 0: post.has_images = False post.put() filestore.delete(img.serving_size_key) filestore.delete(img.original_size_key) img.key.delete() self.response.write(json.dumps({"status" : "ok"}))
def post(self): task_key = ndb.Key(urlsafe=self.request.get('task')) task = task_key.get() task.update('Starting migration...', status='inprogress') logging.info('Starting migration ...') try: images = [ i for i in UserImage.query() if i.filename != i.original_size_key ] task.update('Migrating...', total_images=len(images)) logging.info('Migrating %s images' % len(images)) for img in images: img.migrate_to_gcs() task.migrated_images += 1 if task.migrated_images % 3 == 0: task.update('Migrated %s/%s images' % (task.migrated_images, task.total_images)) logging.info(task.message) task.put() task.update('Finished migrating images. Have a nice day :)', status='finished') logging.info(task.message) except Exception, ex: task.update('Failed to migrate: %s' % ex, status='failed') log_error('Failed migrate images', traceback.format_exc(6))
def process_attachments(self, mail_message, post): attachments = [] try: attachments = mail_message.attachments except exceptions.AttributeError: pass #No attachments, then the attribute doesn't even exist :/ if attachments: logging.info('Received %s attachment(s)' % len(attachments)) for attachment in attachments: original_filename = attachment.filename encoded_payload = attachment.payload content_id = attachment.content_id if content_id: content_id = content_id.replace('<', '').replace( '>', '' ) # Don't want these around the id, messes with our tag handling logging.info('Processing attachment: %s' % original_filename) if re.search('\\.(jpe?g|png|bmp|gif)$', original_filename.lower()): if post.images is None: post.images = [] bytes = encoded_payload.payload if encoded_payload.encoding: bytes = bytes.decode(encoded_payload.encoding) post.has_images = True user_image = UserImage() img_name = UserImage.create_image_name(original_filename, post.date, post.images) user_image.import_image(img_name, original_filename, bytes, post.date, content_id) post.images.append(img_name) user_image.is_inline = False if content_id: placeholder = '$IMG:' + content_id if placeholder in post.text: user_image.is_inline = True #Ok, lets put in a filename instead of the content_id post.text = post.text.replace(placeholder, '$IMG:' + img_name) user_image.put() else: logging.warning('Received unsupported attachment, %s' % original_filename)
def post(self): file_info = self.get_file_infos()[0] self.response.headers['Content-Type'] = "application/json" year = self.request.get('year') month = self.request.get('month') day = self.request.get('day') date = datetime.datetime(int(year), int(month), int(day)) if file_info.content_type.lower() not in ('image/jpeg', 'image/jpg', 'image/png', 'image/gif', 'image/bmp'): return self.response.write(json.dumps({"status" : "error", "message" : "Unsupported content type: " + file_info.content_type})) bytes = filestore.read(file_info.gs_object_name) existing_images = [u.filename for u in UserImage.query(UserImage.date == date).fetch()] filename = UserImage.create_image_name(file_info.filename, date, existing_images) img = UserImage() img.import_image(filename, file_info.filename, bytes, date) img.put() filestore.delete(file_info.gs_object_name) #If there's a post here we should add the image... post = Post.query(Post.date == date).get() if post: post.has_images = True if post.images is None: post.images = [] post.images.append(filename) post.put() self.response.write(json.dumps({"status" : "ok", "filename" : filename}))
def process_attachments(self, mail_message, post): attachments = [] try: attachments = mail_message.attachments except exceptions.AttributeError: pass #No attachments, then the attribute doesn't even exist :/ if attachments: logging.info('Received %s attachment(s)' % len(attachments)) for original_filename, encoded_payload in attachments: logging.info('Processing attachment: %s' % original_filename) if re.search('\\.(jpe?g|png|bmp|gif)$', original_filename.lower()): if post.images is None: post.images = [] bytes = encoded_payload.payload if encoded_payload.encoding: bytes = bytes.decode(encoded_payload.encoding) post.has_images = True user_image = UserImage() img_name = UserImage.create_image_name(original_filename, post.date, post.images) user_image.import_image(img_name, original_filename, bytes, post.date) post.images.append(img_name) user_image.put() else: logging.warning('Received unsupported attachment, %s' % original_filename)
def get(self, filename): image = UserImage.query(UserImage.filename == filename).get() if self.request.get('fullsize'): key = image.original_size_key else: key = image.serving_size_key if not image: self.error(404) else: self.send_blob(filestore.get_blob_key(key))
def add_images_to_zip(self, export_task, archive): export_task.update('Fetching image information...') images = [i for i in UserImage.query().order(UserImage.filename).fetch()] export_task.update('Found %s images...' % len(images)) for i, img in enumerate(images): img_data = filestore.read(img.original_size_key) archive.writestr('/img_%s' % img.filename.replace('.jpg', '.jpeg'), img_data) if i % 5 == 0: export_task.update('Added %s of %s images to zip... ' % (i+1,len(images))) export_task.update('Finished adding images...')
def process_attachments(self, mail_message, post): attachments = [] try: attachments = mail_message.attachments except exceptions.AttributeError: pass #No attachments, then the attribute doesn't even exist :/ if attachments: logging.info('Received %s attachment(s)' % len(attachments)) for attachment in attachments: original_filename = attachment.filename encoded_payload = attachment.payload content_id = attachment.content_id if content_id: content_id = content_id.replace('<', '').replace('>', '') # Don't want these around the id, messes with our tag handling logging.info('Processing attachment: %s' % original_filename) if re.search('\\.(jpe?g|png|bmp|gif)$', original_filename.lower()): if post.images is None: post.images = [] bytes = encoded_payload.payload if encoded_payload.encoding: bytes = bytes.decode(encoded_payload.encoding) post.has_images = True user_image = UserImage() img_name = UserImage.create_image_name(original_filename, post.date, post.images) user_image.import_image(img_name, original_filename, bytes, post.date, content_id) post.images.append(img_name) user_image.is_inline = False if content_id: placeholder = '$IMG:' + content_id if placeholder in post.text: user_image.is_inline = True #Ok, lets put in a filename instead of the content_id post.text = post.text.replace(placeholder, '$IMG:' + img_name) user_image.put() else: logging.warning('Received unsupported attachment, %s' % original_filename)
def delete_post(self, post): images = UserImage.query(UserImage.date == post.date).fetch() for img in images: filestore.delete(img.serving_size_key) filestore.delete(img.original_size_key) img.key.delete() emails = RawMail.query(RawMail.date == post.date).fetch() for email in emails: email.key.delete() post.key.delete() PostCounter.get().decrement(post.date.year, post.date.month) logging.info('Deleted %s images, %s emails and 1 post from %s' % (len(images), len(emails), post.date.strftime('%Y-%m-%d')))
def delete_post(self, post): images = UserImage.query(UserImage.date == post.date).fetch() for img in images: filestore.delete(img.serving_size_key) filestore.delete(img.original_size_key) img.key.delete() emails = RawMail.query(RawMail.date == post.date).fetch() for email in emails: email.delete() post.key.delete() PostCounter.get().decrement(post.date.year, post.date.month) logging.info('Deleted %s images, %s emails and 1 post from %s' % (len(images), len(emails), post.date.strftime('%Y-%m-%d')))
def get(self): #Check whether the migration is done so we can see whether to show the Blobstore Migration #or not... settings = Settings.get() if not settings.blobstore_migration_done: migration_task_finished = bool(MigrateTask.query(MigrateTask.status == 'finished').get()) if migration_task_finished: settings.blobstore_migration_done = True settings.put() else: #Try to figure out whether this is a new user that has nothing in the blobstore... if not UserImage.query().get(): settings.blobstore_migration_done = True settings.put() self._render(settings)
def add_images_to_zip(self, export_task, archive): export_task.update('Fetching image information...') images = [ i for i in UserImage.query().order(UserImage.filename).fetch() ] export_task.update('Found %s images...' % len(images)) for i, img in enumerate(images): img_data = filestore.read(img.original_size_key) archive.writestr('/img_%s' % img.filename.replace('.jpg', '.jpeg'), img_data) if i % 5 == 0: export_task.update('Added %s of %s images to zip... ' % (i + 1, len(images))) export_task.update('Finished adding images...')
def post(self, kind, year, month, day): date = datetime.datetime(int(year),int(month),int(day)).date() post = Post.query(Post.date == date).get() is_new = False if not post: post = Post(date=date, source='web',images=[]) is_new = True post.text = self.request.get('text') save = self.request.get('action') == 'save' delete = self.request.get('action') == 'delete' if save and delete: raise Exception('Something weird happened...') if save: if is_new: post.images = [u.filename for u in UserImage.query(UserImage.date == date).fetch()] post.images.sort() post.has_images = True post.put() if is_new: PostCounter.get().increment(post.date.year, post.date.month) self.redirect_to_date(post.date) elif delete: self.delete_post(post) next_post = Post.query(Post.date > date).order(Post.date).get() if next_post and next_post.date.month == date.month: return self.redirect_to_date(next_post.date) #No way, we'll have to just redirect to the empty month self.redirect('/past/%s' % date.strftime('%Y-%m')) else: raise Exception('How the hell did we get here...?')
def get(self, kind, year, month, day): date = datetime.datetime(int(year),int(month),int(day)).date() post = Post.query(Post.date == date).get() if kind == 'write' and post: return self.redirect('/edit/%s' % date.strftime('%Y-%m-%d')) if kind == 'edit' and not post: return self.redirect('/write/%s' % date.strftime('%Y-%m-%d')) data = { "date" : date, "text" : "", "page" : "write", "kind" : kind } if post: data["page"] = "edit" data["text"] = post.text data["images"] = post.images else: data["images"] = [u.filename for u in UserImage.query(UserImage.date == date).fetch()] self.response.write(get_template('edit.html').render(data))
def post(self): import_task_key = ndb.Key(urlsafe=self.request.get('task')) import_task = import_task_key.get() import_task.update('Unpacking zip file...', status='inprogress') logging.info('Starting import ...') counter = PostCounter.get() try: posts, images = self.read_zip_file(import_task.uploaded_file) import_task.update('Importing...', total_photos=len(images), total_posts=len(posts)) logging.info('Importing %s posts, %s images' % (len(posts), len(images))) posts = self.filter_posts(posts) for date, text in posts: str_date = date.strftime('%Y-%m-%d') p = Post( date=date, source='ohlife', text=text.decode('utf-8') ) p.images = [] p.has_images = False post_images = [(k,images[k]) for k in images.keys() if str_date in k] if len(post_images): logging.info('Importing %s images for date %s' % (len(post_images), str_date)) p.images = [] p.has_images = True for name, bytes in post_images: user_image = UserImage() img_name = name.replace('img_', '').replace('.jpeg', '.jpg') user_image.import_image(img_name, name, bytes, date) p.images.append(img_name) import_task.imported_photos += 1 user_image.put() p.put() counter.increment(p.date.year, p.date.month, False) import_task.imported_posts += 1 if import_task.imported_posts % 10 == 0: import_task.update('Imported %s/%s post, %s/%s photos...' % (import_task.imported_posts, import_task.total_posts,import_task.imported_photos, import_task.total_photos)) logging.info(import_task.message) counter.put() counter.put() skipped_posts = import_task.total_posts - import_task.imported_posts skipped_photos = import_task.total_photos - import_task.imported_photos msg = 'Imported %s posts and %s photos.' % (import_task.imported_posts, import_task.imported_photos) if skipped_posts or skipped_photos: msg += ' %s posts and %s photos already existed and were skipped.' % (skipped_posts, skipped_photos) import_task.update(msg, status='finished') logging.info(import_task.message) filestore.delete(import_task.uploaded_file) except Exception, ex: try: filestore.delete(import_task.uploaded_file) except: pass try: counter.put() except: pass import_task.update('Failed to import: %s' % ex, status='failed') log_error('Failed import', traceback.format_exc(6))
def get(self): images_total = 0 images_backed_up = 0 try: self.response.headers['Content-Type'] = 'text/plain' settings = Settings.get() if not settings.dropbox_access_token: self.log( 'No access token available, no backup will be performed.') return posts = [p for p in Post.query().order(Post.date).fetch()] self.log('Backing up %s posts to Dropbox' % len(posts)) post_text = StringIO() for p in posts: post_text.write(p.date.strftime('%Y-%m-%d')) post_text.write('\r\n\r\n') post_text.write( p.text.replace('\r\n', '\n').replace('\n', '\r\n').rstrip()) post_text.write('\r\n\r\n') result = self.put_file(settings.dropbox_access_token, 'MyLife.txt', post_text.getvalue().encode('utf-8')) post_text.close() self.log('Backed up posts. Revision: %s' % result['rev']) self.log('Fetching Dropbox file list') files_in_dropbox = self.get_dropbox_filelist( settings.dropbox_access_token) self.log('Got %s files from Dropbox' % len(files_in_dropbox)) self.log('Fetching images...') images = [ i for i in UserImage.query().order(UserImage.date).fetch() ] self.log('Total images in MyLife: %s' % len(images)) not_backed_up = [i for i in images if not i.backed_up_in_dropbox] not_in_dropbox = [ i for i in images if not i.filename in files_in_dropbox ] self.log('\nFiles not backed up: \n\n' + '\n'.join([i.filename for i in not_backed_up])) self.log('\nFiles marked as backed up, but not in Dropbox: \n\n' + '\n'.join([i.filename for i in not_in_dropbox])) images = not_backed_up + not_in_dropbox images_total = len(images) self.log('Found %s images that need to be backed up in Dropbox' % images_total) for img in images: self.log('Backing up %s' % img.filename) bytes = filestore.read(img.original_size_key) result = self.put_file(settings.dropbox_access_token, img.filename, bytes) self.log('Backed up %s. Revision: %s' % (img.filename, result['rev'])) img.backed_up_in_dropbox = True img.put() images_backed_up += 1 settings.dropbox_last_backup = datetime.datetime.now() settings.put() self.log('Finished backup successfully') except apiproxy_errors.OverQuotaError, ex: self.log(ex) log_error( 'Error backing up to Dropbox, quota exceeded', 'The backup operation did not complete because it ran out of quota. ' + 'The next time it runs it will continue backing up your posts and images.' + '%s images out of %s were backed up before failing' % (images_backed_up, images_total))
def post(self): import_task_key = ndb.Key(urlsafe=self.request.get('task')) import_task = import_task_key.get() import_task.update('Unpacking zip file...', status='inprogress') logging.info('Starting import ...') counter = PostCounter.get() try: posts, images = self.read_zip_file(import_task.uploaded_file) import_task.update('Importing...', total_photos=len(images), total_posts=len(posts)) logging.info('Importing %s posts, %s images' % (len(posts), len(images))) posts = self.filter_posts(posts) for date, text in posts: str_date = date.strftime('%Y-%m-%d') p = Post(date=date, source='ohlife', text=text.decode('utf-8')) p.images = [] p.has_images = False post_images = [(k, images[k]) for k in images.keys() if str_date in k] if len(post_images): logging.info('Importing %s images for date %s' % (len(post_images), str_date)) p.images = [] p.has_images = True for name, bytes in post_images: user_image = UserImage() img_name = name.replace('img_', '').replace('.jpeg', '.jpg') user_image.import_image(img_name, name, bytes, date) p.images.append(img_name) import_task.imported_photos += 1 user_image.put() p.put() counter.increment(p.date.year, p.date.month, False) import_task.imported_posts += 1 if import_task.imported_posts % 10 == 0: import_task.update( 'Imported %s/%s post, %s/%s photos...' % (import_task.imported_posts, import_task.total_posts, import_task.imported_photos, import_task.total_photos)) logging.info(import_task.message) counter.put() counter.put() skipped_posts = import_task.total_posts - import_task.imported_posts skipped_photos = import_task.total_photos - import_task.imported_photos msg = 'Imported %s posts and %s photos.' % ( import_task.imported_posts, import_task.imported_photos) if skipped_posts or skipped_photos: msg += ' %s posts and %s photos already existed and were skipped.' % ( skipped_posts, skipped_photos) import_task.update(msg, status='finished') logging.info(import_task.message) filestore.delete(import_task.uploaded_file) except Exception, ex: try: filestore.delete(import_task.uploaded_file) except: pass try: counter.put() except: pass import_task.update('Failed to import: %s' % ex, status='failed') log_error('Failed import', traceback.format_exc(6))
def get(self): images_total = 0 images_backed_up = 0 try: self.response.headers['Content-Type'] = 'text/plain' settings = Settings.get() if not settings.dropbox_access_token: self.log('No access token available, no backup will be performed.') return posts = [p for p in Post.query().order(Post.date).fetch()] self.log('Backing up %s posts to Dropbox' % len(posts)) post_text = StringIO() for p in posts: post_text.write(p.date.strftime('%Y-%m-%d')) post_text.write('\r\n\r\n') post_text.write(p.text.replace('\r\n', '\n').replace('\n', '\r\n').rstrip()) post_text.write('\r\n\r\n') result = self.put_file(settings.dropbox_access_token, 'MyLife.txt', post_text.getvalue().encode('utf-8')) post_text.close() self.log('Backed up posts. Revision: %s' % result['rev']) self.log('Fetching Dropbox file list') files_in_dropbox = self.get_dropbox_filelist(settings.dropbox_access_token) self.log('Got %s files from Dropbox' % len(files_in_dropbox)) self.log('Fetching images...') images = [i for i in UserImage.query().order(UserImage.date).fetch()] self.log('Total images in MyLife: %s' % len(images)) not_backed_up = [i for i in images if not i.backed_up_in_dropbox] not_in_dropbox = [i for i in images if not i.filename in files_in_dropbox] self.log('\nFiles not backed up: \n\n' + '\n'.join([i.filename for i in not_backed_up])) self.log('\nFiles marked as backed up, but not in Dropbox: \n\n' + '\n'.join([i.filename for i in not_in_dropbox])) images = not_backed_up + not_in_dropbox images_total = len(images) self.log('Found %s images that need to be backed up in Dropbox' % images_total) for img in images: self.log('Backing up %s' % img.filename) bytes = filestore.read(img.original_size_key) result = self.put_file(settings.dropbox_access_token, img.filename, bytes) self.log('Backed up %s. Revision: %s' % (img.filename, result['rev'])) img.backed_up_in_dropbox = True img.put() images_backed_up += 1 settings.dropbox_last_backup = datetime.datetime.now() settings.put() self.log('Finished backup successfully') except apiproxy_errors.OverQuotaError, ex: self.log(ex) log_error('Error backing up to Dropbox, quota exceeded', 'The backup operation did not complete because it ran out of quota. ' + 'The next time it runs it will continue backing up your posts and images.' + '%s images out of %s were backed up before failing' % (images_backed_up, images_total))