def test_resize(): # clear active storage for key in active_storage.keys(): active_storage.delete(key) # put some file in active storage key = v5_key shutil.copyfile(os.path.join(data_folder, 'v5_images', v5_key), active_storage.object_path(v5_key)) for resized in resized_keys(key): shutil.copyfile(os.path.join(data_folder, 'v5_images', resized), active_storage.object_path(resized)) mtimes = {} for resized in resized_keys(key): mtimes[resized] = active_storage.last_modified(resized) resizer = Resizer() resizer.execute() assert resizer.total == 1 assert resizer.skipped == 0 assert resizer.processed == 1 assert resizer.errors == 0 for key, value in mtimes.items(): assert active_storage.last_modified(key) != value
def do_process_key(self, key): log.debug('{} getting file in temp storage'.format(key)) active_storage.copy(key, temp_storage) log.debug('{} creating resized images'.format(key)) create_resized_images(temp_storage.path(), key) log.debug('{} uploading files to active storage'.format(key)) for resized in resized_keys(key): temp_storage.move(resized, active_storage) temp_storage.delete(key) with self.lock: self.processed += 1
def do_process_key(self, key): log.debug('{} getting file in temp storage'.format(key)) active_storage.copy(key, temp_storage) log.debug('{} creating resized images'.format(key)) create_resized_images(temp_storage.path(), key) log.debug('{} uploading files to active storage'.format(key)) for resized in resized_keys(key): temp_storage.move(resized, active_storage) temp_storage.delete(key) with self.lock: self.processed += 1
def delete(request): filenames = request.POST.getall('filenames') for filename in filenames: try: active_storage.delete(filename) except Exception: log.error('Deleting {} failed'.format(filename), exc_info=True) for key in resized_keys(filename): try: active_storage.delete(key) except Exception: log.error('Deleting {} failed'.format(key), exc_info=True) return {'success': True}
def delete(request): if request.POST['secret'] != os.environ['API_SECRET_KEY']: raise HTTPForbidden('Bad secret key') filenames = request.POST.getall('filenames') for filename in filenames: try: active_storage.delete(filename) except Exception: log.error('Deleting {} failed'.format(filename), exc_info=True) for key in resized_keys(filename): try: active_storage.delete(key) except Exception: log.error('Deleting {} failed'.format(key), exc_info=True) return {'success': True}
def upload(request): pre_key = create_pseudo_unique_key() log.debug('%s - received upload request', pre_key) # Store the original image as raw file raw_file = '%s/%s_raw' % (temp_storage.path(), pre_key) with stats.timer_context(['upload', 'read']): input_file = request.POST['file'].file input_file.seek(0) with open(raw_file, 'wb') as output_file: shutil.copyfileobj(input_file, output_file) log.debug('%s - copied raw file to %s', pre_key, output_file) try: kind = get_format(raw_file, request.POST['file'].filename) log.debug('%s - detected format is %s', pre_key, kind) except Exception: raise HTTPBadRequest('Bad format for %s' % request.POST['file'].filename) if kind == 'JPEG': kind = 'jpg' elif kind in ('PNG', 'GIF', 'SVG'): kind = kind.lower() else: raise HTTPBadRequest('Unsupported image format %s' % kind) # Rename to official extension original_key = "{}.{}".format(pre_key, kind) os.rename(raw_file, temp_storage.object_path(original_key)) if AUTO_ORIENT_ORIGINAL and kind == 'jpg': auto_orient(temp_storage.object_path(original_key)) create_resized_images(temp_storage.path(), original_key) log.debug('%s - uploading original file', pre_key) temp_storage.move(original_key, incoming_storage) for key in resized_keys(original_key): log.debug('%s - uploading resized image %s', pre_key, key) temp_storage.move(key, incoming_storage) log.debug('%s - returning response', pre_key) return {'filename': pre_key + '.' + kind}
def recrop(request): # Retrieve and rename file old_filename = request.POST['filename'] base, ext = os.path.splitext(old_filename) active_storage.copy(old_filename, temp_storage) filename = '{name}{ext}'.format(name=create_pseudo_unique_key(), ext=ext) os.rename(temp_storage.object_path(old_filename), temp_storage.object_path(filename)) temp_storage.copy(filename, active_storage) # Crop and generate thumbnails if 'crop' in request.POST: crop_options = request.POST['crop'] crop_and_publish_thumbs(filename, crop_options) else: create_resized_images(temp_storage.path(), filename) for key in resized_keys(filename): temp_storage.move(key, active_storage) temp_storage.delete(filename) return {'filename': filename}
def publish(request): filename = request.POST['filename'] already_published = active_storage.exists(filename) if 'crop' in request.POST: crop_options = request.POST['crop'] if already_published: active_storage.copy(filename, temp_storage) else: incoming_storage.copy(filename, temp_storage) crop_and_publish_thumbs(filename, crop_options) temp_storage.delete(filename) else: for key in resized_keys(filename): if incoming_storage.exists(key): incoming_storage.move(key, active_storage) if not already_published: incoming_storage.move(filename, active_storage) return {'success': True}
def recrop(request): if request.POST['secret'] != os.environ['API_SECRET_KEY']: raise HTTPForbidden('Bad secret key') # Retrieve and rename file old_filename = request.POST['filename'] base, ext = os.path.splitext(old_filename) active_storage.move(old_filename, temp_storage) filename = '{name}{ext}'.format(name=create_pseudo_unique_key(), ext=ext) os.rename(temp_storage.object_path(old_filename), temp_storage.object_path(filename)) temp_storage.copy(filename, active_storage) # Crop and generate thumbnails if 'crop' in request.POST: crop_options = request.POST['crop'] crop_and_publish_thumbs(filename, crop_options) else: create_resized_images(temp_storage.path(), filename) for key in resized_keys(filename): temp_storage.move(key, active_storage) temp_storage.delete(filename) return {'filename': filename}
def publish(request): if request.POST['secret'] != os.environ['API_SECRET_KEY']: raise HTTPForbidden('Bad secret key') filename = request.POST['filename'] already_published = active_storage.exists(filename) if 'crop' in request.POST: crop_options = request.POST['crop'] if already_published: active_storage.copy(filename, temp_storage) else: incoming_storage.copy(filename, temp_storage) crop_and_publish_thumbs(filename, crop_options) temp_storage.delete(filename) else: for key in resized_keys(filename): if incoming_storage.exists(key): incoming_storage.move(key, active_storage) if not already_published: incoming_storage.move(filename, active_storage) return {'success': True}
def upload(request): pre_key = create_pseudo_unique_key() log.debug('%s - received upload request', pre_key) # Store the original image as raw file raw_file = '%s/%s_raw' % (temp_storage.path(), pre_key) with stats.timer_context(['upload', 'read']): input_file = request.POST['file'].file input_file.seek(0) with open(raw_file, 'wb') as output_file: shutil.copyfileobj(input_file, output_file) log.debug('%s - copied raw file to %s', pre_key, output_file) try: kind = get_format(raw_file, request.POST['file'].filename) log.debug('%s - detected format is %s', pre_key, kind) except: raise HTTPBadRequest('Bad format for %s' % request.POST['file'].filename) if kind == 'JPEG': kind = 'jpg' elif kind in ('PNG', 'GIF', 'SVG'): kind = kind.lower() else: raise HTTPBadRequest('Unsupported image format %s' % kind) # Rename to official extension original_key = "{}.{}".format(pre_key, kind) os.rename(raw_file, temp_storage.object_path(original_key)) create_resized_images(temp_storage.path(), original_key) log.debug('%s - uploading original file', pre_key) temp_storage.move(original_key, incoming_storage) for key in resized_keys(original_key): log.debug('%s - uploading resized image %s', pre_key, key) temp_storage.move(key, incoming_storage) log.debug('%s - returning response', pre_key) return {'filename': pre_key + '.' + kind}
def test_migrate(create_engine): migrator = Migrator() migrator.execute(jobs=1) assert migrator.total == 2 assert migrator.skipped == 0 assert migrator.processed == 2 assert migrator.errors == 0 for key in [file['filename'] for file in files]: if key == '1274781175_1003161687.jpg': assert active_storage.exists('1274781175_1003161687.svg') else: assert active_storage.exists(key) for resized in resized_keys(key): assert active_storage.exists(resized) migrator.execute() assert migrator.total == 2 assert migrator.skipped == 2 assert migrator.processed == 0 assert migrator.errors == 0
def test_migrate(create_engine): migrator = Migrator() migrator.execute(jobs=1) assert migrator.total == 2 assert migrator.skipped == 0 assert migrator.processed == 2 assert migrator.errors == 0 for key in [file['filename'] for file in files]: if key == '1274781175_1003161687.jpg': assert active_storage.exists('1274781175_1003161687.svg') else: assert active_storage.exists(key) for resized in resized_keys(key): assert active_storage.exists(resized) migrator.execute() assert migrator.total == 2 assert migrator.skipped == 2 assert migrator.processed == 0 assert migrator.errors == 0
def do_process_key(self, key): base, ext = os.path.splitext(key) if ext == '.svg': for rasterized in ('{}.jpg'.format(base), '{}.png'.format(base)): if active_storage.exists(rasterized): log.info("{} delete file {}".format(key, rasterized)) active_storage.delete(rasterized) to_create = [key] + resized_keys(key) if not self.force: for key_to_create in list(to_create): if self._is_migrated(key_to_create): to_create.remove(key_to_create) if len(to_create) == 0: log.debug('{} skipping'.format(key)) with self.lock: self.skipped += 1 return log.debug('{} getting file in temp storage'.format(key)) tries = 3 success = False while tries > 0 and not success: try: with requests.get( 'http://s.camptocamp.org/uploads/images/{}'.format( key), stream=True, timeout=120) as r: if r.status_code != 200: log.error("{} return status code {} - {}".format( key, r.status_code, r.reason)) with self.lock: self.errors += 1 return with open(temp_storage.object_path(key), 'wb') as fd: for chunk in r.iter_content(None): fd.write(chunk) success = True except Exception as e: tries -= 1 if tries > 0: log.warning("{} retry download".format(key)) else: raise e log.debug('{} creating resized images'.format(key)) create_resized_images(temp_storage.path(), key) log.debug('{} uploading files to active storage'.format(key)) for key_to_create in to_create: if temp_storage.exists(key_to_create): log.debug('{} uploading {}'.format(key, key_to_create)) temp_storage.move(key_to_create, active_storage) assert active_storage.exists(key_to_create) self._set_migrated(key_to_create) else: log.warning( '{} File does not exists, skipping upload of {}'.format( key, key_to_create)) with self.lock: self.processed += 1
def crop_and_publish_thumbs(filename, crop_options): create_cropped_image(temp_storage.path(), filename, crop_options) create_resized_images(temp_storage.path(), filename) for key in resized_keys(filename): temp_storage.move(key, active_storage)
def crop_and_publish_thumbs(filename, crop_options): create_cropped_image(temp_storage.path(), filename, crop_options) create_resized_images(temp_storage.path(), filename) for key in resized_keys(filename): temp_storage.move(key, active_storage)