def do_process_key(self, key): log.debug('{} getting file in temp storage'.format(key)) active_storage.copy(key, temp_storage) log.debug('{} creating resized images'.format(key)) create_resized_images(temp_storage.path(), key) log.debug('{} uploading files to active storage'.format(key)) for resized in resized_keys(key): temp_storage.move(resized, active_storage) temp_storage.delete(key) with self.lock: self.processed += 1
def upload(request): pre_key = create_pseudo_unique_key() log.debug('%s - received upload request', pre_key) # Store the original image as raw file raw_file = '%s/%s_raw' % (temp_storage.path(), pre_key) with stats.timer_context(['upload', 'read']): input_file = request.POST['file'].file input_file.seek(0) with open(raw_file, 'wb') as output_file: shutil.copyfileobj(input_file, output_file) log.debug('%s - copied raw file to %s', pre_key, output_file) try: kind = get_format(raw_file, request.POST['file'].filename) log.debug('%s - detected format is %s', pre_key, kind) except Exception: raise HTTPBadRequest('Bad format for %s' % request.POST['file'].filename) if kind == 'JPEG': kind = 'jpg' elif kind in ('PNG', 'GIF', 'SVG'): kind = kind.lower() else: raise HTTPBadRequest('Unsupported image format %s' % kind) # Rename to official extension original_key = "{}.{}".format(pre_key, kind) os.rename(raw_file, temp_storage.object_path(original_key)) if AUTO_ORIENT_ORIGINAL and kind == 'jpg': auto_orient(temp_storage.object_path(original_key)) create_resized_images(temp_storage.path(), original_key) log.debug('%s - uploading original file', pre_key) temp_storage.move(original_key, incoming_storage) for key in resized_keys(original_key): log.debug('%s - uploading resized image %s', pre_key, key) temp_storage.move(key, incoming_storage) log.debug('%s - returning response', pre_key) return {'filename': pre_key + '.' + kind}
def recrop(request): # Retrieve and rename file old_filename = request.POST['filename'] base, ext = os.path.splitext(old_filename) active_storage.copy(old_filename, temp_storage) filename = '{name}{ext}'.format(name=create_pseudo_unique_key(), ext=ext) os.rename(temp_storage.object_path(old_filename), temp_storage.object_path(filename)) temp_storage.copy(filename, active_storage) # Crop and generate thumbnails if 'crop' in request.POST: crop_options = request.POST['crop'] crop_and_publish_thumbs(filename, crop_options) else: create_resized_images(temp_storage.path(), filename) for key in resized_keys(filename): temp_storage.move(key, active_storage) temp_storage.delete(filename) return {'filename': filename}
def resizing_protocol(self): # for resizing object is in active storage self.active_storage.put(key, source_file) # get object in temp storage self.active_storage.copy(key, temp_storage) assert temp_storage.exists(key) # resizing ... # moved it back to active storage temp_storage.move(key, self.active_storage) assert self.active_storage.exists(key) assert not temp_storage.exists(key) # cleaning self.active_storage.delete(key) assert not self.active_storage.exists(key)
def recrop(request): if request.POST['secret'] != os.environ['API_SECRET_KEY']: raise HTTPForbidden('Bad secret key') # Retrieve and rename file old_filename = request.POST['filename'] base, ext = os.path.splitext(old_filename) active_storage.move(old_filename, temp_storage) filename = '{name}{ext}'.format(name=create_pseudo_unique_key(), ext=ext) os.rename(temp_storage.object_path(old_filename), temp_storage.object_path(filename)) temp_storage.copy(filename, active_storage) # Crop and generate thumbnails if 'crop' in request.POST: crop_options = request.POST['crop'] crop_and_publish_thumbs(filename, crop_options) else: create_resized_images(temp_storage.path(), filename) for key in resized_keys(filename): temp_storage.move(key, active_storage) temp_storage.delete(filename) return {'filename': filename}
def upload(request): pre_key = create_pseudo_unique_key() log.debug('%s - received upload request', pre_key) # Store the original image as raw file raw_file = '%s/%s_raw' % (temp_storage.path(), pre_key) with stats.timer_context(['upload', 'read']): input_file = request.POST['file'].file input_file.seek(0) with open(raw_file, 'wb') as output_file: shutil.copyfileobj(input_file, output_file) log.debug('%s - copied raw file to %s', pre_key, output_file) try: kind = get_format(raw_file, request.POST['file'].filename) log.debug('%s - detected format is %s', pre_key, kind) except: raise HTTPBadRequest('Bad format for %s' % request.POST['file'].filename) if kind == 'JPEG': kind = 'jpg' elif kind in ('PNG', 'GIF', 'SVG'): kind = kind.lower() else: raise HTTPBadRequest('Unsupported image format %s' % kind) # Rename to official extension original_key = "{}.{}".format(pre_key, kind) os.rename(raw_file, temp_storage.object_path(original_key)) create_resized_images(temp_storage.path(), original_key) log.debug('%s - uploading original file', pre_key) temp_storage.move(original_key, incoming_storage) for key in resized_keys(original_key): log.debug('%s - uploading resized image %s', pre_key, key) temp_storage.move(key, incoming_storage) log.debug('%s - returning response', pre_key) return {'filename': pre_key + '.' + kind}
def standard_protocol(self): # put file in temp storage for processing temp_storage.put(key, source_file) # move it in incoming storage waiting for publication temp_storage.move(key, self.incoming_storage) assert self.incoming_storage.exists(key) assert not temp_storage.exists(key) # Ensure the file is private if isinstance(self.incoming_storage, S3Storage): url = '{}/{}/{}'.format(self.incoming_storage._endpoint_url, self.incoming_storage._bucket_name, key) r = requests.get(url, stream=True, timeout=120) assert r.status_code == 403 # on publishing it is moved to active storage self.incoming_storage.move(key, self.active_storage) assert self.active_storage.exists(key) assert not self.incoming_storage.exists(key) # Ensure the file is public if isinstance(self.active_storage, S3Storage): url = '{}/{}/{}'.format(self.active_storage._endpoint_url, self.active_storage._bucket_name, key) r = requests.get(url, stream=True, timeout=120) assert r.status_code == 200 assert r.headers['content-type'] == 'image/png' # cleaning self.active_storage.delete(key) assert not self.active_storage.exists(key) # delete file that does not exists self.active_storage.delete('test_not_exists.jpg')
def standard_protocol(self): # put file in temp storage for processing temp_storage.put(key, source_file) # move it in incoming storage waiting for publication temp_storage.move(key, self.incoming_storage) assert self.incoming_storage.exists(key) assert not temp_storage.exists(key) # Ensure the file is private if isinstance(self.incoming_storage, S3Storage): url = '{}/{}/{}'.format(self.incoming_storage._endpoint_url, self.incoming_storage._bucket_name, key) r = requests.get(url, stream=True, timeout=120) assert r.status_code == 403 # on publishing it is moved to active storage self.incoming_storage.move(key, self.active_storage) assert self.active_storage.exists(key) assert not self.incoming_storage.exists(key) # Ensure the file is public if isinstance(self.active_storage, S3Storage): url = '{}/{}/{}'.format(self.active_storage._endpoint_url, self.active_storage._bucket_name, key) r = requests.get(url, stream=True, timeout=120) assert r.status_code == 200 assert r.headers['content-type'] == 'image/png' assert r.headers['cache-control'] == 'public, max-age=3600' # cleaning self.active_storage.delete(key) assert not self.active_storage.exists(key) # delete file that does not exists self.active_storage.delete('test_not_exists.jpg')
def do_process_key(self, key): base, ext = os.path.splitext(key) if ext == '.svg': for rasterized in ('{}.jpg'.format(base), '{}.png'.format(base)): if active_storage.exists(rasterized): log.info("{} delete file {}".format(key, rasterized)) active_storage.delete(rasterized) to_create = [key] + resized_keys(key) if not self.force: for key_to_create in list(to_create): if self._is_migrated(key_to_create): to_create.remove(key_to_create) if len(to_create) == 0: log.debug('{} skipping'.format(key)) with self.lock: self.skipped += 1 return log.debug('{} getting file in temp storage'.format(key)) tries = 3 success = False while tries > 0 and not success: try: with requests.get( 'http://s.camptocamp.org/uploads/images/{}'.format( key), stream=True, timeout=120) as r: if r.status_code != 200: log.error("{} return status code {} - {}".format( key, r.status_code, r.reason)) with self.lock: self.errors += 1 return with open(temp_storage.object_path(key), 'wb') as fd: for chunk in r.iter_content(None): fd.write(chunk) success = True except Exception as e: tries -= 1 if tries > 0: log.warning("{} retry download".format(key)) else: raise e log.debug('{} creating resized images'.format(key)) create_resized_images(temp_storage.path(), key) log.debug('{} uploading files to active storage'.format(key)) for key_to_create in to_create: if temp_storage.exists(key_to_create): log.debug('{} uploading {}'.format(key, key_to_create)) temp_storage.move(key_to_create, active_storage) assert active_storage.exists(key_to_create) self._set_migrated(key_to_create) else: log.warning( '{} File does not exists, skipping upload of {}'.format( key, key_to_create)) with self.lock: self.processed += 1
def crop_and_publish_thumbs(filename, crop_options): create_cropped_image(temp_storage.path(), filename, crop_options) create_resized_images(temp_storage.path(), filename) for key in resized_keys(filename): temp_storage.move(key, active_storage)