def do_process_key(self, key):
log.debug('{} getting file in temp storage'.format(key))
active_storage.copy(key, temp_storage)
log.debug('{} creating resized images'.format(key))
create_resized_images(temp_storage.path(), key)
log.debug('{} uploading files to active storage'.format(key))
for resized in resized_keys(key):
temp_storage.move(resized, active_storage)
temp_storage.delete(key)
with self.lock:
self.processed += 1
def do_process_key(self, key):
log.debug('{} getting file in temp storage'.format(key))
active_storage.copy(key, temp_storage)
log.debug('{} creating resized images'.format(key))
create_resized_images(temp_storage.path(), key)
log.debug('{} uploading files to active storage'.format(key))
for resized in resized_keys(key):
temp_storage.move(resized, active_storage)
temp_storage.delete(key)
with self.lock:
self.processed += 1
def upload(request):
pre_key = create_pseudo_unique_key()
log.debug('%s - received upload request', pre_key)
# Store the original image as raw file
raw_file = '%s/%s_raw' % (temp_storage.path(), pre_key)
with stats.timer_context(['upload', 'read']):
input_file = request.POST['file'].file
input_file.seek(0)
with open(raw_file, 'wb') as output_file:
shutil.copyfileobj(input_file, output_file)
log.debug('%s - copied raw file to %s', pre_key, output_file)
try:
kind = get_format(raw_file, request.POST['file'].filename)
log.debug('%s - detected format is %s', pre_key, kind)
except Exception:
raise HTTPBadRequest('Bad format for %s' %
request.POST['file'].filename)
if kind == 'JPEG':
kind = 'jpg'
elif kind in ('PNG', 'GIF', 'SVG'):
kind = kind.lower()
else:
raise HTTPBadRequest('Unsupported image format %s' % kind)
# Rename to official extension
original_key = "{}.{}".format(pre_key, kind)
os.rename(raw_file, temp_storage.object_path(original_key))
if AUTO_ORIENT_ORIGINAL and kind == 'jpg':
auto_orient(temp_storage.object_path(original_key))
create_resized_images(temp_storage.path(), original_key)
log.debug('%s - uploading original file', pre_key)
temp_storage.move(original_key, incoming_storage)
for key in resized_keys(original_key):
log.debug('%s - uploading resized image %s', pre_key, key)
temp_storage.move(key, incoming_storage)
log.debug('%s - returning response', pre_key)
return {'filename': pre_key + '.' + kind}
def recrop(request):
# Retrieve and rename file
old_filename = request.POST['filename']
base, ext = os.path.splitext(old_filename)
active_storage.copy(old_filename, temp_storage)
filename = '{name}{ext}'.format(name=create_pseudo_unique_key(), ext=ext)
os.rename(temp_storage.object_path(old_filename), temp_storage.object_path(filename))
temp_storage.copy(filename, active_storage)
# Crop and generate thumbnails
if 'crop' in request.POST:
crop_options = request.POST['crop']
crop_and_publish_thumbs(filename, crop_options)
else:
create_resized_images(temp_storage.path(), filename)
for key in resized_keys(filename):
temp_storage.move(key, active_storage)
temp_storage.delete(filename)
return {'filename': filename}
def resizing_protocol(self):
# for resizing object is in active storage
self.active_storage.put(key, source_file)
# get object in temp storage
self.active_storage.copy(key, temp_storage)
assert temp_storage.exists(key)
# resizing ...
# moved it back to active storage
temp_storage.move(key, self.active_storage)
assert self.active_storage.exists(key)
assert not temp_storage.exists(key)
# cleaning
self.active_storage.delete(key)
assert not self.active_storage.exists(key)
def resizing_protocol(self):
# for resizing object is in active storage
self.active_storage.put(key, source_file)
# get object in temp storage
self.active_storage.copy(key, temp_storage)
assert temp_storage.exists(key)
# resizing ...
# moved it back to active storage
temp_storage.move(key, self.active_storage)
assert self.active_storage.exists(key)
assert not temp_storage.exists(key)
# cleaning
self.active_storage.delete(key)
assert not self.active_storage.exists(key)
def recrop(request):
if request.POST['secret'] != os.environ['API_SECRET_KEY']:
raise HTTPForbidden('Bad secret key')
# Retrieve and rename file
old_filename = request.POST['filename']
base, ext = os.path.splitext(old_filename)
active_storage.move(old_filename, temp_storage)
filename = '{name}{ext}'.format(name=create_pseudo_unique_key(), ext=ext)
os.rename(temp_storage.object_path(old_filename), temp_storage.object_path(filename))
temp_storage.copy(filename, active_storage)
# Crop and generate thumbnails
if 'crop' in request.POST:
crop_options = request.POST['crop']
crop_and_publish_thumbs(filename, crop_options)
else:
create_resized_images(temp_storage.path(), filename)
for key in resized_keys(filename):
temp_storage.move(key, active_storage)
temp_storage.delete(filename)
return {'filename': filename}
def upload(request):
pre_key = create_pseudo_unique_key()
log.debug('%s - received upload request', pre_key)
# Store the original image as raw file
raw_file = '%s/%s_raw' % (temp_storage.path(), pre_key)
with stats.timer_context(['upload', 'read']):
input_file = request.POST['file'].file
input_file.seek(0)
with open(raw_file, 'wb') as output_file:
shutil.copyfileobj(input_file, output_file)
log.debug('%s - copied raw file to %s', pre_key, output_file)
try:
kind = get_format(raw_file, request.POST['file'].filename)
log.debug('%s - detected format is %s', pre_key, kind)
except:
raise HTTPBadRequest('Bad format for %s' % request.POST['file'].filename)
if kind == 'JPEG':
kind = 'jpg'
elif kind in ('PNG', 'GIF', 'SVG'):
kind = kind.lower()
else:
raise HTTPBadRequest('Unsupported image format %s' % kind)
# Rename to official extension
original_key = "{}.{}".format(pre_key, kind)
os.rename(raw_file, temp_storage.object_path(original_key))
create_resized_images(temp_storage.path(), original_key)
log.debug('%s - uploading original file', pre_key)
temp_storage.move(original_key, incoming_storage)
for key in resized_keys(original_key):
log.debug('%s - uploading resized image %s', pre_key, key)
temp_storage.move(key, incoming_storage)
log.debug('%s - returning response', pre_key)
return {'filename': pre_key + '.' + kind}
def standard_protocol(self):
# put file in temp storage for processing
temp_storage.put(key, source_file)
# move it in incoming storage waiting for publication
temp_storage.move(key, self.incoming_storage)
assert self.incoming_storage.exists(key)
assert not temp_storage.exists(key)
# Ensure the file is private
if isinstance(self.incoming_storage, S3Storage):
url = '{}/{}/{}'.format(self.incoming_storage._endpoint_url,
self.incoming_storage._bucket_name,
key)
r = requests.get(url, stream=True, timeout=120)
assert r.status_code == 403
# on publishing it is moved to active storage
self.incoming_storage.move(key, self.active_storage)
assert self.active_storage.exists(key)
assert not self.incoming_storage.exists(key)
# Ensure the file is public
if isinstance(self.active_storage, S3Storage):
url = '{}/{}/{}'.format(self.active_storage._endpoint_url,
self.active_storage._bucket_name,
key)
r = requests.get(url, stream=True, timeout=120)
assert r.status_code == 200
assert r.headers['content-type'] == 'image/png'
# cleaning
self.active_storage.delete(key)
assert not self.active_storage.exists(key)
# delete file that does not exists
self.active_storage.delete('test_not_exists.jpg')
def standard_protocol(self):
# put file in temp storage for processing
temp_storage.put(key, source_file)
# move it in incoming storage waiting for publication
temp_storage.move(key, self.incoming_storage)
assert self.incoming_storage.exists(key)
assert not temp_storage.exists(key)
# Ensure the file is private
if isinstance(self.incoming_storage, S3Storage):
url = '{}/{}/{}'.format(self.incoming_storage._endpoint_url,
self.incoming_storage._bucket_name, key)
r = requests.get(url, stream=True, timeout=120)
assert r.status_code == 403
# on publishing it is moved to active storage
self.incoming_storage.move(key, self.active_storage)
assert self.active_storage.exists(key)
assert not self.incoming_storage.exists(key)
# Ensure the file is public
if isinstance(self.active_storage, S3Storage):
url = '{}/{}/{}'.format(self.active_storage._endpoint_url,
self.active_storage._bucket_name, key)
r = requests.get(url, stream=True, timeout=120)
assert r.status_code == 200
assert r.headers['content-type'] == 'image/png'
assert r.headers['cache-control'] == 'public, max-age=3600'
# cleaning
self.active_storage.delete(key)
assert not self.active_storage.exists(key)
# delete file that does not exists
self.active_storage.delete('test_not_exists.jpg')
def do_process_key(self, key):
base, ext = os.path.splitext(key)
if ext == '.svg':
for rasterized in ('{}.jpg'.format(base), '{}.png'.format(base)):
if active_storage.exists(rasterized):
log.info("{} delete file {}".format(key, rasterized))
active_storage.delete(rasterized)
to_create = [key] + resized_keys(key)
if not self.force:
for key_to_create in list(to_create):
if self._is_migrated(key_to_create):
to_create.remove(key_to_create)
if len(to_create) == 0:
log.debug('{} skipping'.format(key))
with self.lock:
self.skipped += 1
return
log.debug('{} getting file in temp storage'.format(key))
tries = 3
success = False
while tries > 0 and not success:
try:
with requests.get(
'http://s.camptocamp.org/uploads/images/{}'.format(
key),
stream=True,
timeout=120) as r:
if r.status_code != 200:
log.error("{} return status code {} - {}".format(
key, r.status_code, r.reason))
with self.lock:
self.errors += 1
return
with open(temp_storage.object_path(key), 'wb') as fd:
for chunk in r.iter_content(None):
fd.write(chunk)
success = True
except Exception as e:
tries -= 1
if tries > 0:
log.warning("{} retry download".format(key))
else:
raise e
log.debug('{} creating resized images'.format(key))
create_resized_images(temp_storage.path(), key)
log.debug('{} uploading files to active storage'.format(key))
for key_to_create in to_create:
if temp_storage.exists(key_to_create):
log.debug('{} uploading {}'.format(key, key_to_create))
temp_storage.move(key_to_create, active_storage)
assert active_storage.exists(key_to_create)
self._set_migrated(key_to_create)
else:
log.warning(
'{} File does not exists, skipping upload of {}'.format(
key, key_to_create))
with self.lock:
self.processed += 1
def crop_and_publish_thumbs(filename, crop_options):
create_cropped_image(temp_storage.path(), filename, crop_options)
create_resized_images(temp_storage.path(), filename)
for key in resized_keys(filename):
temp_storage.move(key, active_storage)
def crop_and_publish_thumbs(filename, crop_options):
create_cropped_image(temp_storage.path(), filename, crop_options)
create_resized_images(temp_storage.path(), filename)
for key in resized_keys(filename):
temp_storage.move(key, active_storage)
