def test_resize():
# clear active storage
for key in active_storage.keys():
active_storage.delete(key)
# put some file in active storage
key = v5_key
shutil.copyfile(os.path.join(data_folder, 'v5_images', v5_key),
active_storage.object_path(v5_key))
for resized in resized_keys(key):
shutil.copyfile(os.path.join(data_folder, 'v5_images', resized),
active_storage.object_path(resized))
mtimes = {}
for resized in resized_keys(key):
mtimes[resized] = active_storage.last_modified(resized)
resizer = Resizer()
resizer.execute()
assert resizer.total == 1
assert resizer.skipped == 0
assert resizer.processed == 1
assert resizer.errors == 0
for key, value in mtimes.items():
assert active_storage.last_modified(key) != value
def do_process_key(self, key):
log.debug('{} getting file in temp storage'.format(key))
active_storage.copy(key, temp_storage)
log.debug('{} creating resized images'.format(key))
create_resized_images(temp_storage.path(), key)
log.debug('{} uploading files to active storage'.format(key))
for resized in resized_keys(key):
temp_storage.move(resized, active_storage)
temp_storage.delete(key)
with self.lock:
self.processed += 1
def do_process_key(self, key):
log.debug('{} getting file in temp storage'.format(key))
active_storage.copy(key, temp_storage)
log.debug('{} creating resized images'.format(key))
create_resized_images(temp_storage.path(), key)
log.debug('{} uploading files to active storage'.format(key))
for resized in resized_keys(key):
temp_storage.move(resized, active_storage)
temp_storage.delete(key)
with self.lock:
self.processed += 1
def delete(request):
filenames = request.POST.getall('filenames')
for filename in filenames:
try:
active_storage.delete(filename)
except Exception:
log.error('Deleting {} failed'.format(filename), exc_info=True)
for key in resized_keys(filename):
try:
active_storage.delete(key)
except Exception:
log.error('Deleting {} failed'.format(key), exc_info=True)
return {'success': True}
def delete(request):
if request.POST['secret'] != os.environ['API_SECRET_KEY']:
raise HTTPForbidden('Bad secret key')
filenames = request.POST.getall('filenames')
for filename in filenames:
try:
active_storage.delete(filename)
except Exception:
log.error('Deleting {} failed'.format(filename), exc_info=True)
for key in resized_keys(filename):
try:
active_storage.delete(key)
except Exception:
log.error('Deleting {} failed'.format(key), exc_info=True)
return {'success': True}
def upload(request):
pre_key = create_pseudo_unique_key()
log.debug('%s - received upload request', pre_key)
# Store the original image as raw file
raw_file = '%s/%s_raw' % (temp_storage.path(), pre_key)
with stats.timer_context(['upload', 'read']):
input_file = request.POST['file'].file
input_file.seek(0)
with open(raw_file, 'wb') as output_file:
shutil.copyfileobj(input_file, output_file)
log.debug('%s - copied raw file to %s', pre_key, output_file)
try:
kind = get_format(raw_file, request.POST['file'].filename)
log.debug('%s - detected format is %s', pre_key, kind)
except Exception:
raise HTTPBadRequest('Bad format for %s' %
request.POST['file'].filename)
if kind == 'JPEG':
kind = 'jpg'
elif kind in ('PNG', 'GIF', 'SVG'):
kind = kind.lower()
else:
raise HTTPBadRequest('Unsupported image format %s' % kind)
# Rename to official extension
original_key = "{}.{}".format(pre_key, kind)
os.rename(raw_file, temp_storage.object_path(original_key))
if AUTO_ORIENT_ORIGINAL and kind == 'jpg':
auto_orient(temp_storage.object_path(original_key))
create_resized_images(temp_storage.path(), original_key)
log.debug('%s - uploading original file', pre_key)
temp_storage.move(original_key, incoming_storage)
for key in resized_keys(original_key):
log.debug('%s - uploading resized image %s', pre_key, key)
temp_storage.move(key, incoming_storage)
log.debug('%s - returning response', pre_key)
return {'filename': pre_key + '.' + kind}
def recrop(request):
# Retrieve and rename file
old_filename = request.POST['filename']
base, ext = os.path.splitext(old_filename)
active_storage.copy(old_filename, temp_storage)
filename = '{name}{ext}'.format(name=create_pseudo_unique_key(), ext=ext)
os.rename(temp_storage.object_path(old_filename), temp_storage.object_path(filename))
temp_storage.copy(filename, active_storage)
# Crop and generate thumbnails
if 'crop' in request.POST:
crop_options = request.POST['crop']
crop_and_publish_thumbs(filename, crop_options)
else:
create_resized_images(temp_storage.path(), filename)
for key in resized_keys(filename):
temp_storage.move(key, active_storage)
temp_storage.delete(filename)
return {'filename': filename}
def publish(request):
filename = request.POST['filename']
already_published = active_storage.exists(filename)
if 'crop' in request.POST:
crop_options = request.POST['crop']
if already_published:
active_storage.copy(filename, temp_storage)
else:
incoming_storage.copy(filename, temp_storage)
crop_and_publish_thumbs(filename, crop_options)
temp_storage.delete(filename)
else:
for key in resized_keys(filename):
if incoming_storage.exists(key):
incoming_storage.move(key, active_storage)
if not already_published:
incoming_storage.move(filename, active_storage)
return {'success': True}
def recrop(request):
if request.POST['secret'] != os.environ['API_SECRET_KEY']:
raise HTTPForbidden('Bad secret key')
# Retrieve and rename file
old_filename = request.POST['filename']
base, ext = os.path.splitext(old_filename)
active_storage.move(old_filename, temp_storage)
filename = '{name}{ext}'.format(name=create_pseudo_unique_key(), ext=ext)
os.rename(temp_storage.object_path(old_filename), temp_storage.object_path(filename))
temp_storage.copy(filename, active_storage)
# Crop and generate thumbnails
if 'crop' in request.POST:
crop_options = request.POST['crop']
crop_and_publish_thumbs(filename, crop_options)
else:
create_resized_images(temp_storage.path(), filename)
for key in resized_keys(filename):
temp_storage.move(key, active_storage)
temp_storage.delete(filename)
return {'filename': filename}
def publish(request):
if request.POST['secret'] != os.environ['API_SECRET_KEY']:
raise HTTPForbidden('Bad secret key')
filename = request.POST['filename']
already_published = active_storage.exists(filename)
if 'crop' in request.POST:
crop_options = request.POST['crop']
if already_published:
active_storage.copy(filename, temp_storage)
else:
incoming_storage.copy(filename, temp_storage)
crop_and_publish_thumbs(filename, crop_options)
temp_storage.delete(filename)
else:
for key in resized_keys(filename):
if incoming_storage.exists(key):
incoming_storage.move(key, active_storage)
if not already_published:
incoming_storage.move(filename, active_storage)
return {'success': True}
def upload(request):
pre_key = create_pseudo_unique_key()
log.debug('%s - received upload request', pre_key)
# Store the original image as raw file
raw_file = '%s/%s_raw' % (temp_storage.path(), pre_key)
with stats.timer_context(['upload', 'read']):
input_file = request.POST['file'].file
input_file.seek(0)
with open(raw_file, 'wb') as output_file:
shutil.copyfileobj(input_file, output_file)
log.debug('%s - copied raw file to %s', pre_key, output_file)
try:
kind = get_format(raw_file, request.POST['file'].filename)
log.debug('%s - detected format is %s', pre_key, kind)
except:
raise HTTPBadRequest('Bad format for %s' % request.POST['file'].filename)
if kind == 'JPEG':
kind = 'jpg'
elif kind in ('PNG', 'GIF', 'SVG'):
kind = kind.lower()
else:
raise HTTPBadRequest('Unsupported image format %s' % kind)
# Rename to official extension
original_key = "{}.{}".format(pre_key, kind)
os.rename(raw_file, temp_storage.object_path(original_key))
create_resized_images(temp_storage.path(), original_key)
log.debug('%s - uploading original file', pre_key)
temp_storage.move(original_key, incoming_storage)
for key in resized_keys(original_key):
log.debug('%s - uploading resized image %s', pre_key, key)
temp_storage.move(key, incoming_storage)
log.debug('%s - returning response', pre_key)
return {'filename': pre_key + '.' + kind}
def test_migrate(create_engine):
migrator = Migrator()
migrator.execute(jobs=1)
assert migrator.total == 2
assert migrator.skipped == 0
assert migrator.processed == 2
assert migrator.errors == 0
for key in [file['filename'] for file in files]:
if key == '1274781175_1003161687.jpg':
assert active_storage.exists('1274781175_1003161687.svg')
else:
assert active_storage.exists(key)
for resized in resized_keys(key):
assert active_storage.exists(resized)
migrator.execute()
assert migrator.total == 2
assert migrator.skipped == 2
assert migrator.processed == 0
assert migrator.errors == 0
def test_migrate(create_engine):
migrator = Migrator()
migrator.execute(jobs=1)
assert migrator.total == 2
assert migrator.skipped == 0
assert migrator.processed == 2
assert migrator.errors == 0
for key in [file['filename'] for file in files]:
if key == '1274781175_1003161687.jpg':
assert active_storage.exists('1274781175_1003161687.svg')
else:
assert active_storage.exists(key)
for resized in resized_keys(key):
assert active_storage.exists(resized)
migrator.execute()
assert migrator.total == 2
assert migrator.skipped == 2
assert migrator.processed == 0
assert migrator.errors == 0
def do_process_key(self, key):
base, ext = os.path.splitext(key)
if ext == '.svg':
for rasterized in ('{}.jpg'.format(base), '{}.png'.format(base)):
if active_storage.exists(rasterized):
log.info("{} delete file {}".format(key, rasterized))
active_storage.delete(rasterized)
to_create = [key] + resized_keys(key)
if not self.force:
for key_to_create in list(to_create):
if self._is_migrated(key_to_create):
to_create.remove(key_to_create)
if len(to_create) == 0:
log.debug('{} skipping'.format(key))
with self.lock:
self.skipped += 1
return
log.debug('{} getting file in temp storage'.format(key))
tries = 3
success = False
while tries > 0 and not success:
try:
with requests.get(
'http://s.camptocamp.org/uploads/images/{}'.format(
key),
stream=True,
timeout=120) as r:
if r.status_code != 200:
log.error("{} return status code {} - {}".format(
key, r.status_code, r.reason))
with self.lock:
self.errors += 1
return
with open(temp_storage.object_path(key), 'wb') as fd:
for chunk in r.iter_content(None):
fd.write(chunk)
success = True
except Exception as e:
tries -= 1
if tries > 0:
log.warning("{} retry download".format(key))
else:
raise e
log.debug('{} creating resized images'.format(key))
create_resized_images(temp_storage.path(), key)
log.debug('{} uploading files to active storage'.format(key))
for key_to_create in to_create:
if temp_storage.exists(key_to_create):
log.debug('{} uploading {}'.format(key, key_to_create))
temp_storage.move(key_to_create, active_storage)
assert active_storage.exists(key_to_create)
self._set_migrated(key_to_create)
else:
log.warning(
'{} File does not exists, skipping upload of {}'.format(
key, key_to_create))
with self.lock:
self.processed += 1
def crop_and_publish_thumbs(filename, crop_options):
create_cropped_image(temp_storage.path(), filename, crop_options)
create_resized_images(temp_storage.path(), filename)
for key in resized_keys(filename):
temp_storage.move(key, active_storage)
def crop_and_publish_thumbs(filename, crop_options):
create_cropped_image(temp_storage.path(), filename, crop_options)
create_resized_images(temp_storage.path(), filename)
for key in resized_keys(filename):
temp_storage.move(key, active_storage)