def _is_migrated(self, key):
if self._is_marked_migrated(key):
log.debug('{} is marked as migrated'.format(key))
return True
if active_storage.exists(key):
log.debug('{} exists in active_storage'.format(key))
self._set_migrated(key)
return True
def test_migrate(create_engine):
migrator = Migrator()
migrator.execute(jobs=1)
assert migrator.total == 2
assert migrator.skipped == 0
assert migrator.processed == 2
assert migrator.errors == 0
for key in [file['filename'] for file in files]:
if key == '1274781175_1003161687.jpg':
assert active_storage.exists('1274781175_1003161687.svg')
else:
assert active_storage.exists(key)
for resized in resized_keys(key):
assert active_storage.exists(resized)
migrator.execute()
assert migrator.total == 2
assert migrator.skipped == 2
assert migrator.processed == 0
assert migrator.errors == 0
def test_migrate(create_engine):
migrator = Migrator()
migrator.execute(jobs=1)
assert migrator.total == 2
assert migrator.skipped == 0
assert migrator.processed == 2
assert migrator.errors == 0
for key in [file['filename'] for file in files]:
if key == '1274781175_1003161687.jpg':
assert active_storage.exists('1274781175_1003161687.svg')
else:
assert active_storage.exists(key)
for resized in resized_keys(key):
assert active_storage.exists(resized)
migrator.execute()
assert migrator.total == 2
assert migrator.skipped == 2
assert migrator.processed == 0
assert migrator.errors == 0
def publish(request):
filename = request.POST['filename']
already_published = active_storage.exists(filename)
if 'crop' in request.POST:
crop_options = request.POST['crop']
if already_published:
active_storage.copy(filename, temp_storage)
else:
incoming_storage.copy(filename, temp_storage)
crop_and_publish_thumbs(filename, crop_options)
temp_storage.delete(filename)
else:
for key in resized_keys(filename):
if incoming_storage.exists(key):
incoming_storage.move(key, active_storage)
if not already_published:
incoming_storage.move(filename, active_storage)
return {'success': True}
def publish(request):
if request.POST['secret'] != os.environ['API_SECRET_KEY']:
raise HTTPForbidden('Bad secret key')
filename = request.POST['filename']
already_published = active_storage.exists(filename)
if 'crop' in request.POST:
crop_options = request.POST['crop']
if already_published:
active_storage.copy(filename, temp_storage)
else:
incoming_storage.copy(filename, temp_storage)
crop_and_publish_thumbs(filename, crop_options)
temp_storage.delete(filename)
else:
for key in resized_keys(filename):
if incoming_storage.exists(key):
incoming_storage.move(key, active_storage)
if not already_published:
incoming_storage.move(filename, active_storage)
return {'success': True}
def do_process_key(self, key):
base, ext = os.path.splitext(key)
if ext == '.svg':
for rasterized in ('{}.jpg'.format(base), '{}.png'.format(base)):
if active_storage.exists(rasterized):
log.info("{} delete file {}".format(key, rasterized))
active_storage.delete(rasterized)
to_create = [key] + resized_keys(key)
if not self.force:
for key_to_create in list(to_create):
if self._is_migrated(key_to_create):
to_create.remove(key_to_create)
if len(to_create) == 0:
log.debug('{} skipping'.format(key))
with self.lock:
self.skipped += 1
return
log.debug('{} getting file in temp storage'.format(key))
tries = 3
success = False
while tries > 0 and not success:
try:
with requests.get(
'http://s.camptocamp.org/uploads/images/{}'.format(
key),
stream=True,
timeout=120) as r:
if r.status_code != 200:
log.error("{} return status code {} - {}".format(
key, r.status_code, r.reason))
with self.lock:
self.errors += 1
return
with open(temp_storage.object_path(key), 'wb') as fd:
for chunk in r.iter_content(None):
fd.write(chunk)
success = True
except Exception as e:
tries -= 1
if tries > 0:
log.warning("{} retry download".format(key))
else:
raise e
log.debug('{} creating resized images'.format(key))
create_resized_images(temp_storage.path(), key)
log.debug('{} uploading files to active storage'.format(key))
for key_to_create in to_create:
if temp_storage.exists(key_to_create):
log.debug('{} uploading {}'.format(key, key_to_create))
temp_storage.move(key_to_create, active_storage)
assert active_storage.exists(key_to_create)
self._set_migrated(key_to_create)
else:
log.warning(
'{} File does not exists, skipping upload of {}'.format(
key, key_to_create))
with self.lock:
self.processed += 1