def _is_migrated(self, key): if self._is_marked_migrated(key): log.debug('{} is marked as migrated'.format(key)) return True if active_storage.exists(key): log.debug('{} exists in active_storage'.format(key)) self._set_migrated(key) return True
def test_migrate(create_engine): migrator = Migrator() migrator.execute(jobs=1) assert migrator.total == 2 assert migrator.skipped == 0 assert migrator.processed == 2 assert migrator.errors == 0 for key in [file['filename'] for file in files]: if key == '1274781175_1003161687.jpg': assert active_storage.exists('1274781175_1003161687.svg') else: assert active_storage.exists(key) for resized in resized_keys(key): assert active_storage.exists(resized) migrator.execute() assert migrator.total == 2 assert migrator.skipped == 2 assert migrator.processed == 0 assert migrator.errors == 0
def publish(request): filename = request.POST['filename'] already_published = active_storage.exists(filename) if 'crop' in request.POST: crop_options = request.POST['crop'] if already_published: active_storage.copy(filename, temp_storage) else: incoming_storage.copy(filename, temp_storage) crop_and_publish_thumbs(filename, crop_options) temp_storage.delete(filename) else: for key in resized_keys(filename): if incoming_storage.exists(key): incoming_storage.move(key, active_storage) if not already_published: incoming_storage.move(filename, active_storage) return {'success': True}
def publish(request): if request.POST['secret'] != os.environ['API_SECRET_KEY']: raise HTTPForbidden('Bad secret key') filename = request.POST['filename'] already_published = active_storage.exists(filename) if 'crop' in request.POST: crop_options = request.POST['crop'] if already_published: active_storage.copy(filename, temp_storage) else: incoming_storage.copy(filename, temp_storage) crop_and_publish_thumbs(filename, crop_options) temp_storage.delete(filename) else: for key in resized_keys(filename): if incoming_storage.exists(key): incoming_storage.move(key, active_storage) if not already_published: incoming_storage.move(filename, active_storage) return {'success': True}
def do_process_key(self, key): base, ext = os.path.splitext(key) if ext == '.svg': for rasterized in ('{}.jpg'.format(base), '{}.png'.format(base)): if active_storage.exists(rasterized): log.info("{} delete file {}".format(key, rasterized)) active_storage.delete(rasterized) to_create = [key] + resized_keys(key) if not self.force: for key_to_create in list(to_create): if self._is_migrated(key_to_create): to_create.remove(key_to_create) if len(to_create) == 0: log.debug('{} skipping'.format(key)) with self.lock: self.skipped += 1 return log.debug('{} getting file in temp storage'.format(key)) tries = 3 success = False while tries > 0 and not success: try: with requests.get( 'http://s.camptocamp.org/uploads/images/{}'.format( key), stream=True, timeout=120) as r: if r.status_code != 200: log.error("{} return status code {} - {}".format( key, r.status_code, r.reason)) with self.lock: self.errors += 1 return with open(temp_storage.object_path(key), 'wb') as fd: for chunk in r.iter_content(None): fd.write(chunk) success = True except Exception as e: tries -= 1 if tries > 0: log.warning("{} retry download".format(key)) else: raise e log.debug('{} creating resized images'.format(key)) create_resized_images(temp_storage.path(), key) log.debug('{} uploading files to active storage'.format(key)) for key_to_create in to_create: if temp_storage.exists(key_to_create): log.debug('{} uploading {}'.format(key, key_to_create)) temp_storage.move(key_to_create, active_storage) assert active_storage.exists(key_to_create) self._set_migrated(key_to_create) else: log.warning( '{} File does not exists, skipping upload of {}'.format( key, key_to_create)) with self.lock: self.processed += 1