def imagelist(): # Check parameters try: from_path = request.args.get('path', '') want_info = parse_boolean(request.args.get('attributes', '')) limit = parse_int(request.args.get('limit', '1000')) validate_string(from_path, 1, 1024) except ValueError as e: raise ParameterError(e) # Get extra parameters for image URL construction image_params = request.args.to_dict() if 'path' in image_params: del image_params['path'] if 'attributes' in image_params: del image_params['attributes'] if 'limit' in image_params: del image_params['limit'] # Get directory listing directory_info = get_directory_listing(from_path, False, limit) if not directory_info.exists(): raise DoesNotExistError('Invalid path') ret_list = [] db_session = data_engine.db_get_session() db_commit = False try: # Auto-populate the folders database db_folder = auto_sync_folder( from_path, data_engine, task_engine, _db_session=db_session ) db_session.commit() # Require view permission or file admin permissions_engine.ensure_folder_permitted( db_folder, FolderPermission.ACCESS_VIEW, get_session_user() ) # Create the response file_list = directory_info.contents() img_types = image_engine.get_image_formats() base_folder = add_sep(directory_info.name()) for f in file_list: # Filter out non-images if get_file_extension(f['filename']) in img_types: entry_path = base_folder + f['filename'] entry = { 'filename': f['filename'], 'url': external_url_for('image', src=entry_path, **image_params) } if want_info: db_entry = auto_sync_existing_file( entry_path, data_engine, task_engine, burst_pdf=False, # Don't burst a PDF just by finding it here _db_session=db_session ) entry['id'] = db_entry.id if db_entry else 0 entry['folder_id'] = db_entry.folder_id if db_entry else 0 entry['title'] = db_entry.title if db_entry else '' entry['description'] = db_entry.description if db_entry else '' entry['width'] = db_entry.width if db_entry else 0 entry['height'] = db_entry.height if db_entry else 0 ret_list.append(entry) db_commit = True finally: try: if db_commit: db_session.commit() else: db_session.rollback() finally: db_session.close() return make_api_success_response(ret_list)
def imagelist(): # Check parameters try: from_path = request.args.get('path', '') want_info = parse_boolean(request.args.get('attributes', '')) start = parse_int(request.args.get('start', '0')) limit = parse_int(request.args.get('limit', '1000')) validate_string(from_path, 1, 1024) validate_number(start, 0, 999999999) validate_number(limit, 1, 1000) except ValueError as e: raise ParameterError(e) # Get extra parameters for image URL construction, remove API parameters image_params = request.args.to_dict() image_params.pop('path', None) image_params.pop('attributes', None) image_params.pop('start', None) image_params.pop('limit', None) # Get directory listing directory_info = get_directory_listing(from_path, False, 2, start, limit) if not directory_info.exists(): raise DoesNotExistError('Invalid path') ret_list = [] db_session = data_engine.db_get_session() db_commit = False try: # Auto-populate the folders database db_folder = auto_sync_folder( from_path, data_engine, task_engine, _db_session=db_session ) db_session.commit() # Require view permission or file admin permissions_engine.ensure_folder_permitted( db_folder, FolderPermission.ACCESS_VIEW, get_session_user() ) # Get download permission in case we need to return it later can_download = permissions_engine.is_folder_permitted( db_folder, FolderPermission.ACCESS_DOWNLOAD, get_session_user() ) # Create the response file_list = directory_info.contents() supported_img_types = image_engine.get_image_formats(supported_only=True) base_folder = add_sep(directory_info.name()) for f in file_list: # v2.6.4 Return unsupported files too. If you want to reverse this change, # the filtering needs to be elsewhere for 'start' and 'limit' to work properly supported_file = get_file_extension(f['filename']) in supported_img_types file_path = base_folder + f['filename'] if want_info: # Need to return the database fields too if supported_file: db_entry = auto_sync_existing_file( file_path, data_engine, task_engine, burst_pdf=False, # Don't burst a PDF just by finding it here _db_session=db_session ) db_entry = _prep_image_object(db_entry, can_download, **image_params) else: db_entry = _prep_blank_image_object() db_entry.filename = f['filename'] db_entry.supported = False # Return images in full (standard) image dict format entry = object_to_dict(db_entry, _omit_fields) else: # Return images in short dict format entry = { 'filename': f['filename'], 'supported': supported_file, 'url': (external_url_for('image', src=file_path, **image_params) if supported_file else '') } ret_list.append(entry) db_commit = True finally: try: if db_commit: db_session.commit() else: db_session.rollback() finally: db_session.close() return make_api_success_response(ret_list)
def precache_images(start_dir, file_specs, templates): """ Performs the main pre-caching function as described by the file header. """ from imageserver.flask_app import app from imageserver.filesystem_sync import auto_sync_existing_file from imageserver.image_attrs import ImageAttrs from imageserver.util import add_sep # Disable logging to prevent app startup and image errors going to main log # app.log.set_enabled(False) # Validate params rc = validate_params(start_dir, file_specs, templates) if (rc != RETURN_OK): return rc # Get base path with trailing / images_base_dir = add_sep(os.path.abspath(app.config['IMAGES_BASE_DIR'])) # Init stats and stop conditions stats = PreCacheStats() last_cache_pct = 0 cache_full = False keyboard_interrupt = False # Get directory walking errors def walk_err(os_error): stats.inc_dir_skipped_count() # Walk directory tree try: for cur_dir, sub_dirs, files in os.walk( start_dir, onerror=walk_err, followlinks=True ): log(cur_dir) stats.inc_total_dir_count() # Remove files and directories beginning with '.' for d in sub_dirs: if d.startswith('.'): sub_dirs.remove(d) for f in files: if f.startswith('.'): files.remove(f) # Get relative path from IMAGES_BASE_DIR/ if not cur_dir.startswith(images_base_dir): log('ERROR: Cannot calculate relative image path from ' + str(cur_dir)) stats.inc_dir_skipped_count() else: relative_dir = cur_dir[len(images_base_dir):] # Apply file specs for file_spec in file_specs: file_matches = fnmatch.filter(files, file_spec) for file_name in file_matches: # Check whether the cache is full (or now self-emptying) once in a while if stats.total_file_count % 10 == 0: cache_pct = app.cache_engine.size_percent() log('\tCache level %d%%' % cache_pct) if (cache_pct < last_cache_pct) or (cache_pct >= 80): raise CacheFullException() else: last_cache_pct = cache_pct # Process matched file stats.inc_total_file_count() for template in templates: try: log('\tProcessing %s with template %s' % (file_name, template)) image_path = os.path.join(relative_dir, file_name) db_image = auto_sync_existing_file( image_path, app.data_engine, app.task_engine ) image_attrs = ImageAttrs( db_image.src, db_image.id, template=template ) app.image_engine.finalise_image_attrs(image_attrs) gen_image = app.image_engine.get_image(image_attrs) if gen_image.is_from_cache(): log('\tImage already in cache') stats.inc_images_already_cached_count() else: log('\tGenerated %s image, %d bytes' % ( gen_image.attrs().format(), len(gen_image.data()) )) stats.inc_images_created_count() except KeyboardInterrupt as kbe: raise kbe except Exception as e: log('ERROR: ' + str(e)) stats.inc_images_error_count() except CacheFullException: cache_full = True except KeyboardInterrupt: keyboard_interrupt = True # Show stop reason and stats if keyboard_interrupt: log('---\nInterrupted by user.\n---') elif cache_full: log('---\nCache is full.\n---') else: log('---\nNo more files.\n---') log('%d matching file(s) found in %d directories.' % ( stats.total_file_count, stats.total_dir_count )) log('%d image(s) were generated and cached.' % stats.images_created_count) log('%d image(s) were already in cache.' % stats.images_already_cached_count) log('%d image(s) skipped due to error.' % stats.images_error_count) log('%d directories skipped due to error.' % stats.dir_skipped_count) log('Cache is now %d%% full.' % app.cache_engine.size_percent()) return RETURN_OK
def precache_images(start_dir, file_specs, templates): """ Performs the main pre-caching function as described by the file header. """ from imageserver.flask_app import app from imageserver.filesystem_sync import auto_sync_existing_file from imageserver.image_attrs import ImageAttrs from imageserver.util import add_sep # Disable logging to prevent app startup and image errors going to main log # app.log.set_enabled(False) # Validate params rc = validate_params(start_dir, file_specs, templates) if (rc != RETURN_OK): return rc # Get base path with trailing / images_base_dir = add_sep(os.path.abspath(app.config['IMAGES_BASE_DIR'])) # Init stats and stop conditions stats = PreCacheStats() last_cache_pct = 0 cache_full = False keyboard_interrupt = False # Get directory walking errors def walk_err(os_error): stats.inc_dir_skipped_count() # Walk directory tree try: for cur_dir, sub_dirs, files in os.walk( unicode(start_dir), onerror=walk_err, followlinks=True ): log(cur_dir) stats.inc_total_dir_count() # Remove files and directories beginning with '.' for d in sub_dirs: if d.startswith('.'): sub_dirs.remove(d) for f in files: if f.startswith('.'): files.remove(f) # Get relative path from IMAGES_BASE_DIR/ if not cur_dir.startswith(images_base_dir): log('ERROR: Cannot calculate relative image path from ' + str(cur_dir)) stats.inc_dir_skipped_count() else: relative_dir = cur_dir[len(images_base_dir):] # Apply file specs for file_spec in file_specs: file_matches = fnmatch.filter(files, file_spec) for file_name in file_matches: # Check whether the cache is full (or now self-emptying) once in a while if stats.total_file_count % 10 == 0: cache_pct = app.cache_engine.size_percent() log('\tCache level %d%%' % cache_pct) if (cache_pct < last_cache_pct) or (cache_pct >= 80): raise CacheFullException() else: last_cache_pct = cache_pct # Process matched file stats.inc_total_file_count() for template in templates: try: log('\tProcessing %s with template %s' % (file_name, template)) image_path = os.path.join(relative_dir, file_name) db_image = auto_sync_existing_file( image_path, app.data_engine, app.task_engine ) image_attrs = ImageAttrs( db_image.src, db_image.id, template=template ) app.image_engine.finalise_image_attrs(image_attrs) gen_image = app.image_engine.get_image(image_attrs) if gen_image.is_from_cache(): log('\tImage already in cache') stats.inc_images_already_cached_count() else: log('\tGenerated %s image, %d bytes' % ( gen_image.attrs().format(), len(gen_image.data()) )) stats.inc_images_created_count() except KeyboardInterrupt as kbe: raise kbe except Exception as e: log('ERROR: ' + str(e)) stats.inc_images_error_count() except CacheFullException: cache_full = True except KeyboardInterrupt: keyboard_interrupt = True # Show stop reason and stats if keyboard_interrupt: log('---\nInterrupted by user.\n---') elif cache_full: log('---\nCache is full.\n---') else: log('---\nNo more files.\n---') log('%d matching file(s) found in %d directories.' % ( stats.total_file_count, stats.total_dir_count )) log('%d image(s) were generated and cached.' % stats.images_created_count) log('%d image(s) were already in cache.' % stats.images_already_cached_count) log('%d image(s) skipped due to error.' % stats.images_error_count) log('%d directories skipped due to error.' % stats.dir_skipped_count) log('Cache is now %d%% full.' % app.cache_engine.size_percent()) return RETURN_OK