def test_load(self): """Test loading packages from a configuration dictionary.""" os.environ[env.VIZIERSERVER_PACKAGE_PATH] = PACKAGES_DIR config = AppConfig() packages = load_packages(config.engine.package_path) for key in [PACKAGE_PLOT, PACKAGE_PYTHON, PACKAGE_VIZUAL]: self.assertTrue(key in packages)
def setUp(self): """Create engine for an empty repository.""" # Drop directory if it exists if os.path.isdir(SERVER_DIR): shutil.rmtree(SERVER_DIR) os.makedirs(SERVER_DIR) os.environ[app.VIZIERSERVER_ENGINE] = MIMIR_ENGINE os.environ[app.VIZIERENGINE_DATA_DIR] = SERVER_DIR os.environ[app.VIZIERSERVER_PACKAGE_PATH] = PACKAGES_DIR os.environ[app.VIZIERSERVER_PROCESSOR_PATH] = PROCESSORS_DIR self.engine = get_engine(AppConfig())
def setUp(self): """Create an instance of the default vizier processor for an empty server directory. """ # Drop directory if it exists if os.path.isdir(SERVER_DIR): shutil.rmtree(SERVER_DIR) os.makedirs(SERVER_DIR) os.environ[app.VIZIERENGINE_DATA_DIR] = SERVER_DIR os.environ[app.VIZIERSERVER_PACKAGE_PATH] = PACKAGES_DIR os.environ[app.VIZIERSERVER_PROCESSOR_PATH] = PROCESSORS_DIR self.engine = get_engine(AppConfig())
def setUp(self): """Create an instance of the default backend with an empty directory """ if os.path.isdir(SERVER_DIR): shutil.rmtree(SERVER_DIR) os.makedirs(SERVER_DIR) os.environ[app.VIZIERENGINE_DATA_DIR] = SERVER_DIR os.environ[app.VIZIERSERVER_PACKAGE_PATH] = PACKAGES_DIR os.environ[app.VIZIERSERVER_PROCESSOR_PATH] = PROCESSORS_DIR os.environ[app.VIZIERENGINE_SYNCHRONOUS] = ':'.join([ vizual.PACKAGE_VIZUAL + '.' + vizual.VIZUAL_LOAD, vizual.PACKAGE_VIZUAL + '.' + vizual.VIZUAL_UPD_CELL ]) self.engine = get_engine(AppConfig())
def test_create_cache(self): """Test accessing and deleting projects for an empty repository.""" viztrails = OSViztrailRepository(base_path=VIZTRAILS_DIR) vt1 = viztrails.create_viztrail( properties={PROPERTY_NAME: 'My Project'}) vt2 = viztrails.create_viztrail( properties={PROPERTY_NAME: 'A Project'}) filename = os.path.join(SERVER_DIR, 'container.json') DefaultObjectStore().write_object(object_path=filename, content=[{ 'projectId': vt1.identifier, 'url': 'API1', 'port': 80, 'containerId': 'ID1' }, { 'projectId': vt2.identifier, 'url': 'API2', 'port': 81, 'containerId': 'ID2' }]) # Initialize the project cache viztrails = OSViztrailRepository(base_path=VIZTRAILS_DIR) filestores_dir = os.path.join(SERVER_DIR, DEFAULT_FILESTORES_DIR) datastores_dir = os.path.join(SERVER_DIR, DEFAULT_DATASTORES_DIR) projects = ContainerProjectCache( viztrails=viztrails, container_file=filename, config=AppConfig(), datastores=MimirDatastoreFactory(datastores_dir), filestores=FileSystemFilestoreFactory(filestores_dir)) self.assertEqual(len(projects.list_projects()), 2) self.assertEqual( projects.get_project(vt1.identifier).container_api, 'API1') self.assertEqual( projects.get_project(vt2.identifier).container_api, 'API2') self.assertEqual( projects.get_project(vt1.identifier).container_id, 'ID1') self.assertEqual( projects.get_project(vt2.identifier).container_id, 'ID2') self.assertEqual(projects.get_project(vt1.identifier).port, 80) self.assertEqual(projects.get_project(vt2.identifier).port, 81)
def test_default_config(self): """Test the default configuration settings.""" config = AppConfig() self.assertEqual(config.webservice.name, env.DEFAULT_SETTINGS[env.VIZIERSERVER_NAME]) self.assertEqual(config.webservice.server_url, env.DEFAULT_SETTINGS[env.VIZIERSERVER_BASE_URL]) self.assertEqual(config.webservice.server_port, env.DEFAULT_SETTINGS[env.VIZIERSERVER_SERVER_PORT]) self.assertEqual(config.webservice.server_local_port, env.DEFAULT_SETTINGS[env.VIZIERSERVER_SERVER_LOCAL_PORT]) self.assertEqual(config.webservice.app_path, env.DEFAULT_SETTINGS[env.VIZIERSERVER_APP_PATH]) self.assertEqual(config.webservice.defaults.row_limit, env.DEFAULT_SETTINGS[env.VIZIERSERVER_ROW_LIMIT]) self.assertEqual(config.webservice.defaults.max_row_limit, env.DEFAULT_SETTINGS[env.VIZIERSERVER_MAX_ROW_LIMIT]) self.assertEqual(config.webservice.defaults.max_file_size, env.DEFAULT_SETTINGS[env.VIZIERSERVER_MAX_UPLOAD_SIZE]) self.assertEqual(config.run.debug, env.DEFAULT_SETTINGS[env.VIZIERSERVER_DEBUG]) self.assertEqual(config.logs.server, env.DEFAULT_SETTINGS[env.VIZIERSERVER_LOG_DIR]) self.assertEqual(config.engine.identifier, env.DEFAULT_SETTINGS[env.VIZIERSERVER_ENGINE]) self.assertEqual(config.engine.data_dir, env.DEFAULT_SETTINGS[env.VIZIERENGINE_DATA_DIR]) self.assertEqual(config.engine.package_path, env.DEFAULT_SETTINGS[env.VIZIERSERVER_PACKAGE_PATH]) self.assertEqual(config.engine.processor_path, env.DEFAULT_SETTINGS[env.VIZIERSERVER_PROCESSOR_PATH]) self.assertEqual(config.engine.use_short_ids, env.DEFAULT_SETTINGS[env.VIZIERENGINE_USE_SHORT_IDENTIFIER]) self.assertEqual(config.engine.sync_commands, env.DEFAULT_SETTINGS[env.VIZIERENGINE_SYNCHRONOUS]) self.assertEqual(config.engine.backend.identifier, env.DEFAULT_SETTINGS[env.VIZIERENGINE_BACKEND]) self.assertEqual(config.engine.backend.celery.routes, env.DEFAULT_SETTINGS[env.VIZIERENGINE_CELERY_ROUTES]) self.assertEqual(config.engine.backend.container.ports, env.DEFAULT_SETTINGS[env.VIZIERENGINE_CONTAINER_PORTS]) self.assertEqual(config.engine.backend.container.image, env.DEFAULT_SETTINGS[env.VIZIERENGINE_CONTAINER_IMAGE])
def test_env_config(self): """Test app config using environment variables.""" os.environ[env.VIZIERSERVER_NAME] = 'Some Name' os.environ[env.VIZIERSERVER_LOG_DIR] = 'logdir' os.environ[env.VIZIERSERVER_DEBUG] = 'bla' os.environ[env.VIZIERSERVER_BASE_URL] = 'http://webapi' os.environ[env.VIZIERSERVER_SERVER_PORT] = '80' os.environ[env.VIZIERSERVER_SERVER_LOCAL_PORT] = '90' os.environ[env.VIZIERSERVER_APP_PATH] = 'vizier/v2' os.environ[env.VIZIERSERVER_ROW_LIMIT] = '111' os.environ[env.VIZIERSERVER_MAX_ROW_LIMIT] = '222' os.environ[env.VIZIERSERVER_MAX_UPLOAD_SIZE] = '333' os.environ[env.VIZIERSERVER_ENGINE] = 'CELERY' os.environ[env.VIZIERENGINE_USE_SHORT_IDENTIFIER] = str(not env.DEFAULT_SETTINGS[env.VIZIERENGINE_USE_SHORT_IDENTIFIER]) os.environ[env.VIZIERENGINE_SYNCHRONOUS] = 'ABC' os.environ[env.VIZIERENGINE_BACKEND] = 'THE_BACKEND' os.environ[env.VIZIERENGINE_CELERY_ROUTES] = 'Some Routes' os.environ[env.VIZIERENGINE_CONTAINER_PORTS] = '8080-8084,9000,10001-10010' config = AppConfig() self.assertEqual(config.webservice.name, 'Some Name') self.assertEqual(config.webservice.server_url, 'http://webapi') self.assertEqual(config.webservice.server_port, 80) self.assertEqual(config.webservice.server_local_port, 90) self.assertEqual(config.webservice.app_path, 'vizier/v2') self.assertEqual(config.webservice.defaults.row_limit, 111) self.assertEqual(config.webservice.defaults.max_row_limit, 222) self.assertEqual(config.webservice.defaults.max_file_size, 333) self.assertEqual(config.run.debug, False) self.assertEqual(config.logs.server, 'logdir') self.assertEqual(config.engine.identifier, 'CELERY') self.assertEqual(config.engine.use_short_ids, not env.DEFAULT_SETTINGS[env.VIZIERENGINE_USE_SHORT_IDENTIFIER]) self.assertEqual(config.engine.sync_commands, 'ABC') self.assertEqual(config.engine.backend.identifier, 'THE_BACKEND') self.assertEqual(config.engine.backend.celery.routes, 'Some Routes') ports = list(range(8080, 8084)) + [9000] + list(range(10001,10010)) self.assertEqual(config.engine.backend.container.ports, ports) self.assertEqual(config.engine.backend.container.image, env.DEFAULT_SETTINGS[env.VIZIERENGINE_CONTAINER_IMAGE])
from vizier.api.webservice.base import VizierApi from vizier.config.app import AppConfig import vizier.api.base as srv import vizier.api.serialize.deserialize as deserialize import vizier.api.serialize.labels as labels import vizier.config.app as app import json # ----------------------------------------------------------------------------- # # App Blueprint # # ----------------------------------------------------------------------------- """Get application configuration parameters from environment variables.""" config = AppConfig() webui_file_dir = os.getenv( 'WEB_UI_STATIC_FILES', "./web-ui/build/" ) #pkg_resources.resource_filename(__name__, os.getenv('WEB_UI_STATIC_FILES', "./web-ui/build/")) print(webui_file_dir) global api api = VizierApi(config, init=True) # Create the application blueprint bp = Blueprint('app', __name__, url_prefix=config.webservice.app_path, static_folder=webui_file_dir)
def test_create_synchronous_workflow(self): """Create workflow by appending a sequence of modules that are executed synchronously. """ project = self.engine.projects.create_project() # MODULE 1 # -------- # LOAD people fh = project.filestore.upload_file(PEOPLE_FILE) module = self.engine.append_workflow_module( project_id=project.identifier, branch_id=project.get_default_branch().identifier, command=load_dataset(dataset_name='people', file={pckg.FILE_ID: fh.identifier}, validate=True)) self.assertTrue(module.is_success) self.assertTrue('people' in module.provenance.write) self.assertEqual(len(module.provenance.write['people'].columns), 2) # MODULE 2 # -------- # UPDATE CELL module = self.engine.append_workflow_module( project_id=project.identifier, branch_id=project.get_default_branch().identifier, command=update_cell(dataset_name='people', column=1, row=0, value='42', validate=True)) print("STATUS: {}".format(module)) self.assertTrue(module.is_success) self.assertTrue('people' in module.provenance.write) # MODULE 3 # -------- # LOAD employee fh = project.filestore.upload_file(EMPLOYEE_FILE) module = self.engine.append_workflow_module( project_id=project.identifier, branch_id=project.get_default_branch().identifier, command=load_dataset(dataset_name='employee', file={pckg.FILE_ID: fh.identifier}, validate=True)) self.assertTrue(module.is_success) self.assertFalse('people' in module.provenance.write) self.assertTrue('employee' in module.provenance.write) # # Reload engine and check the module states # self.engine = get_engine(AppConfig()) project = self.engine.projects.get_project(project.identifier) modules = project.get_default_branch().get_head().modules self.assertEqual(len(modules), 3) for m in modules: self.assertTrue(m.is_success) self.assertIsNotNone(m.timestamp.created_at) self.assertIsNotNone(m.timestamp.started_at) self.assertIsNotNone(m.timestamp.finished_at) self.assertIsNotNone(m.provenance.write) self.assertTrue('people' in modules[0].provenance.write) self.assertTrue('employee' in modules[-1].provenance.write) self.assertNotEqual(modules[0].provenance.write['people'].identifier, modules[1].provenance.write['people'].identifier)
def test_create_synchronous_workflow_with_errors(self): """Create workflow by appending a sequence of modules that are executed synchronously. """ project = self.engine.projects.create_project() # MODULE 1 # -------- # LOAD people fh = project.filestore.upload_file(PEOPLE_FILE) module = self.engine.append_workflow_module( project_id=project.identifier, branch_id=project.get_default_branch().identifier, command=load_dataset(dataset_name='people', file={ pckg.FILE_ID: fh.identifier, pckg.FILE_NAME: os.path.basename(PEOPLE_FILE) }, validate=True)) project = self.engine.projects.get_project(project.identifier) modules = project.get_default_branch().get_head().modules for m in modules: print(m) self.assertTrue(m.is_success) # MODULE 2 # -------- # UPDATE CELL module = self.engine.append_workflow_module( project_id=project.identifier, branch_id=project.get_default_branch().identifier, command=update_cell(dataset_name='employee', column=1, row=0, value='42', validate=True)) self.assertTrue(module.is_error) # MODULE 2 # -------- # INSERT employee fh = project.filestore.upload_file(EMPLOYEE_FILE) result = self.engine.insert_workflow_module( project_id=project.identifier, branch_id=project.get_default_branch().identifier, before_module_id=module.identifier, command=load_dataset(dataset_name='employee', file={ pckg.FILE_ID: fh.identifier, pckg.FILE_NAME: os.path.basename(EMPLOYEE_FILE) }, validate=True)) self.assertEqual(len(result), 2) # Wait for the operations to finish while project.viztrail.default_branch.head.is_active: time.sleep(0.1) # # Reload engine and check the module states # self.engine = get_engine(AppConfig()) project = self.engine.projects.get_project(project.identifier) modules = project.get_default_branch().get_head().modules for m in modules: print(m) self.assertTrue(m.is_success) # MODULE 1 # -------- # UPDATE CELL module = self.engine.insert_workflow_module( project_id=project.identifier, branch_id=project.get_default_branch().identifier, before_module_id=modules[0].identifier, command=update_cell(dataset_name='friends', column=1, row=0, value='43', validate=True)) # Wait for the operations to finish while project.viztrail.default_branch.head.is_active: time.sleep(0.1) modules = project.get_default_branch().get_head().modules self.assertEqual(len(modules), 4) self.assertTrue(modules[0].is_error) for m in modules[1:]: self.assertTrue(m.is_canceled) # MODULE 1 # -------- # INSERT friends fh = project.filestore.upload_file(EMPLOYEE_FILE) result = self.engine.insert_workflow_module( project_id=project.identifier, branch_id=project.get_default_branch().identifier, before_module_id=modules[0].identifier, command=load_dataset(dataset_name='friends', file={ pckg.FILE_ID: fh.identifier, pckg.FILE_NAME: os.path.basename(EMPLOYEE_FILE) }, validate=True)) self.assertEqual(len(result), 5) # Wait for the operations to finish while project.viztrail.default_branch.head.is_active: time.sleep(0.1) modules = project.get_default_branch().get_head().modules self.assertEqual(len(modules), 5) for m in modules: self.assertTrue(m.is_success) self.assertEqual(len(modules[0].provenance.write['friends'].columns), 3) # REPLACE MODULE 1 # ---------------- # Load people dataset instead employee fh = project.filestore.upload_file(PEOPLE_FILE) result = self.engine.replace_workflow_module( project_id=project.identifier, branch_id=project.get_default_branch().identifier, module_id=modules[0].identifier, command=load_dataset(dataset_name='friends', file={ pckg.FILE_ID: fh.identifier, pckg.FILE_NAME: os.path.basename(PEOPLE_FILE) }, validate=True)) self.assertEqual(len(result), 5) # Wait for the operations to finish while project.viztrail.default_branch.head.is_active: time.sleep(0.1) modules = project.get_default_branch().get_head().modules self.assertEqual(len(modules), 5) for m in modules: self.assertTrue(m.is_success) self.assertEqual(len(modules[0].provenance.write['friends'].columns), 2) ds = project.datastore.get_dataset( modules[0].provenance.write['friends'].identifier) self.assertEqual(ds.fetch_rows()[0].values[1], 23) # # Reload engine and check the module states # self.engine = get_engine(AppConfig()) project = self.engine.projects.get_project(project.identifier) modules = project.get_default_branch().get_head().modules self.assertEqual(len(modules), 5) for m in modules: self.assertTrue(m.is_success) self.assertEqual(len(modules[0].provenance.write['friends'].columns), 2) ds = project.datastore.get_dataset( modules[0].provenance.write['friends'].identifier) self.assertEqual(ds.fetch_rows()[0].values[1], 23)
def create_app(): """Factory pattern for Flask. Initialize the Flask application object. Returns ------- Flask """ #Get application configuration parameters from environment variables. config = AppConfig() # Create the app and enable cross-origin resource sharing app = Flask(__name__) #app.config['APPLICATION_ROOT'] = config.webservice.app_path #app.config['DEBUG'] = True # Set size limit for uploaded files app.config['MAX_CONTENT_LENGTH'] = config.webservice.defaults.max_file_size # Enable CORS CORS(app) # Switch logging on log_dir = os.path.abspath(config.logs.server) # Create the directory if it does not exist if not os.path.isdir(log_dir): os.makedirs(log_dir) # File handle for server logs file_handler = RotatingFileHandler(os.path.join(log_dir, 'vizier-webapi.log'), maxBytes=1024 * 1024 * 100, backupCount=20) file_handler.setLevel(logging.ERROR) file_handler.setFormatter( logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")) app.logger.addHandler(file_handler) # -------------------------------------------------------------------------- # # Error Handler # # -------------------------------------------------------------------------- @app.errorhandler(srv.ServerRequestException) def invalid_request_or_resource_not_found(error): """JSON response handler for invalid requests or requests that access unknown resources. Parameters ---------- error : Exception Exception thrown by request Handler Returns ------- Http response """ app.logger.error(error.message) response = jsonify(error.to_dict()) response.status_code = error.status_code return response @app.errorhandler(413) def upload_error(exception): """Exception handler for file uploads that exceed the file size limit.""" app.logger.error(exception) return make_response(jsonify({'error': str(exception)}), 413) @app.errorhandler(500) def internal_error(exception): """Exception handler that logs exceptions.""" app.logger.error(exception) return make_response(jsonify({'error': str(exception)}), 500) # Register the API blueprint from . import server app.register_blueprint(server.bp) # Return the applicatio object # -------------------------------------------------------------------------- # # Initialize # # -------------------------------------------------------------------------- @app.before_first_request def initialize(): """Initialize Mimir gateway (if necessary) before the first request. """ # Initialize the Mimir gateway if using Mimir engine if config.engine.identifier == const.MIMIR_ENGINE: import vizier.mimir as mimir root_redirect_path = "{}/web-ui/vizier-db".format(server.bp.url_prefix) @app.route("/") def handle_root(): """Redirect users to the web UI """ return redirect(root_redirect_path) return app