def test_load(self):
     """Test loading packages from a configuration dictionary."""
     os.environ[env.VIZIERSERVER_PACKAGE_PATH] = PACKAGES_DIR
     config = AppConfig()
     packages = load_packages(config.engine.package_path)
     for key in [PACKAGE_PLOT, PACKAGE_PYTHON, PACKAGE_VIZUAL]:
         self.assertTrue(key in packages)
 def setUp(self):
     """Create engine for an empty repository."""
     # Drop directory if it exists
     if os.path.isdir(SERVER_DIR):
         shutil.rmtree(SERVER_DIR)
     os.makedirs(SERVER_DIR)
     os.environ[app.VIZIERSERVER_ENGINE] = MIMIR_ENGINE
     os.environ[app.VIZIERENGINE_DATA_DIR] = SERVER_DIR
     os.environ[app.VIZIERSERVER_PACKAGE_PATH] = PACKAGES_DIR
     os.environ[app.VIZIERSERVER_PROCESSOR_PATH] = PROCESSORS_DIR
     self.engine = get_engine(AppConfig())
 def setUp(self):
     """Create an instance of the default vizier processor for an empty server
     directory.
     """
     # Drop directory if it exists
     if os.path.isdir(SERVER_DIR):
         shutil.rmtree(SERVER_DIR)
     os.makedirs(SERVER_DIR)
     os.environ[app.VIZIERENGINE_DATA_DIR] = SERVER_DIR
     os.environ[app.VIZIERSERVER_PACKAGE_PATH] = PACKAGES_DIR
     os.environ[app.VIZIERSERVER_PROCESSOR_PATH] = PROCESSORS_DIR
     self.engine = get_engine(AppConfig())
Example #4
0
 def setUp(self):
     """Create an instance of the default backend with an empty directory
     """
     if os.path.isdir(SERVER_DIR):
         shutil.rmtree(SERVER_DIR)
     os.makedirs(SERVER_DIR)
     os.environ[app.VIZIERENGINE_DATA_DIR] = SERVER_DIR
     os.environ[app.VIZIERSERVER_PACKAGE_PATH] = PACKAGES_DIR
     os.environ[app.VIZIERSERVER_PROCESSOR_PATH] = PROCESSORS_DIR
     os.environ[app.VIZIERENGINE_SYNCHRONOUS] = ':'.join([
         vizual.PACKAGE_VIZUAL + '.' + vizual.VIZUAL_LOAD,
         vizual.PACKAGE_VIZUAL + '.' + vizual.VIZUAL_UPD_CELL
     ])
     self.engine = get_engine(AppConfig())
 def test_create_cache(self):
     """Test accessing and deleting projects for an empty repository."""
     viztrails = OSViztrailRepository(base_path=VIZTRAILS_DIR)
     vt1 = viztrails.create_viztrail(
         properties={PROPERTY_NAME: 'My Project'})
     vt2 = viztrails.create_viztrail(
         properties={PROPERTY_NAME: 'A Project'})
     filename = os.path.join(SERVER_DIR, 'container.json')
     DefaultObjectStore().write_object(object_path=filename,
                                       content=[{
                                           'projectId': vt1.identifier,
                                           'url': 'API1',
                                           'port': 80,
                                           'containerId': 'ID1'
                                       }, {
                                           'projectId': vt2.identifier,
                                           'url': 'API2',
                                           'port': 81,
                                           'containerId': 'ID2'
                                       }])
     # Initialize the project cache
     viztrails = OSViztrailRepository(base_path=VIZTRAILS_DIR)
     filestores_dir = os.path.join(SERVER_DIR, DEFAULT_FILESTORES_DIR)
     datastores_dir = os.path.join(SERVER_DIR, DEFAULT_DATASTORES_DIR)
     projects = ContainerProjectCache(
         viztrails=viztrails,
         container_file=filename,
         config=AppConfig(),
         datastores=MimirDatastoreFactory(datastores_dir),
         filestores=FileSystemFilestoreFactory(filestores_dir))
     self.assertEqual(len(projects.list_projects()), 2)
     self.assertEqual(
         projects.get_project(vt1.identifier).container_api, 'API1')
     self.assertEqual(
         projects.get_project(vt2.identifier).container_api, 'API2')
     self.assertEqual(
         projects.get_project(vt1.identifier).container_id, 'ID1')
     self.assertEqual(
         projects.get_project(vt2.identifier).container_id, 'ID2')
     self.assertEqual(projects.get_project(vt1.identifier).port, 80)
     self.assertEqual(projects.get_project(vt2.identifier).port, 81)
 def test_default_config(self):
     """Test the default configuration settings."""
     config = AppConfig()
     self.assertEqual(config.webservice.name, env.DEFAULT_SETTINGS[env.VIZIERSERVER_NAME])
     self.assertEqual(config.webservice.server_url, env.DEFAULT_SETTINGS[env.VIZIERSERVER_BASE_URL])
     self.assertEqual(config.webservice.server_port, env.DEFAULT_SETTINGS[env.VIZIERSERVER_SERVER_PORT])
     self.assertEqual(config.webservice.server_local_port, env.DEFAULT_SETTINGS[env.VIZIERSERVER_SERVER_LOCAL_PORT])
     self.assertEqual(config.webservice.app_path, env.DEFAULT_SETTINGS[env.VIZIERSERVER_APP_PATH])
     self.assertEqual(config.webservice.defaults.row_limit, env.DEFAULT_SETTINGS[env.VIZIERSERVER_ROW_LIMIT])
     self.assertEqual(config.webservice.defaults.max_row_limit, env.DEFAULT_SETTINGS[env.VIZIERSERVER_MAX_ROW_LIMIT])
     self.assertEqual(config.webservice.defaults.max_file_size, env.DEFAULT_SETTINGS[env.VIZIERSERVER_MAX_UPLOAD_SIZE])
     self.assertEqual(config.run.debug, env.DEFAULT_SETTINGS[env.VIZIERSERVER_DEBUG])
     self.assertEqual(config.logs.server, env.DEFAULT_SETTINGS[env.VIZIERSERVER_LOG_DIR])
     self.assertEqual(config.engine.identifier, env.DEFAULT_SETTINGS[env.VIZIERSERVER_ENGINE])
     self.assertEqual(config.engine.data_dir, env.DEFAULT_SETTINGS[env.VIZIERENGINE_DATA_DIR])
     self.assertEqual(config.engine.package_path, env.DEFAULT_SETTINGS[env.VIZIERSERVER_PACKAGE_PATH])
     self.assertEqual(config.engine.processor_path, env.DEFAULT_SETTINGS[env.VIZIERSERVER_PROCESSOR_PATH])
     self.assertEqual(config.engine.use_short_ids, env.DEFAULT_SETTINGS[env.VIZIERENGINE_USE_SHORT_IDENTIFIER])
     self.assertEqual(config.engine.sync_commands, env.DEFAULT_SETTINGS[env.VIZIERENGINE_SYNCHRONOUS])
     self.assertEqual(config.engine.backend.identifier, env.DEFAULT_SETTINGS[env.VIZIERENGINE_BACKEND])
     self.assertEqual(config.engine.backend.celery.routes, env.DEFAULT_SETTINGS[env.VIZIERENGINE_CELERY_ROUTES])
     self.assertEqual(config.engine.backend.container.ports, env.DEFAULT_SETTINGS[env.VIZIERENGINE_CONTAINER_PORTS])
     self.assertEqual(config.engine.backend.container.image, env.DEFAULT_SETTINGS[env.VIZIERENGINE_CONTAINER_IMAGE])
 def test_env_config(self):
     """Test app config using environment variables."""
     os.environ[env.VIZIERSERVER_NAME] = 'Some Name'
     os.environ[env.VIZIERSERVER_LOG_DIR] = 'logdir'
     os.environ[env.VIZIERSERVER_DEBUG] = 'bla'
     os.environ[env.VIZIERSERVER_BASE_URL] = 'http://webapi'
     os.environ[env.VIZIERSERVER_SERVER_PORT] = '80'
     os.environ[env.VIZIERSERVER_SERVER_LOCAL_PORT] = '90'
     os.environ[env.VIZIERSERVER_APP_PATH] = 'vizier/v2'
     os.environ[env.VIZIERSERVER_ROW_LIMIT] = '111'
     os.environ[env.VIZIERSERVER_MAX_ROW_LIMIT] = '222'
     os.environ[env.VIZIERSERVER_MAX_UPLOAD_SIZE] = '333'
     os.environ[env.VIZIERSERVER_ENGINE] = 'CELERY'
     os.environ[env.VIZIERENGINE_USE_SHORT_IDENTIFIER] = str(not env.DEFAULT_SETTINGS[env.VIZIERENGINE_USE_SHORT_IDENTIFIER])
     os.environ[env.VIZIERENGINE_SYNCHRONOUS] = 'ABC'
     os.environ[env.VIZIERENGINE_BACKEND] = 'THE_BACKEND'
     os.environ[env.VIZIERENGINE_CELERY_ROUTES] = 'Some Routes'
     os.environ[env.VIZIERENGINE_CONTAINER_PORTS] = '8080-8084,9000,10001-10010'
     config = AppConfig()
     self.assertEqual(config.webservice.name, 'Some Name')
     self.assertEqual(config.webservice.server_url, 'http://webapi')
     self.assertEqual(config.webservice.server_port, 80)
     self.assertEqual(config.webservice.server_local_port, 90)
     self.assertEqual(config.webservice.app_path, 'vizier/v2')
     self.assertEqual(config.webservice.defaults.row_limit, 111)
     self.assertEqual(config.webservice.defaults.max_row_limit, 222)
     self.assertEqual(config.webservice.defaults.max_file_size, 333)
     self.assertEqual(config.run.debug, False)
     self.assertEqual(config.logs.server, 'logdir')
     self.assertEqual(config.engine.identifier, 'CELERY')
     self.assertEqual(config.engine.use_short_ids, not env.DEFAULT_SETTINGS[env.VIZIERENGINE_USE_SHORT_IDENTIFIER])
     self.assertEqual(config.engine.sync_commands, 'ABC')
     self.assertEqual(config.engine.backend.identifier, 'THE_BACKEND')
     self.assertEqual(config.engine.backend.celery.routes, 'Some Routes')
     ports = list(range(8080, 8084)) + [9000] + list(range(10001,10010))
     self.assertEqual(config.engine.backend.container.ports, ports)
     self.assertEqual(config.engine.backend.container.image, env.DEFAULT_SETTINGS[env.VIZIERENGINE_CONTAINER_IMAGE])
Example #8
0
from vizier.api.webservice.base import VizierApi
from vizier.config.app import AppConfig

import vizier.api.base as srv
import vizier.api.serialize.deserialize as deserialize
import vizier.api.serialize.labels as labels
import vizier.config.app as app
import json

# -----------------------------------------------------------------------------
#
# App Blueprint
#
# -----------------------------------------------------------------------------
"""Get application configuration parameters from environment variables."""
config = AppConfig()

webui_file_dir = os.getenv(
    'WEB_UI_STATIC_FILES', "./web-ui/build/"
)  #pkg_resources.resource_filename(__name__, os.getenv('WEB_UI_STATIC_FILES', "./web-ui/build/"))
print(webui_file_dir)

global api
api = VizierApi(config, init=True)

# Create the application blueprint
bp = Blueprint('app',
               __name__,
               url_prefix=config.webservice.app_path,
               static_folder=webui_file_dir)
Example #9
0
 def test_create_synchronous_workflow(self):
     """Create workflow by appending a sequence of modules that are executed
     synchronously.
     """
     project = self.engine.projects.create_project()
     # MODULE 1
     # --------
     # LOAD people
     fh = project.filestore.upload_file(PEOPLE_FILE)
     module = self.engine.append_workflow_module(
         project_id=project.identifier,
         branch_id=project.get_default_branch().identifier,
         command=load_dataset(dataset_name='people',
                              file={pckg.FILE_ID: fh.identifier},
                              validate=True))
     self.assertTrue(module.is_success)
     self.assertTrue('people' in module.provenance.write)
     self.assertEqual(len(module.provenance.write['people'].columns), 2)
     # MODULE 2
     # --------
     # UPDATE CELL
     module = self.engine.append_workflow_module(
         project_id=project.identifier,
         branch_id=project.get_default_branch().identifier,
         command=update_cell(dataset_name='people',
                             column=1,
                             row=0,
                             value='42',
                             validate=True))
     print("STATUS: {}".format(module))
     self.assertTrue(module.is_success)
     self.assertTrue('people' in module.provenance.write)
     # MODULE 3
     # --------
     # LOAD employee
     fh = project.filestore.upload_file(EMPLOYEE_FILE)
     module = self.engine.append_workflow_module(
         project_id=project.identifier,
         branch_id=project.get_default_branch().identifier,
         command=load_dataset(dataset_name='employee',
                              file={pckg.FILE_ID: fh.identifier},
                              validate=True))
     self.assertTrue(module.is_success)
     self.assertFalse('people' in module.provenance.write)
     self.assertTrue('employee' in module.provenance.write)
     #
     # Reload engine and check the module states
     #
     self.engine = get_engine(AppConfig())
     project = self.engine.projects.get_project(project.identifier)
     modules = project.get_default_branch().get_head().modules
     self.assertEqual(len(modules), 3)
     for m in modules:
         self.assertTrue(m.is_success)
         self.assertIsNotNone(m.timestamp.created_at)
         self.assertIsNotNone(m.timestamp.started_at)
         self.assertIsNotNone(m.timestamp.finished_at)
         self.assertIsNotNone(m.provenance.write)
     self.assertTrue('people' in modules[0].provenance.write)
     self.assertTrue('employee' in modules[-1].provenance.write)
     self.assertNotEqual(modules[0].provenance.write['people'].identifier,
                         modules[1].provenance.write['people'].identifier)
Example #10
0
 def test_create_synchronous_workflow_with_errors(self):
     """Create workflow by appending a sequence of modules that are executed
     synchronously.
     """
     project = self.engine.projects.create_project()
     # MODULE 1
     # --------
     # LOAD people
     fh = project.filestore.upload_file(PEOPLE_FILE)
     module = self.engine.append_workflow_module(
         project_id=project.identifier,
         branch_id=project.get_default_branch().identifier,
         command=load_dataset(dataset_name='people',
                              file={
                                  pckg.FILE_ID: fh.identifier,
                                  pckg.FILE_NAME:
                                  os.path.basename(PEOPLE_FILE)
                              },
                              validate=True))
     project = self.engine.projects.get_project(project.identifier)
     modules = project.get_default_branch().get_head().modules
     for m in modules:
         print(m)
         self.assertTrue(m.is_success)
     # MODULE 2
     # --------
     # UPDATE CELL
     module = self.engine.append_workflow_module(
         project_id=project.identifier,
         branch_id=project.get_default_branch().identifier,
         command=update_cell(dataset_name='employee',
                             column=1,
                             row=0,
                             value='42',
                             validate=True))
     self.assertTrue(module.is_error)
     # MODULE 2
     # --------
     # INSERT employee
     fh = project.filestore.upload_file(EMPLOYEE_FILE)
     result = self.engine.insert_workflow_module(
         project_id=project.identifier,
         branch_id=project.get_default_branch().identifier,
         before_module_id=module.identifier,
         command=load_dataset(dataset_name='employee',
                              file={
                                  pckg.FILE_ID:
                                  fh.identifier,
                                  pckg.FILE_NAME:
                                  os.path.basename(EMPLOYEE_FILE)
                              },
                              validate=True))
     self.assertEqual(len(result), 2)
     # Wait for the operations to finish
     while project.viztrail.default_branch.head.is_active:
         time.sleep(0.1)
     #
     # Reload engine and check the module states
     #
     self.engine = get_engine(AppConfig())
     project = self.engine.projects.get_project(project.identifier)
     modules = project.get_default_branch().get_head().modules
     for m in modules:
         print(m)
         self.assertTrue(m.is_success)
     # MODULE 1
     # --------
     # UPDATE CELL
     module = self.engine.insert_workflow_module(
         project_id=project.identifier,
         branch_id=project.get_default_branch().identifier,
         before_module_id=modules[0].identifier,
         command=update_cell(dataset_name='friends',
                             column=1,
                             row=0,
                             value='43',
                             validate=True))
     # Wait for the operations to finish
     while project.viztrail.default_branch.head.is_active:
         time.sleep(0.1)
     modules = project.get_default_branch().get_head().modules
     self.assertEqual(len(modules), 4)
     self.assertTrue(modules[0].is_error)
     for m in modules[1:]:
         self.assertTrue(m.is_canceled)
     # MODULE 1
     # --------
     # INSERT friends
     fh = project.filestore.upload_file(EMPLOYEE_FILE)
     result = self.engine.insert_workflow_module(
         project_id=project.identifier,
         branch_id=project.get_default_branch().identifier,
         before_module_id=modules[0].identifier,
         command=load_dataset(dataset_name='friends',
                              file={
                                  pckg.FILE_ID:
                                  fh.identifier,
                                  pckg.FILE_NAME:
                                  os.path.basename(EMPLOYEE_FILE)
                              },
                              validate=True))
     self.assertEqual(len(result), 5)
     # Wait for the operations to finish
     while project.viztrail.default_branch.head.is_active:
         time.sleep(0.1)
     modules = project.get_default_branch().get_head().modules
     self.assertEqual(len(modules), 5)
     for m in modules:
         self.assertTrue(m.is_success)
     self.assertEqual(len(modules[0].provenance.write['friends'].columns),
                      3)
     # REPLACE MODULE 1
     # ----------------
     # Load people dataset instead employee
     fh = project.filestore.upload_file(PEOPLE_FILE)
     result = self.engine.replace_workflow_module(
         project_id=project.identifier,
         branch_id=project.get_default_branch().identifier,
         module_id=modules[0].identifier,
         command=load_dataset(dataset_name='friends',
                              file={
                                  pckg.FILE_ID: fh.identifier,
                                  pckg.FILE_NAME:
                                  os.path.basename(PEOPLE_FILE)
                              },
                              validate=True))
     self.assertEqual(len(result), 5)
     # Wait for the operations to finish
     while project.viztrail.default_branch.head.is_active:
         time.sleep(0.1)
     modules = project.get_default_branch().get_head().modules
     self.assertEqual(len(modules), 5)
     for m in modules:
         self.assertTrue(m.is_success)
     self.assertEqual(len(modules[0].provenance.write['friends'].columns),
                      2)
     ds = project.datastore.get_dataset(
         modules[0].provenance.write['friends'].identifier)
     self.assertEqual(ds.fetch_rows()[0].values[1], 23)
     #
     # Reload engine and check the module states
     #
     self.engine = get_engine(AppConfig())
     project = self.engine.projects.get_project(project.identifier)
     modules = project.get_default_branch().get_head().modules
     self.assertEqual(len(modules), 5)
     for m in modules:
         self.assertTrue(m.is_success)
     self.assertEqual(len(modules[0].provenance.write['friends'].columns),
                      2)
     ds = project.datastore.get_dataset(
         modules[0].provenance.write['friends'].identifier)
     self.assertEqual(ds.fetch_rows()[0].values[1], 23)
Example #11
0
def create_app():
    """Factory pattern for Flask. Initialize the Flask application object.

    Returns
    -------
    Flask
    """
    #Get application configuration parameters from environment variables.
    config = AppConfig()
    # Create the app and enable cross-origin resource sharing
    app = Flask(__name__)
    #app.config['APPLICATION_ROOT'] = config.webservice.app_path
    #app.config['DEBUG'] = True
    # Set size limit for uploaded files
    app.config['MAX_CONTENT_LENGTH'] = config.webservice.defaults.max_file_size
    # Enable CORS
    CORS(app)
    # Switch logging on
    log_dir = os.path.abspath(config.logs.server)
    # Create the directory if it does not exist
    if not os.path.isdir(log_dir):
        os.makedirs(log_dir)
    # File handle for server logs
    file_handler = RotatingFileHandler(os.path.join(log_dir,
                                                    'vizier-webapi.log'),
                                       maxBytes=1024 * 1024 * 100,
                                       backupCount=20)
    file_handler.setLevel(logging.ERROR)
    file_handler.setFormatter(
        logging.Formatter("%(asctime)s - %(levelname)s - %(message)s"))
    app.logger.addHandler(file_handler)

    # --------------------------------------------------------------------------
    #
    # Error Handler
    #
    # --------------------------------------------------------------------------

    @app.errorhandler(srv.ServerRequestException)
    def invalid_request_or_resource_not_found(error):
        """JSON response handler for invalid requests or requests that access
        unknown resources.

        Parameters
        ----------
        error : Exception
            Exception thrown by request Handler

        Returns
        -------
        Http response
        """
        app.logger.error(error.message)
        response = jsonify(error.to_dict())
        response.status_code = error.status_code
        return response

    @app.errorhandler(413)
    def upload_error(exception):
        """Exception handler for file uploads that exceed the file size limit."""
        app.logger.error(exception)
        return make_response(jsonify({'error': str(exception)}), 413)

    @app.errorhandler(500)
    def internal_error(exception):
        """Exception handler that logs exceptions."""
        app.logger.error(exception)
        return make_response(jsonify({'error': str(exception)}), 500)

    # Register the API blueprint
    from . import server
    app.register_blueprint(server.bp)
    # Return the applicatio object

    # --------------------------------------------------------------------------
    #
    # Initialize
    #
    # --------------------------------------------------------------------------

    @app.before_first_request
    def initialize():
        """Initialize Mimir gateway (if necessary) before the first request.
        """
        # Initialize the Mimir gateway if using Mimir engine
        if config.engine.identifier == const.MIMIR_ENGINE:
            import vizier.mimir as mimir

    root_redirect_path = "{}/web-ui/vizier-db".format(server.bp.url_prefix)

    @app.route("/")
    def handle_root():
        """Redirect users to the web UI
        """
        return redirect(root_redirect_path)

    return app