def test_load(self):
"""Test loading packages from a configuration dictionary."""
os.environ[env.VIZIERSERVER_PACKAGE_PATH] = PACKAGES_DIR
config = AppConfig()
packages = load_packages(config.engine.package_path)
for key in [PACKAGE_PLOT, PACKAGE_PYTHON, PACKAGE_VIZUAL]:
self.assertTrue(key in packages)
def setUp(self):
"""Create engine for an empty repository."""
# Drop directory if it exists
if os.path.isdir(SERVER_DIR):
shutil.rmtree(SERVER_DIR)
os.makedirs(SERVER_DIR)
os.environ[app.VIZIERSERVER_ENGINE] = MIMIR_ENGINE
os.environ[app.VIZIERENGINE_DATA_DIR] = SERVER_DIR
os.environ[app.VIZIERSERVER_PACKAGE_PATH] = PACKAGES_DIR
os.environ[app.VIZIERSERVER_PROCESSOR_PATH] = PROCESSORS_DIR
self.engine = get_engine(AppConfig())
def setUp(self):
"""Create an instance of the default vizier processor for an empty server
directory.
"""
# Drop directory if it exists
if os.path.isdir(SERVER_DIR):
shutil.rmtree(SERVER_DIR)
os.makedirs(SERVER_DIR)
os.environ[app.VIZIERENGINE_DATA_DIR] = SERVER_DIR
os.environ[app.VIZIERSERVER_PACKAGE_PATH] = PACKAGES_DIR
os.environ[app.VIZIERSERVER_PROCESSOR_PATH] = PROCESSORS_DIR
self.engine = get_engine(AppConfig())
def setUp(self):
"""Create an instance of the default backend with an empty directory
"""
if os.path.isdir(SERVER_DIR):
shutil.rmtree(SERVER_DIR)
os.makedirs(SERVER_DIR)
os.environ[app.VIZIERENGINE_DATA_DIR] = SERVER_DIR
os.environ[app.VIZIERSERVER_PACKAGE_PATH] = PACKAGES_DIR
os.environ[app.VIZIERSERVER_PROCESSOR_PATH] = PROCESSORS_DIR
os.environ[app.VIZIERENGINE_SYNCHRONOUS] = ':'.join([
vizual.PACKAGE_VIZUAL + '.' + vizual.VIZUAL_LOAD,
vizual.PACKAGE_VIZUAL + '.' + vizual.VIZUAL_UPD_CELL
])
self.engine = get_engine(AppConfig())
def test_create_cache(self):
"""Test accessing and deleting projects for an empty repository."""
viztrails = OSViztrailRepository(base_path=VIZTRAILS_DIR)
vt1 = viztrails.create_viztrail(
properties={PROPERTY_NAME: 'My Project'})
vt2 = viztrails.create_viztrail(
properties={PROPERTY_NAME: 'A Project'})
filename = os.path.join(SERVER_DIR, 'container.json')
DefaultObjectStore().write_object(object_path=filename,
content=[{
'projectId': vt1.identifier,
'url': 'API1',
'port': 80,
'containerId': 'ID1'
}, {
'projectId': vt2.identifier,
'url': 'API2',
'port': 81,
'containerId': 'ID2'
}])
# Initialize the project cache
viztrails = OSViztrailRepository(base_path=VIZTRAILS_DIR)
filestores_dir = os.path.join(SERVER_DIR, DEFAULT_FILESTORES_DIR)
datastores_dir = os.path.join(SERVER_DIR, DEFAULT_DATASTORES_DIR)
projects = ContainerProjectCache(
viztrails=viztrails,
container_file=filename,
config=AppConfig(),
datastores=MimirDatastoreFactory(datastores_dir),
filestores=FileSystemFilestoreFactory(filestores_dir))
self.assertEqual(len(projects.list_projects()), 2)
self.assertEqual(
projects.get_project(vt1.identifier).container_api, 'API1')
self.assertEqual(
projects.get_project(vt2.identifier).container_api, 'API2')
self.assertEqual(
projects.get_project(vt1.identifier).container_id, 'ID1')
self.assertEqual(
projects.get_project(vt2.identifier).container_id, 'ID2')
self.assertEqual(projects.get_project(vt1.identifier).port, 80)
self.assertEqual(projects.get_project(vt2.identifier).port, 81)
def test_default_config(self):
"""Test the default configuration settings."""
config = AppConfig()
self.assertEqual(config.webservice.name, env.DEFAULT_SETTINGS[env.VIZIERSERVER_NAME])
self.assertEqual(config.webservice.server_url, env.DEFAULT_SETTINGS[env.VIZIERSERVER_BASE_URL])
self.assertEqual(config.webservice.server_port, env.DEFAULT_SETTINGS[env.VIZIERSERVER_SERVER_PORT])
self.assertEqual(config.webservice.server_local_port, env.DEFAULT_SETTINGS[env.VIZIERSERVER_SERVER_LOCAL_PORT])
self.assertEqual(config.webservice.app_path, env.DEFAULT_SETTINGS[env.VIZIERSERVER_APP_PATH])
self.assertEqual(config.webservice.defaults.row_limit, env.DEFAULT_SETTINGS[env.VIZIERSERVER_ROW_LIMIT])
self.assertEqual(config.webservice.defaults.max_row_limit, env.DEFAULT_SETTINGS[env.VIZIERSERVER_MAX_ROW_LIMIT])
self.assertEqual(config.webservice.defaults.max_file_size, env.DEFAULT_SETTINGS[env.VIZIERSERVER_MAX_UPLOAD_SIZE])
self.assertEqual(config.run.debug, env.DEFAULT_SETTINGS[env.VIZIERSERVER_DEBUG])
self.assertEqual(config.logs.server, env.DEFAULT_SETTINGS[env.VIZIERSERVER_LOG_DIR])
self.assertEqual(config.engine.identifier, env.DEFAULT_SETTINGS[env.VIZIERSERVER_ENGINE])
self.assertEqual(config.engine.data_dir, env.DEFAULT_SETTINGS[env.VIZIERENGINE_DATA_DIR])
self.assertEqual(config.engine.package_path, env.DEFAULT_SETTINGS[env.VIZIERSERVER_PACKAGE_PATH])
self.assertEqual(config.engine.processor_path, env.DEFAULT_SETTINGS[env.VIZIERSERVER_PROCESSOR_PATH])
self.assertEqual(config.engine.use_short_ids, env.DEFAULT_SETTINGS[env.VIZIERENGINE_USE_SHORT_IDENTIFIER])
self.assertEqual(config.engine.sync_commands, env.DEFAULT_SETTINGS[env.VIZIERENGINE_SYNCHRONOUS])
self.assertEqual(config.engine.backend.identifier, env.DEFAULT_SETTINGS[env.VIZIERENGINE_BACKEND])
self.assertEqual(config.engine.backend.celery.routes, env.DEFAULT_SETTINGS[env.VIZIERENGINE_CELERY_ROUTES])
self.assertEqual(config.engine.backend.container.ports, env.DEFAULT_SETTINGS[env.VIZIERENGINE_CONTAINER_PORTS])
self.assertEqual(config.engine.backend.container.image, env.DEFAULT_SETTINGS[env.VIZIERENGINE_CONTAINER_IMAGE])
def test_env_config(self):
"""Test app config using environment variables."""
os.environ[env.VIZIERSERVER_NAME] = 'Some Name'
os.environ[env.VIZIERSERVER_LOG_DIR] = 'logdir'
os.environ[env.VIZIERSERVER_DEBUG] = 'bla'
os.environ[env.VIZIERSERVER_BASE_URL] = 'http://webapi'
os.environ[env.VIZIERSERVER_SERVER_PORT] = '80'
os.environ[env.VIZIERSERVER_SERVER_LOCAL_PORT] = '90'
os.environ[env.VIZIERSERVER_APP_PATH] = 'vizier/v2'
os.environ[env.VIZIERSERVER_ROW_LIMIT] = '111'
os.environ[env.VIZIERSERVER_MAX_ROW_LIMIT] = '222'
os.environ[env.VIZIERSERVER_MAX_UPLOAD_SIZE] = '333'
os.environ[env.VIZIERSERVER_ENGINE] = 'CELERY'
os.environ[env.VIZIERENGINE_USE_SHORT_IDENTIFIER] = str(not env.DEFAULT_SETTINGS[env.VIZIERENGINE_USE_SHORT_IDENTIFIER])
os.environ[env.VIZIERENGINE_SYNCHRONOUS] = 'ABC'
os.environ[env.VIZIERENGINE_BACKEND] = 'THE_BACKEND'
os.environ[env.VIZIERENGINE_CELERY_ROUTES] = 'Some Routes'
os.environ[env.VIZIERENGINE_CONTAINER_PORTS] = '8080-8084,9000,10001-10010'
config = AppConfig()
self.assertEqual(config.webservice.name, 'Some Name')
self.assertEqual(config.webservice.server_url, 'http://webapi')
self.assertEqual(config.webservice.server_port, 80)
self.assertEqual(config.webservice.server_local_port, 90)
self.assertEqual(config.webservice.app_path, 'vizier/v2')
self.assertEqual(config.webservice.defaults.row_limit, 111)
self.assertEqual(config.webservice.defaults.max_row_limit, 222)
self.assertEqual(config.webservice.defaults.max_file_size, 333)
self.assertEqual(config.run.debug, False)
self.assertEqual(config.logs.server, 'logdir')
self.assertEqual(config.engine.identifier, 'CELERY')
self.assertEqual(config.engine.use_short_ids, not env.DEFAULT_SETTINGS[env.VIZIERENGINE_USE_SHORT_IDENTIFIER])
self.assertEqual(config.engine.sync_commands, 'ABC')
self.assertEqual(config.engine.backend.identifier, 'THE_BACKEND')
self.assertEqual(config.engine.backend.celery.routes, 'Some Routes')
ports = list(range(8080, 8084)) + [9000] + list(range(10001,10010))
self.assertEqual(config.engine.backend.container.ports, ports)
self.assertEqual(config.engine.backend.container.image, env.DEFAULT_SETTINGS[env.VIZIERENGINE_CONTAINER_IMAGE])
from vizier.api.webservice.base import VizierApi
from vizier.config.app import AppConfig
import vizier.api.base as srv
import vizier.api.serialize.deserialize as deserialize
import vizier.api.serialize.labels as labels
import vizier.config.app as app
import json
# -----------------------------------------------------------------------------
#
# App Blueprint
#
# -----------------------------------------------------------------------------
"""Get application configuration parameters from environment variables."""
config = AppConfig()
webui_file_dir = os.getenv(
'WEB_UI_STATIC_FILES', "./web-ui/build/"
) #pkg_resources.resource_filename(__name__, os.getenv('WEB_UI_STATIC_FILES', "./web-ui/build/"))
print(webui_file_dir)
global api
api = VizierApi(config, init=True)
# Create the application blueprint
bp = Blueprint('app',
__name__,
url_prefix=config.webservice.app_path,
static_folder=webui_file_dir)
def test_create_synchronous_workflow(self):
"""Create workflow by appending a sequence of modules that are executed
synchronously.
"""
project = self.engine.projects.create_project()
# MODULE 1
# --------
# LOAD people
fh = project.filestore.upload_file(PEOPLE_FILE)
module = self.engine.append_workflow_module(
project_id=project.identifier,
branch_id=project.get_default_branch().identifier,
command=load_dataset(dataset_name='people',
file={pckg.FILE_ID: fh.identifier},
validate=True))
self.assertTrue(module.is_success)
self.assertTrue('people' in module.provenance.write)
self.assertEqual(len(module.provenance.write['people'].columns), 2)
# MODULE 2
# --------
# UPDATE CELL
module = self.engine.append_workflow_module(
project_id=project.identifier,
branch_id=project.get_default_branch().identifier,
command=update_cell(dataset_name='people',
column=1,
row=0,
value='42',
validate=True))
print("STATUS: {}".format(module))
self.assertTrue(module.is_success)
self.assertTrue('people' in module.provenance.write)
# MODULE 3
# --------
# LOAD employee
fh = project.filestore.upload_file(EMPLOYEE_FILE)
module = self.engine.append_workflow_module(
project_id=project.identifier,
branch_id=project.get_default_branch().identifier,
command=load_dataset(dataset_name='employee',
file={pckg.FILE_ID: fh.identifier},
validate=True))
self.assertTrue(module.is_success)
self.assertFalse('people' in module.provenance.write)
self.assertTrue('employee' in module.provenance.write)
#
# Reload engine and check the module states
#
self.engine = get_engine(AppConfig())
project = self.engine.projects.get_project(project.identifier)
modules = project.get_default_branch().get_head().modules
self.assertEqual(len(modules), 3)
for m in modules:
self.assertTrue(m.is_success)
self.assertIsNotNone(m.timestamp.created_at)
self.assertIsNotNone(m.timestamp.started_at)
self.assertIsNotNone(m.timestamp.finished_at)
self.assertIsNotNone(m.provenance.write)
self.assertTrue('people' in modules[0].provenance.write)
self.assertTrue('employee' in modules[-1].provenance.write)
self.assertNotEqual(modules[0].provenance.write['people'].identifier,
modules[1].provenance.write['people'].identifier)
def test_create_synchronous_workflow_with_errors(self):
"""Create workflow by appending a sequence of modules that are executed
synchronously.
"""
project = self.engine.projects.create_project()
# MODULE 1
# --------
# LOAD people
fh = project.filestore.upload_file(PEOPLE_FILE)
module = self.engine.append_workflow_module(
project_id=project.identifier,
branch_id=project.get_default_branch().identifier,
command=load_dataset(dataset_name='people',
file={
pckg.FILE_ID: fh.identifier,
pckg.FILE_NAME:
os.path.basename(PEOPLE_FILE)
},
validate=True))
project = self.engine.projects.get_project(project.identifier)
modules = project.get_default_branch().get_head().modules
for m in modules:
print(m)
self.assertTrue(m.is_success)
# MODULE 2
# --------
# UPDATE CELL
module = self.engine.append_workflow_module(
project_id=project.identifier,
branch_id=project.get_default_branch().identifier,
command=update_cell(dataset_name='employee',
column=1,
row=0,
value='42',
validate=True))
self.assertTrue(module.is_error)
# MODULE 2
# --------
# INSERT employee
fh = project.filestore.upload_file(EMPLOYEE_FILE)
result = self.engine.insert_workflow_module(
project_id=project.identifier,
branch_id=project.get_default_branch().identifier,
before_module_id=module.identifier,
command=load_dataset(dataset_name='employee',
file={
pckg.FILE_ID:
fh.identifier,
pckg.FILE_NAME:
os.path.basename(EMPLOYEE_FILE)
},
validate=True))
self.assertEqual(len(result), 2)
# Wait for the operations to finish
while project.viztrail.default_branch.head.is_active:
time.sleep(0.1)
#
# Reload engine and check the module states
#
self.engine = get_engine(AppConfig())
project = self.engine.projects.get_project(project.identifier)
modules = project.get_default_branch().get_head().modules
for m in modules:
print(m)
self.assertTrue(m.is_success)
# MODULE 1
# --------
# UPDATE CELL
module = self.engine.insert_workflow_module(
project_id=project.identifier,
branch_id=project.get_default_branch().identifier,
before_module_id=modules[0].identifier,
command=update_cell(dataset_name='friends',
column=1,
row=0,
value='43',
validate=True))
# Wait for the operations to finish
while project.viztrail.default_branch.head.is_active:
time.sleep(0.1)
modules = project.get_default_branch().get_head().modules
self.assertEqual(len(modules), 4)
self.assertTrue(modules[0].is_error)
for m in modules[1:]:
self.assertTrue(m.is_canceled)
# MODULE 1
# --------
# INSERT friends
fh = project.filestore.upload_file(EMPLOYEE_FILE)
result = self.engine.insert_workflow_module(
project_id=project.identifier,
branch_id=project.get_default_branch().identifier,
before_module_id=modules[0].identifier,
command=load_dataset(dataset_name='friends',
file={
pckg.FILE_ID:
fh.identifier,
pckg.FILE_NAME:
os.path.basename(EMPLOYEE_FILE)
},
validate=True))
self.assertEqual(len(result), 5)
# Wait for the operations to finish
while project.viztrail.default_branch.head.is_active:
time.sleep(0.1)
modules = project.get_default_branch().get_head().modules
self.assertEqual(len(modules), 5)
for m in modules:
self.assertTrue(m.is_success)
self.assertEqual(len(modules[0].provenance.write['friends'].columns),
3)
# REPLACE MODULE 1
# ----------------
# Load people dataset instead employee
fh = project.filestore.upload_file(PEOPLE_FILE)
result = self.engine.replace_workflow_module(
project_id=project.identifier,
branch_id=project.get_default_branch().identifier,
module_id=modules[0].identifier,
command=load_dataset(dataset_name='friends',
file={
pckg.FILE_ID: fh.identifier,
pckg.FILE_NAME:
os.path.basename(PEOPLE_FILE)
},
validate=True))
self.assertEqual(len(result), 5)
# Wait for the operations to finish
while project.viztrail.default_branch.head.is_active:
time.sleep(0.1)
modules = project.get_default_branch().get_head().modules
self.assertEqual(len(modules), 5)
for m in modules:
self.assertTrue(m.is_success)
self.assertEqual(len(modules[0].provenance.write['friends'].columns),
2)
ds = project.datastore.get_dataset(
modules[0].provenance.write['friends'].identifier)
self.assertEqual(ds.fetch_rows()[0].values[1], 23)
#
# Reload engine and check the module states
#
self.engine = get_engine(AppConfig())
project = self.engine.projects.get_project(project.identifier)
modules = project.get_default_branch().get_head().modules
self.assertEqual(len(modules), 5)
for m in modules:
self.assertTrue(m.is_success)
self.assertEqual(len(modules[0].provenance.write['friends'].columns),
2)
ds = project.datastore.get_dataset(
modules[0].provenance.write['friends'].identifier)
self.assertEqual(ds.fetch_rows()[0].values[1], 23)
def create_app():
"""Factory pattern for Flask. Initialize the Flask application object.
Returns
-------
Flask
"""
#Get application configuration parameters from environment variables.
config = AppConfig()
# Create the app and enable cross-origin resource sharing
app = Flask(__name__)
#app.config['APPLICATION_ROOT'] = config.webservice.app_path
#app.config['DEBUG'] = True
# Set size limit for uploaded files
app.config['MAX_CONTENT_LENGTH'] = config.webservice.defaults.max_file_size
# Enable CORS
CORS(app)
# Switch logging on
log_dir = os.path.abspath(config.logs.server)
# Create the directory if it does not exist
if not os.path.isdir(log_dir):
os.makedirs(log_dir)
# File handle for server logs
file_handler = RotatingFileHandler(os.path.join(log_dir,
'vizier-webapi.log'),
maxBytes=1024 * 1024 * 100,
backupCount=20)
file_handler.setLevel(logging.ERROR)
file_handler.setFormatter(
logging.Formatter("%(asctime)s - %(levelname)s - %(message)s"))
app.logger.addHandler(file_handler)
# --------------------------------------------------------------------------
#
# Error Handler
#
# --------------------------------------------------------------------------
@app.errorhandler(srv.ServerRequestException)
def invalid_request_or_resource_not_found(error):
"""JSON response handler for invalid requests or requests that access
unknown resources.
Parameters
----------
error : Exception
Exception thrown by request Handler
Returns
-------
Http response
"""
app.logger.error(error.message)
response = jsonify(error.to_dict())
response.status_code = error.status_code
return response
@app.errorhandler(413)
def upload_error(exception):
"""Exception handler for file uploads that exceed the file size limit."""
app.logger.error(exception)
return make_response(jsonify({'error': str(exception)}), 413)
@app.errorhandler(500)
def internal_error(exception):
"""Exception handler that logs exceptions."""
app.logger.error(exception)
return make_response(jsonify({'error': str(exception)}), 500)
# Register the API blueprint
from . import server
app.register_blueprint(server.bp)
# Return the applicatio object
# --------------------------------------------------------------------------
#
# Initialize
#
# --------------------------------------------------------------------------
@app.before_first_request
def initialize():
"""Initialize Mimir gateway (if necessary) before the first request.
"""
# Initialize the Mimir gateway if using Mimir engine
if config.engine.identifier == const.MIMIR_ENGINE:
import vizier.mimir as mimir
root_redirect_path = "{}/web-ui/vizier-db".format(server.bp.url_prefix)
@app.route("/")
def handle_root():
"""Redirect users to the web UI
"""
return redirect(root_redirect_path)
return app