def get_url_factory( config: AppConfig, projects: ProjectCache ) -> UrlFactory: """Get the url factory for a given configuration. In most cases we use the default url factory. Only for the configuration where each project is running in a separate container we need a different factory. Parameter --------- config: vizier.config.app.AppConfig Application configuration object projects: vizier.engine.project.cache.base.ProjectCache Cache for projects (only used for container engine) Returns ------- vizier.api.routes.base.UrlFactory """ if config.engine.identifier == base.CONTAINER_ENGINE: return ContainerEngineUrlFactory( base_url=config.app_base_url, api_doc_url=config.webservice.doc_url, projects=cast(ContainerProjectCache, projects) ) else: return UrlFactory( base_url=config.app_base_url, api_doc_url=config.webservice.doc_url )
def main(args): """Read user input from stdin until either quit, exit or CTRL-D is entered. """ # Initialize the url factory and read default values. app_dir = get_base_directory() config_file = os.path.join(app_dir, CONFIG_FILE) if len(args) in [1, 2] and args[0] == 'init': if len(args) == 2: url = args[1] elif not os.path.isfile(config_file): url = 'http://localhost:5000/vizier-db/api/v1' else: # Print the URL of the API in the configuration file config = read_object_from_file(config_file) print_header() print('\nConnected to API at ' + config['url']) return if not os.path.isdir(app_dir): os.makedirs(app_dir) with open(config_file, 'w') as f: json.dump({'url': url}, f) elif not os.path.isfile(config_file): raise ValueError('vizier client is not initialized') else: config = read_object_from_file(config_file) defaults_file = os.path.join(app_dir, 'defaults.json') defaults = PersistentAnnotationSet(object_path=defaults_file) CommandInterpreter(urls=UrlFactory(base_url=config['url']), defaults=defaults).eval(args)
def test_init_url_factory(self): """Test initializing the main url factory.""" urls = UrlFactory(base_url='http://abc.com/////') self.assertEqual(urls.base_url, 'http://abc.com') self.assertIsNone(urls.api_doc_url) urls = UrlFactory(base_url='http://abc.com/////', api_doc_url='ABC') self.assertEqual(urls.base_url, 'http://abc.com') self.assertEqual(urls.api_doc_url, 'ABC') # Override API doc url via properties urls = UrlFactory(base_url='http://abc.com/////', api_doc_url='ABC', properties={PROPERTIES_APIDOCURL: 'XYZ'}) self.assertEqual(urls.base_url, 'http://abc.com') self.assertEqual(urls.api_doc_url, 'XYZ') # Override base url via properties urls = UrlFactory(base_url='http://abc.com/////', api_doc_url='ABC', properties={PROPERTIES_BASEURL: 'XYZ'}) self.assertEqual(urls.base_url, 'XYZ') self.assertEqual(urls.api_doc_url, 'ABC') # Initialize only via properties urls = UrlFactory(properties={ PROPERTIES_BASEURL: 'XYZ', PROPERTIES_APIDOCURL: 'ABC' }) self.assertEqual(urls.base_url, 'XYZ') self.assertEqual(urls.api_doc_url, 'ABC') # Value error if base url is not set with self.assertRaises(ValueError): urls = UrlFactory(api_doc_url='ABC', properties={PROPERTIES_APIDOCURL: 'XYZ'})
def get_datastore(self, identifier): """Get the datastore client for the project with the given identifier. Paramaters ---------- identifier: string Unique identifier for datastore Returns ------- vizier.api.client.datastore.base.DatastoreClient """ return DatastoreClient( urls=DatastoreClientUrlFactory(urls=UrlFactory( base_url=self.webservice_url), project_id=identifier))
def WORKFLOW_HANDLE_LINKS(urls: UrlFactory, project_id: str, branch_id: str, workflow_id: Optional[str] = None, links: Optional[Dict[str, Optional[str]]] = None): """Get basic set of HATEOAS references for workflow handles. For an empty workflow the identifier is None. In that case the result will not contain a self reference. Parameters ---------- urls: vizier.api.routes.base.UrlFactory Factory for resource urls project_id: string Unique project identifier branch_id: string Unique branch identifier workflow_id: string, optional Unique workflow identifier Returns ------- dict """ if links is None: links = dict() links[ref.WORKFLOW_APPEND] = urls.workflow_module_append( project_id=project_id, branch_id=branch_id) # References to the workflow branch links[ref.WORKFLOW_BRANCH] = urls.get_branch(project_id=project_id, branch_id=branch_id) links[ref.BRANCH_HEAD] = urls.get_branch_head(project_id=project_id, branch_id=branch_id) links[ref.WORKFLOW_PROJECT] = urls.get_project(project_id) links[ref.FILE_UPLOAD] = urls.upload_file(project_id) # Only include self reference if workflow identifier is given if not workflow_id is None: links[ref.SELF] = urls.get_workflow(project_id=project_id, branch_id=branch_id, workflow_id=workflow_id) return serialize.HATEOAS(links)
"""Test the vizier client datastore. This requires the web service API to run and a project with datastores to have been setup. """ from vizier.api.client.base import VizierApiClient from vizier.api.client.datastore.base import DatastoreClient from vizier.api.routes.base import UrlFactory from vizier.api.routes.datastore import DatastoreClientUrlFactory from vizier.datastore.dataset import DatasetColumn, DatasetRow from atexit import register as at_exit URLS = UrlFactory(base_url='http://localhost:5000/vizier-db/api/v1') api = VizierApiClient(URLS) PROJECT_ID = api.create_project({"name": "Test Client Datastore"}).identifier at_exit(api.delete_project, PROJECT_ID) # We're just doing some unit testing on the fields specific to DatastoreClient, so # ignore complaints about instantiating an abstract class store = DatastoreClient( # type: ignore[abstract] urls=DatastoreClientUrlFactory(urls=URLS, project_id=PROJECT_ID)) ds = store.create_dataset(columns=[ DatasetColumn(identifier=0, name='Name'), DatasetColumn(identifier=1, name='Age', data_type="int") ], rows=[ DatasetRow(identifier=0, values=['Alice', 32]), DatasetRow(identifier=1, values=['Bob', 23])
def MODULE_HANDLE(project: "ProjectHandle", branch: BranchHandle, module: ModuleHandle, urls: UrlFactory, workflow: Optional[WorkflowHandle] = None, charts: List[ChartViewHandle] = None, include_self: bool = True) -> Dict[str, Any]: """Dictionary serialization for a handle in the workflow at the branch head. The list of references will only contain a self referene if the include_self flag is True. Parameters ---------- project: vizier.engine.project.base.ProjectHandle Handle for the containing project branch : vizier.viztrail.branch.BranchHandle Branch handle workflow: vizier.viztrail.workflow.WorkflowHandle Workflow handle module: vizier.viztrail.module.base.ModuleHandle Module handle charts: list(vizier.view.chart.ChartViewHandle) List of handles for available chart views urls: vizier.api.routes.base.UrlFactory Factory for resource urls include_self: bool, optional Indicate if self link is included Returns ------- dict """ project_id = project.identifier branch_id = branch.identifier module_id = module.identifier cmd = module.command timestamp = module.timestamp actual_workflow = branch.get_head() if workflow is None else workflow obj: Dict[str, Any] = { labels.ID: module_id, 'state': module.state, labels.COMMAND: { labels.COMMAND_PACKAGE: cmd.package_id, labels.COMMAND_ID: cmd.command_id, labels.COMMAND_ARGS: cmd.arguments.to_list() }, 'text': module.external_form, labels.TIMESTAMPS: { labels.CREATED_AT: timestamp.created_at.isoformat() }, labels.LINKS: serialize.HATEOAS({} if module_id is None else { ref.MODULE_INSERT: urls.workflow_module_insert(project_id=project_id, branch_id=branch_id, module_id=module_id) }) } if include_self: obj[labels.LINKS].extend( serialize.HATEOAS({} if module_id is None else { ref.SELF: urls.get_workflow_module(project_id=project_id, branch_id=branch_id, module_id=module_id), ref.MODULE_DELETE: urls.workflow_module_delete(project_id=project_id, branch_id=branch_id, module_id=module_id), ref.MODULE_REPLACE: urls.workflow_module_replace(project_id=project_id, branch_id=branch_id, module_id=module_id) })) if not timestamp.started_at is None: obj[labels.TIMESTAMPS][ labels.STARTED_AT] = timestamp.started_at.isoformat() # Add outputs and datasets if module is not active. if not module.is_active: artifacts: Dict[str, ArtifactDescriptor] = dict() for precursor in actual_workflow.modules: artifacts = precursor.provenance.get_database_state(artifacts) if precursor == module: break datasets = list() other_artifacts = list() for artifact_name in artifacts: artifact = artifacts[artifact_name] if artifact.is_dataset: datasets.append( serialds.DATASET_IDENTIFIER(identifier=artifact.identifier, name=artifact_name)) else: other_artifacts.append( serialds.ARTIFACT_DESCRIPTOR(artifact=artifact, project=project_id)) available_charts = list() if charts is not None: for c_handle in charts: available_charts.append({ labels.NAME: c_handle.chart_name, labels.LINKS: serialize.HATEOAS({} if module_id is None else { ref.SELF: urls.get_chart_view(project_id=project_id, branch_id=branch_id, workflow_id=actual_workflow. identifier, module_id=module_id, chart_id=c_handle.identifier) }) }) obj[labels.DATASETS] = datasets obj[labels.CHARTS] = available_charts obj[labels.OUTPUTS] = serialize.OUTPUTS(module.outputs) obj[labels.ARTIFACTS] = other_artifacts if not timestamp.finished_at is None: obj[labels.TIMESTAMPS][ labels.FINISHED_AT] = timestamp.finished_at.isoformat() else: # Add empty lists for outputs, datasets and charts if the module is # active obj[labels.DATASETS] = list() obj[labels.CHARTS] = list() obj[labels.OUTPUTS] = serialize.OUTPUTS(ModuleOutputs()) obj[labels.ARTIFACTS] = list() return obj
def WORKFLOW_HANDLE(project: "ProjectHandle", branch: "BranchHandle", workflow: "WorkflowHandle", urls: UrlFactory) -> Dict[str, Any]: """Dictionary serialization for a workflow handle. Parameters ---------- project: vizier.engine.project.base.ProjectHandle Handle for the containing project branch : vizier.viztrail.branch.BranchHandle Branch handle workflow: vizier.viztrail.workflow.WorkflowHandle Workflow handle urls: vizier.api.routes.base.UrlFactory Factory for resource urls Returns ------- dict """ project_id = project.identifier branch_id = branch.identifier workflow_id = workflow.identifier descriptor = workflow.descriptor read_only = (branch.get_head().identifier != workflow_id) # Create lists of module handles and dataset handles modules = list() datasets = dict() dataset_names = list() dataobjects = dict() charts = dict() for m in workflow.modules: if not m.provenance.charts is None: for chart_name, chart in m.provenance.charts: charts[chart_name] = chart available_charts = list() # Only include charts for modules that completed successful for artifact in m.artifacts: if artifact.is_dataset: datasets[artifact.identifier] = serialds.DATASET_DESCRIPTOR( dataset=artifact, project=project, urls=urls) dataset_names.append(artifact.name) else: dataobjects[ artifact.identifier] = serialds.ARTIFACT_DESCRIPTOR( artifact=artifact, project=project, urls=urls) if m.is_success: for c_handle in list(charts.values()): if c_handle.dataset_name in dataset_names: available_charts.append(c_handle) modules.append( serialmd.MODULE_HANDLE(project=project, branch=branch, workflow=workflow, module=m, charts=available_charts, urls=urls, include_self=(not read_only))) handle_links: Optional[Dict[str, Optional[str]]] = None if workflow.is_active: handle_links = { ref.WORKFLOW_CANCEL: urls.cancel_workflow(project_id=project_id, branch_id=branch_id) } links = WORKFLOW_HANDLE_LINKS(project_id=project_id, branch_id=branch_id, workflow_id=workflow_id, urls=urls, links=handle_links) return { 'id': workflow_id, 'createdAt': descriptor.created_at.isoformat(), 'action': descriptor.action, labels.COMMAND_PACKAGE: descriptor.package_id, labels.COMMAND_ID: descriptor.command_id, 'state': workflow.get_state().state, 'modules': modules, 'datasets': list(datasets.values()), 'dataobjects': list(dataobjects.values()), 'readOnly': read_only, labels.LINKS: links }
def DATASET_HANDLE( project: ProjectHandle, dataset: DatasetHandle, rows: List[DatasetRow], defaults: Any, # ConfigObject uses type hacking... pretend it's an any urls: UrlFactory, offset: int = 0, limit: int = -1): """Dictionary serialization for dataset handle. Includes (part of) the dataset rows. Parameters ---------- project: vizier.engine.project.base.ProjectHandle Handle for project containing the dataset dataset : vizier.datastore.dataset.DatasetDescriptor Dataset descriptor rows: list(vizier.datastore.dataset.DatasetRow) List of rows from the dataset defaults : vizier.config.base.ConfigObject Web service default values urls: vizier.api.routes.base.UrlFactory Factory for resource urls offset: int, optional Number of rows at the beginning of the list that are skipped. limit: int, optional Limits the number of rows that are returned. Returns ------- dict """ # Use the dataset descriptor as the base obj = DATASET_DESCRIPTOR(dataset=dataset, project=project, urls=urls) # Serialize rows. The default dictionary representation for a row does # not include the row index position nor the annotation information. serialized_rows = list() for row in rows: serialized_rows.append(DATASET_ROW(row)) # Serialize the dataset schema and cells obj[labels.ROWS] = serialized_rows obj[labels.ROWCOUNT] = dataset.row_count obj[labels.OFFSET] = offset obj[labels.PROPERTIES] = dataset.get_properties() # Add pagination references links = obj[labels.LINKS] # Max. number of records shown if not limit is None and int(limit) >= 0: max_rows_per_request = int(limit) elif defaults.row_limit >= 0: max_rows_per_request = defaults.row_limit elif defaults.max_row_limit >= 0: max_rows_per_request = defaults.max_row_limit else: max_rows_per_request = -1 # List of pagination Urls # FIRST: Always include Url's to access the first page project_id = project.identifier dataset_id = dataset.identifier links.extend( serialize.HATEOAS({ ref.PAGE_FIRST: urls.dataset_pagination(project_id=project_id, dataset_id=dataset_id, offset=offset, limit=limit) })) # PREV: If offset is greater than zero allow to fetch previous page if not offset is None and offset > 0: if max_rows_per_request >= 0: if offset > 0: prev_offset = max(offset - max_rows_per_request, 0) links.extend( serialize.HATEOAS({ ref.PAGE_PREV: urls.dataset_pagination(project_id=project_id, dataset_id=dataset_id, offset=prev_offset, limit=limit) })) # NEXT & LAST: If there are rows beyond the current offset+limit include # Url's to fetch next page and last page. if offset < dataset.row_count and max_rows_per_request >= 0: next_offset = offset + max_rows_per_request if next_offset < dataset.row_count: links.extend( serialize.HATEOAS({ ref.PAGE_NEXT: urls.dataset_pagination(project_id=project_id, dataset_id=dataset_id, offset=next_offset, limit=limit) })) last_offset = (dataset.row_count - max_rows_per_request) if last_offset > offset: links.extend( serialize.HATEOAS({ ref.PAGE_LAST: urls.dataset_pagination(project_id=project_id, dataset_id=dataset_id, offset=last_offset, limit=limit) })) # Return pagination Url list return obj