def api_client(): """Obtain an API client to request the API of this DSS instance""" import dataikuapi backend_url = intercom.get_backend_url() location_data = intercom.get_location_data() if location_data["auth_mode"] == "API_KEY": return dataikuapi.DSSClient(backend_url, api_key=location_data["api_key"]) else: return dataikuapi.DSSClient(backend_url, internal_ticket=location_data["api_ticket"])
def __init__(self, gds_name, devops_team, apikey): #client object for this class to use self.__secondary_key = 'ouMfCYAS2Nbgq4BUUXOt3ICL3Q78wIlZ' assert(isinstance(apikey, str)), "Ensure your API key is formatted as string." self.__client = dataikuapi.DSSClient(host='http://10.189.40.171:8443', api_key=apikey) #Validate user is allowed to do this self.__verification_client = dataikuapi.DSSClient(host='http://10.189.40.171:8443', api_key=self.__secondary_key) self.gds_name, self.devops_team = ProjectValidator(self.__verification_client, gds_name, devops_team)
def _retrieve_data(self, data_source: DataikuDataSource) -> pd.DataFrame: client = dataikuapi.DSSClient(self.host, self.apiKey) data_url = f'/projects/{self.project}/datasets/{data_source.dataset}/data/' stream = client._perform_raw('GET', data_url, params={'format': 'tsv-excel-header'}) return pd.read_csv(StringIO(stream.text), sep='\t')
def dss_clients(request): """ The client fixture that is used by each of the test that will target a DSS instance. The scope of that fixture is set to module, so upon exiting a test module the fixture is destroyed Args: request: A pytest obejct allowing to introspect the test context. It allows us to access the value of host set in `pytest_generate_tests` Returns: dssclient: return a instance of a DSS client. It will be the same reference for each test withing the associated context. """ dss_clients = {} current_run_config = ScenarioConfiguration() excluded_targets = [] if request.config.getoption("--exclude-dss-targets"): excluded_targets = request.config.getoption("--exclude-dss-targets") excluded_targets = excluded_targets.split(",") logger.info( "Instanciating all the DSS clients for each user and DSS instance") for host in current_run_config.hosts: target = host["target"] if target not in excluded_targets: dss_clients.update({target: {}}) url = host["url"] for user, api_key in host["users"].items(): dss_clients[target].update( {user: dataikuapi.DSSClient(url, api_key=api_key)}) return dss_clients
def run(self, progress_callback): # You should really comment your code better # get remote key and url remote_url = self.config.get('remote_url') remote_key = self.config.get('remote_api') project_key = self.project_key # get local and remote wikis for current project rp = dataikuapi.DSSClient(remote_url, api_key=remote_key).get_project(project_key) cp = dataiku.api_client().get_project(project_key) local_wiki = cp.get_wiki() remote_wiki = rp.get_wiki() # replace or create new articles in the project wikis for l_article in local_wiki.list_articles(): for r_article in remote_wiki.list_articles(): if l_article.article_id == r_article.article_id: r_article.delete() remote_wiki.create_article(l_article.article_id, content=l_article.get_data().get_body()) return '<body>Wiki Updated on instance running at: ' + remote_url + '</body>'
def get_dataset(project, dataset_name): host = 'https://dsproj1.dataiku.com' with open('credentials.json', 'r') as f: credentials = json.load(f) client = dataikuapi.DSSClient(host, credentials['secret']) project = client.get_project(project) dataset = project.get_dataset(dataset_name) return dataset
def test_run_scenario(params, scenario_id): print("*************************") print("Executing scenario ", scenario_id) client = dataikuapi.DSSClient(params["host"], params["api"]) project = client.get_project(params["project"]) scenario_result = project.get_scenario(scenario_id).run_and_wait() print("Scenario info: ", scenario_result.get_info()) print("Scenario duration: ", scenario_result.get_duration()) print(scenario_result.get_details()["scenarioRun"]["result"]) print(scenario_result.get_details()["scenarioRun"]["result"]["outcome"]) assert scenario_result.get_details( )["scenarioRun"]["result"]["outcome"] == "SUCCESS"
def pytest_generate_tests(metafunc): if "scenario_id" in metafunc.fixturenames: p_host = metafunc.config.getoption('--host') p_api = metafunc.config.getoption('--api') p_project = metafunc.config.getoption('--project') client = dataikuapi.DSSClient(p_host, p_api) project = client.get_project(p_project) list_scenarios = [] for scenario in project.list_scenarios(): if scenario["id"].startswith("TEST_"): print("Adding scenario to test :", scenario["id"]) list_scenarios.append(scenario["id"]) metafunc.parametrize("scenario_id", list_scenarios)
def test_scenario(params): client = dataikuapi.DSSClient(params["host"], params["api"]) project = client.get_project(params["project"]) # Check that there is at least one scenario TEST_XXXXX & one TEST_SMOKE scenarios = project.list_scenarios() test_scenario = False smoketest_scenario = False for scenario in scenarios: if scenario["id"].startswith("TEST"): test_scenario = True if scenario["id"] == "TEST_SMOKE": smoketest_scenario = True assert test_scenario, "You need at least one test scenario (name starts with 'TEST_')" assert smoketest_scenario, "You need at least one smoke test scenario (name 'TEST_SMOKE')"
def test_coding_recipes_complexity(params): client = dataikuapi.DSSClient(params["host"], params["api"]) project = client.get_project(params["project"]) recipes = project.list_recipes() for recipe in recipes: if recipe["type"] == "python": print(recipe) payload = project.get_recipe( recipe["name"]).get_definition_and_payload().get_payload() code_analysis = cc_raw.analyze(payload) print(code_analysis) assert code_analysis.loc < 100 assert code_analysis.lloc < 50 v = cc_visitors.ComplexityVisitor.from_code(payload) assert v.complexity < 21, "Code complexity of recipe " + recipe[ "name"] + " is too complex: " + v.complexity + " > max value (21)"
def run(self, progress_callback): """ Do stuff here. Can return a string or raise an exception. The progress_callback is a function expecting 1 value: current progress """ #Initialise local project project = self.client.get_project(self.project_key) export_options = self.config.get('Export_Options') #Initialise remote client remote_host = self.config.get('Target_Instance') rapi_key = self.config.get('Target_apikey') remote_client = dataikuapi.DSSClient(remote_host, rapi_key) dict = { 'exportAnalysisModels': ('true' if 'export_analysis_models' in export_options else 'false'), 'exportSavedModels': ('true' if 'export_saved_models' in export_options else 'false'), 'exportAllInputDatasets': ('true' if 'export_input_datasets' in export_options else 'false'), 'exportUploads': ('true' if 'export_uploads' in export_options else 'false'), 'exportAllDatasets': ('true' if 'export_all_datasets' in export_options else 'false'), 'exportManagedFS': ('true' if 'export_managed_fs' in export_options else 'false'), 'exportManagedFolders': ('true' if 'export_managed_folders' in export_options else 'false'), 'exportAllInputManagedFolders': ('true' if 'export_all_input_managed_folders' in export_options else 'false') } #try: with project.get_export_stream(dict) as s: handle = remote_client.prepare_project_import(s) handle.execute() return ("Project Successfully exported to target instance")
try: param_val = user[param] if param != 'groups': param_val = param_val.lower() except: param_val = '' return param_val # instantiate lists now = datetime.datetime.now() prof_limit_list = [] df_data = [] for url, api_key in api_url_dict.iteritems(): client = dataikuapi.DSSClient(url, api_key) if ignore_ssl_certs: client._session.verify = False # get licensing status licensing_status = client.get_licensing_status() license_id = licensing_status['base']['licenseContent']['licenseId'] # add to prof limit dict prof_limits = licensing_status['limits']['profileLimits'] for user_prof, vals in prof_limits.iteritems(): limit = prof_limits[user_prof]['licensed']['licensedLimit'] prof_limit_list.append([license_id, user_prof, limit]) # get user-specific info for user in client.list_users():
import dataikuapi import sys host = sys.argv[1] apiKey = sys.argv[2] api_service_id = sys.argv[4] api_package_id = sys.argv[5] infra_dev_id = sys.argv[6] client = dataikuapi.DSSClient(host, apiKey) #################### # Retrieve the service object from API Deployer api_deployer = client.get_apideployer() deployer_service = "" for serv in api_deployer.list_services(): if serv.id() == api_service_id: print("Found existing Deployer API service (" + api_service_id + ")") deployer_service = serv if deployer_service == "": print("Cannot find the required API service named '{}'".format(api_service_id)) sys.exit(1) #################### # Find if there is an existing deployment to update or a new to create print("Looking for existing deployments of '" + deployer_service.id() + "' on infrastructure '" + infra_dev_id + "'") serv_status = deployer_service.get_status() # DSSAPIDeployerServiceStatus dep_to_update = "" for dep in serv_status.get_raw()['deployments']:
def run(self, progress_callback): # verify inputs bundle_id = self.config.get("bundle_id", "") if bundle_id == "": raise Exception("bundle_id is required") remote_host = self.config.get("remote_host", "") if remote_host == "": raise Exception("destination is required") api_key = self.config.get("api_key", "") if api_key == "": raise Exception("API key is required") activate_scenarios = self.config.get("activate_scenarios") # get client and connect to project client = dataiku.api_client() project = client.get_project(self.project_key) # use public python api to get access to remote host remote_client = dataikuapi.DSSClient(remote_host, api_key) # ignore SSL Certificates if selected if self.config.get("ignore_ssl_certs"): remote_client._session.verify = False html = '<div> Successfully connected to remote host: %s</div>' % ( remote_client.host) # get list of connections used in the initial project datasets = project.list_datasets() connections_used = [] for dataset in datasets: try: connections_used.append(dataset['params']['connection']) except: continue connections_used = list(set(connections_used)) # get list of connections in remote project remote_connections_list = remote_client.list_connections() remote_connections_names = remote_connections_list.keys() # check if connections used in the initial project also exist on the remote instance for connection in connections_used: if connection not in remote_connections_names: error_msg = 'Failed - Connection %s used in initial project does not exist on instance %s.' % ( connection, remote_host) raise Exception(error_msg) else: continue html += '<div><div>All connections exist on both instances</div>' # check if any plugins installed in the original instance are not installed in remote instance # get list of plugins in original instance original_plugin_names = {} for plugin in client.list_plugins(): original_plugin_names[plugin['meta']['label']] = plugin['version'] # compare to list of plugins in remote instance remote_plugin_names = {} for plugin in remote_client.list_plugins(): remote_plugin_names[plugin['meta']['label']] = plugin['version'] missing_plugins = { k: original_plugin_names[k] for k in original_plugin_names if k not in remote_plugin_names or original_plugin_names[k] == remote_plugin_names[k] } if len(missing_plugins) > 0: html += '<div> <b> Warning: the following plugins (and versions) are installed on this instance, but not on the remote instance. Please ensure that your project does not use these plugins before proceeding. </b> </div>' html += '<table>' html += '<tr>' html += '<th>Plugin</th>' html += '<th>Version</th>' html += '</tr>' for plugin in missing_plugins: html += '<tr>' html += '<td> %s </td>' % (plugin) html += '<td> %s </td>' % (missing_plugins[plugin]) html += '</tr>' if len(missing_plugins) > 0: html += '</table>' ''' TRIGGER THE BELOW SCENARIO CODE IF CHECKBOX CHECKED ''' # get a list of active scenario ids on the project if activate_scenarios: project_active_scenarios = [] for scenario in project.list_scenarios(): s = project.get_scenario(scenario['id']) scenario_def = s.get_definition() if scenario_def['active']: project_active_scenarios.append(scenario_def['id']) # create the bundle (verify that the bundle_id does not exist) try: project.export_bundle(bundle_id) html += '<div><div>Successfully created bundle: %s</div>' % ( bundle_id) except DataikuException as de: error_msg = 'Failed - Bundle %s already exists for project %s' % ( bundle_id, self.project_key) raise Exception(error_msg) # check if there are any projects in remote instance try: remote_projects = remote_client.list_project_keys() except: remote_projects = [] if not self.project_key in remote_projects: remote_client.create_project(self.project_key, self.project_key, 'admin', 'placeholder description') html += '<div> Project %s does not exist on instance %s. Creating it.</div>' % ( self.project_key, remote_host) else: html += '<div> Project %s already exists on instance %s. Updating with new bundle version %s.</div>' % ( self.project_key, remote_host, bundle_id) # connect to remote project remote_project = remote_client.get_project(self.project_key) with project.get_exported_bundle_archive_stream(bundle_id) as fp: remote_project.import_bundle_from_stream(fp.content) # "preload bundle" to create/update custom code environments used throughout the project preload = remote_project.preload_bundle(bundle_id) # activate bundle remote_project.activate_bundle(bundle_id) html += '<div> Bundle activated </div>' ''' TRIGGER THE BELOW SCENARIO CODE IF CHECKBOX CHECKED ''' # activate scenarios that were active on design instance if activate_scenarios: for active_scenario_id in project_active_scenarios: scenario = remote_project.get_scenario(active_scenario_id) scenario_def = scenario.get_definition() scenario_def['active'] = True scenario.set_definition(scenario_def) # check if remote instance is a design instance, and create a bundle on it if so remote_design_instance = self.config.get("remote_design_instance") if remote_design_instance: create_bundle_on_remote_design_instance = self.config.get( "create_bundle_on_remote_design_instance") if create_bundle_on_remote_design_instance: remote_project.export_bundle(bundle_id) html += '<div> Bundle Created on Remote Design Instance </div>' html += '</div>' return html
def build_apinode_client(params): client_design = dataikuapi.DSSClient(params["host"], params["api"]) api_deployer = client_design.get_apideployer() api_url = api_deployer.get_infra(params["api_dev_infra_id"]).get_settings( ).get_raw()['apiNodes'][0]['url'] return dataikuapi.APINodeClient(api_url, params["api_service_id"])
import sys pdpl_host = sys.argv[1] pdpl_apiKey = sys.argv[2] project = sys.argv[3] bundle_id = sys.argv[4] infra = sys.argv[5] auto_host = sys.argv[6] auto_apiKey = sys.argv[7] previous_bundle_id = "" failed_deployment = False # Deploy the new bundle print("Searching for existing deployment of '{}' on infra '{}'".format( project , infra)) pdpl_client = dataikuapi.DSSClient(pdpl_host, pdpl_apiKey) pdpl = pdpl_client.get_projectdeployer() pdpl_proj = pdpl.get_project(project) deployments = pdpl_proj.get_status().get_deployments(infra) if deployments : #update deployment = deployments[0] print("Using existing deployment '{}'".format(deployment.id)) depl_settings = deployment.get_settings() previous_bundle_id = depl_settings.get_raw()['bundleId'] depl_settings.get_raw()['bundleId'] = bundle_id depl_settings.save() else : #create print("Need to create a new deployment (no rollback possible)")