def api_client():
"""Obtain an API client to request the API of this DSS instance"""
import dataikuapi
backend_url = intercom.get_backend_url()
location_data = intercom.get_location_data()
if location_data["auth_mode"] == "API_KEY":
return dataikuapi.DSSClient(backend_url, api_key=location_data["api_key"])
else:
return dataikuapi.DSSClient(backend_url, internal_ticket=location_data["api_ticket"])
def __init__(self, gds_name, devops_team, apikey):
#client object for this class to use
self.__secondary_key = 'ouMfCYAS2Nbgq4BUUXOt3ICL3Q78wIlZ'
assert(isinstance(apikey, str)), "Ensure your API key is formatted as string."
self.__client = dataikuapi.DSSClient(host='http://10.189.40.171:8443', api_key=apikey)
#Validate user is allowed to do this
self.__verification_client = dataikuapi.DSSClient(host='http://10.189.40.171:8443', api_key=self.__secondary_key)
self.gds_name, self.devops_team = ProjectValidator(self.__verification_client, gds_name, devops_team)
def _retrieve_data(self, data_source: DataikuDataSource) -> pd.DataFrame:
client = dataikuapi.DSSClient(self.host, self.apiKey)
data_url = f'/projects/{self.project}/datasets/{data_source.dataset}/data/'
stream = client._perform_raw('GET',
data_url,
params={'format': 'tsv-excel-header'})
return pd.read_csv(StringIO(stream.text), sep='\t')
def dss_clients(request):
"""
The client fixture that is used by each of the test that will target a DSS instance.
The scope of that fixture is set to module, so upon exiting a test module the fixture is destroyed
Args:
request: A pytest obejct allowing to introspect the test context. It allows us to access
the value of host set in `pytest_generate_tests`
Returns:
dssclient: return a instance of a DSS client. It will be the same reference for each test withing the associated context.
"""
dss_clients = {}
current_run_config = ScenarioConfiguration()
excluded_targets = []
if request.config.getoption("--exclude-dss-targets"):
excluded_targets = request.config.getoption("--exclude-dss-targets")
excluded_targets = excluded_targets.split(",")
logger.info(
"Instanciating all the DSS clients for each user and DSS instance")
for host in current_run_config.hosts:
target = host["target"]
if target not in excluded_targets:
dss_clients.update({target: {}})
url = host["url"]
for user, api_key in host["users"].items():
dss_clients[target].update(
{user: dataikuapi.DSSClient(url, api_key=api_key)})
return dss_clients
def run(self, progress_callback):
# You should really comment your code better
# get remote key and url
remote_url = self.config.get('remote_url')
remote_key = self.config.get('remote_api')
project_key = self.project_key
# get local and remote wikis for current project
rp = dataikuapi.DSSClient(remote_url,
api_key=remote_key).get_project(project_key)
cp = dataiku.api_client().get_project(project_key)
local_wiki = cp.get_wiki()
remote_wiki = rp.get_wiki()
# replace or create new articles in the project wikis
for l_article in local_wiki.list_articles():
for r_article in remote_wiki.list_articles():
if l_article.article_id == r_article.article_id:
r_article.delete()
remote_wiki.create_article(l_article.article_id,
content=l_article.get_data().get_body())
return '<body>Wiki Updated on instance running at: ' + remote_url + '</body>'
def get_dataset(project, dataset_name):
host = 'https://dsproj1.dataiku.com'
with open('credentials.json', 'r') as f:
credentials = json.load(f)
client = dataikuapi.DSSClient(host, credentials['secret'])
project = client.get_project(project)
dataset = project.get_dataset(dataset_name)
return dataset
def test_run_scenario(params, scenario_id):
print("*************************")
print("Executing scenario ", scenario_id)
client = dataikuapi.DSSClient(params["host"], params["api"])
project = client.get_project(params["project"])
scenario_result = project.get_scenario(scenario_id).run_and_wait()
print("Scenario info: ", scenario_result.get_info())
print("Scenario duration: ", scenario_result.get_duration())
print(scenario_result.get_details()["scenarioRun"]["result"])
print(scenario_result.get_details()["scenarioRun"]["result"]["outcome"])
assert scenario_result.get_details(
)["scenarioRun"]["result"]["outcome"] == "SUCCESS"
def pytest_generate_tests(metafunc):
if "scenario_id" in metafunc.fixturenames:
p_host = metafunc.config.getoption('--host')
p_api = metafunc.config.getoption('--api')
p_project = metafunc.config.getoption('--project')
client = dataikuapi.DSSClient(p_host, p_api)
project = client.get_project(p_project)
list_scenarios = []
for scenario in project.list_scenarios():
if scenario["id"].startswith("TEST_"):
print("Adding scenario to test :", scenario["id"])
list_scenarios.append(scenario["id"])
metafunc.parametrize("scenario_id", list_scenarios)
def test_scenario(params):
client = dataikuapi.DSSClient(params["host"], params["api"])
project = client.get_project(params["project"])
# Check that there is at least one scenario TEST_XXXXX & one TEST_SMOKE
scenarios = project.list_scenarios()
test_scenario = False
smoketest_scenario = False
for scenario in scenarios:
if scenario["id"].startswith("TEST"):
test_scenario = True
if scenario["id"] == "TEST_SMOKE":
smoketest_scenario = True
assert test_scenario, "You need at least one test scenario (name starts with 'TEST_')"
assert smoketest_scenario, "You need at least one smoke test scenario (name 'TEST_SMOKE')"
def test_coding_recipes_complexity(params):
client = dataikuapi.DSSClient(params["host"], params["api"])
project = client.get_project(params["project"])
recipes = project.list_recipes()
for recipe in recipes:
if recipe["type"] == "python":
print(recipe)
payload = project.get_recipe(
recipe["name"]).get_definition_and_payload().get_payload()
code_analysis = cc_raw.analyze(payload)
print(code_analysis)
assert code_analysis.loc < 100
assert code_analysis.lloc < 50
v = cc_visitors.ComplexityVisitor.from_code(payload)
assert v.complexity < 21, "Code complexity of recipe " + recipe[
"name"] + " is too complex: " + v.complexity + " > max value (21)"
def run(self, progress_callback):
"""
Do stuff here. Can return a string or raise an exception.
The progress_callback is a function expecting 1 value: current progress
"""
#Initialise local project
project = self.client.get_project(self.project_key)
export_options = self.config.get('Export_Options')
#Initialise remote client
remote_host = self.config.get('Target_Instance')
rapi_key = self.config.get('Target_apikey')
remote_client = dataikuapi.DSSClient(remote_host, rapi_key)
dict = {
'exportAnalysisModels': ('true' if 'export_analysis_models'
in export_options else 'false'),
'exportSavedModels':
('true' if 'export_saved_models' in export_options else 'false'),
'exportAllInputDatasets':
('true' if 'export_input_datasets' in export_options else 'false'),
'exportUploads':
('true' if 'export_uploads' in export_options else 'false'),
'exportAllDatasets':
('true' if 'export_all_datasets' in export_options else 'false'),
'exportManagedFS':
('true' if 'export_managed_fs' in export_options else 'false'),
'exportManagedFolders': ('true' if 'export_managed_folders'
in export_options else 'false'),
'exportAllInputManagedFolders':
('true' if 'export_all_input_managed_folders' in export_options
else 'false')
}
#try:
with project.get_export_stream(dict) as s:
handle = remote_client.prepare_project_import(s)
handle.execute()
return ("Project Successfully exported to target instance")
try:
param_val = user[param]
if param != 'groups':
param_val = param_val.lower()
except:
param_val = ''
return param_val
# instantiate lists
now = datetime.datetime.now()
prof_limit_list = []
df_data = []
for url, api_key in api_url_dict.iteritems():
client = dataikuapi.DSSClient(url, api_key)
if ignore_ssl_certs:
client._session.verify = False
# get licensing status
licensing_status = client.get_licensing_status()
license_id = licensing_status['base']['licenseContent']['licenseId']
# add to prof limit dict
prof_limits = licensing_status['limits']['profileLimits']
for user_prof, vals in prof_limits.iteritems():
limit = prof_limits[user_prof]['licensed']['licensedLimit']
prof_limit_list.append([license_id, user_prof, limit])
# get user-specific info
for user in client.list_users():
import dataikuapi
import sys
host = sys.argv[1]
apiKey = sys.argv[2]
api_service_id = sys.argv[4]
api_package_id = sys.argv[5]
infra_dev_id = sys.argv[6]
client = dataikuapi.DSSClient(host, apiKey)
####################
# Retrieve the service object from API Deployer
api_deployer = client.get_apideployer()
deployer_service = ""
for serv in api_deployer.list_services():
if serv.id() == api_service_id:
print("Found existing Deployer API service (" + api_service_id + ")")
deployer_service = serv
if deployer_service == "":
print("Cannot find the required API service named '{}'".format(api_service_id))
sys.exit(1)
####################
# Find if there is an existing deployment to update or a new to create
print("Looking for existing deployments of '" + deployer_service.id() + "' on infrastructure '" + infra_dev_id + "'")
serv_status = deployer_service.get_status() # DSSAPIDeployerServiceStatus
dep_to_update = ""
for dep in serv_status.get_raw()['deployments']:
def run(self, progress_callback):
# verify inputs
bundle_id = self.config.get("bundle_id", "")
if bundle_id == "":
raise Exception("bundle_id is required")
remote_host = self.config.get("remote_host", "")
if remote_host == "":
raise Exception("destination is required")
api_key = self.config.get("api_key", "")
if api_key == "":
raise Exception("API key is required")
activate_scenarios = self.config.get("activate_scenarios")
# get client and connect to project
client = dataiku.api_client()
project = client.get_project(self.project_key)
# use public python api to get access to remote host
remote_client = dataikuapi.DSSClient(remote_host, api_key)
# ignore SSL Certificates if selected
if self.config.get("ignore_ssl_certs"):
remote_client._session.verify = False
html = '<div> Successfully connected to remote host: %s</div>' % (
remote_client.host)
# get list of connections used in the initial project
datasets = project.list_datasets()
connections_used = []
for dataset in datasets:
try:
connections_used.append(dataset['params']['connection'])
except:
continue
connections_used = list(set(connections_used))
# get list of connections in remote project
remote_connections_list = remote_client.list_connections()
remote_connections_names = remote_connections_list.keys()
# check if connections used in the initial project also exist on the remote instance
for connection in connections_used:
if connection not in remote_connections_names:
error_msg = 'Failed - Connection %s used in initial project does not exist on instance %s.' % (
connection, remote_host)
raise Exception(error_msg)
else:
continue
html += '<div><div>All connections exist on both instances</div>'
# check if any plugins installed in the original instance are not installed in remote instance
# get list of plugins in original instance
original_plugin_names = {}
for plugin in client.list_plugins():
original_plugin_names[plugin['meta']['label']] = plugin['version']
# compare to list of plugins in remote instance
remote_plugin_names = {}
for plugin in remote_client.list_plugins():
remote_plugin_names[plugin['meta']['label']] = plugin['version']
missing_plugins = {
k: original_plugin_names[k]
for k in original_plugin_names if k not in remote_plugin_names
or original_plugin_names[k] == remote_plugin_names[k]
}
if len(missing_plugins) > 0:
html += '<div> <b> Warning: the following plugins (and versions) are installed on this instance, but not on the remote instance. Please ensure that your project does not use these plugins before proceeding. </b> </div>'
html += '<table>'
html += '<tr>'
html += '<th>Plugin</th>'
html += '<th>Version</th>'
html += '</tr>'
for plugin in missing_plugins:
html += '<tr>'
html += '<td> %s </td>' % (plugin)
html += '<td> %s </td>' % (missing_plugins[plugin])
html += '</tr>'
if len(missing_plugins) > 0:
html += '</table>'
''' TRIGGER THE BELOW SCENARIO CODE IF CHECKBOX CHECKED '''
# get a list of active scenario ids on the project
if activate_scenarios:
project_active_scenarios = []
for scenario in project.list_scenarios():
s = project.get_scenario(scenario['id'])
scenario_def = s.get_definition()
if scenario_def['active']:
project_active_scenarios.append(scenario_def['id'])
# create the bundle (verify that the bundle_id does not exist)
try:
project.export_bundle(bundle_id)
html += '<div><div>Successfully created bundle: %s</div>' % (
bundle_id)
except DataikuException as de:
error_msg = 'Failed - Bundle %s already exists for project %s' % (
bundle_id, self.project_key)
raise Exception(error_msg)
# check if there are any projects in remote instance
try:
remote_projects = remote_client.list_project_keys()
except:
remote_projects = []
if not self.project_key in remote_projects:
remote_client.create_project(self.project_key, self.project_key,
'admin', 'placeholder description')
html += '<div> Project %s does not exist on instance %s. Creating it.</div>' % (
self.project_key, remote_host)
else:
html += '<div> Project %s already exists on instance %s. Updating with new bundle version %s.</div>' % (
self.project_key, remote_host, bundle_id)
# connect to remote project
remote_project = remote_client.get_project(self.project_key)
with project.get_exported_bundle_archive_stream(bundle_id) as fp:
remote_project.import_bundle_from_stream(fp.content)
# "preload bundle" to create/update custom code environments used throughout the project
preload = remote_project.preload_bundle(bundle_id)
# activate bundle
remote_project.activate_bundle(bundle_id)
html += '<div> Bundle activated </div>'
''' TRIGGER THE BELOW SCENARIO CODE IF CHECKBOX CHECKED '''
# activate scenarios that were active on design instance
if activate_scenarios:
for active_scenario_id in project_active_scenarios:
scenario = remote_project.get_scenario(active_scenario_id)
scenario_def = scenario.get_definition()
scenario_def['active'] = True
scenario.set_definition(scenario_def)
# check if remote instance is a design instance, and create a bundle on it if so
remote_design_instance = self.config.get("remote_design_instance")
if remote_design_instance:
create_bundle_on_remote_design_instance = self.config.get(
"create_bundle_on_remote_design_instance")
if create_bundle_on_remote_design_instance:
remote_project.export_bundle(bundle_id)
html += '<div> Bundle Created on Remote Design Instance </div>'
html += '</div>'
return html
def build_apinode_client(params):
client_design = dataikuapi.DSSClient(params["host"], params["api"])
api_deployer = client_design.get_apideployer()
api_url = api_deployer.get_infra(params["api_dev_infra_id"]).get_settings(
).get_raw()['apiNodes'][0]['url']
return dataikuapi.APINodeClient(api_url, params["api_service_id"])
import sys
pdpl_host = sys.argv[1]
pdpl_apiKey = sys.argv[2]
project = sys.argv[3]
bundle_id = sys.argv[4]
infra = sys.argv[5]
auto_host = sys.argv[6]
auto_apiKey = sys.argv[7]
previous_bundle_id = ""
failed_deployment = False
# Deploy the new bundle
print("Searching for existing deployment of '{}' on infra '{}'".format( project , infra))
pdpl_client = dataikuapi.DSSClient(pdpl_host, pdpl_apiKey)
pdpl = pdpl_client.get_projectdeployer()
pdpl_proj = pdpl.get_project(project)
deployments = pdpl_proj.get_status().get_deployments(infra)
if deployments :
#update
deployment = deployments[0]
print("Using existing deployment '{}'".format(deployment.id))
depl_settings = deployment.get_settings()
previous_bundle_id = depl_settings.get_raw()['bundleId']
depl_settings.get_raw()['bundleId'] = bundle_id
depl_settings.save()
else :
#create
print("Need to create a new deployment (no rollback possible)")
