def job_start_abort_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
count = len(client.get_project(testProjectKey).list_jobs())
job = project.start_job({
"initiator":
"me",
"name":
"some job",
"triggeredFrom":
"API",
"type":
"NON_RECURSIVE_FORCED_BUILD",
"outputs": [{
"type": "DATASET",
"projectKey": testProjectKey,
"id": testDataset,
"partition": "NP"
}]
})
job.abort()
eq_(count + 1, len(client.get_project(testProjectKey).list_jobs()))
def job_status_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
jobs = project.list_jobs()
for job in jobs:
status = project.get_job(job['def']['id']).get_status()
ok_(status is not None)
def job_log_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
jobs = project.list_jobs()
for job in jobs:
log = project.get_job(job['def']['id']).get_log()
ok_(log is not None)
def updload_replace_delete_test():
temp_folder = tempfile.mkdtemp()
stuff = osp.join(temp_folder, "test.txt")
stuffb = osp.join(temp_folder, "testb.txt")
with open(stuff, "w") as f:
f.write('some contents\n on several\nlines')
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
managedfolder = project.create_managed_folder("titi")
count = len(managedfolder.list_contents()['items'])
with open(stuff, "r") as f:
managedfolder.put_file('stuff', f)
eq_(count + 1, len(managedfolder.list_contents()['items']))
with open(stuffb, "w") as f:
with closing(managedfolder.get_file('stuff')) as s:
f.write(s.raw.read())
eq_(True, filecmp.cmp(stuff, stuffb))
managedfolder.delete_file('stuff')
eq_(count, len(managedfolder.list_contents()['items']))
managedfolder.delete()
def delete_contents_outside_test():
temp_folder = tempfile.mkdtemp()
file_content = 'some contents\n on several\nlines'
stuff = osp.join(temp_folder, "test.txt")
with open(stuff, "w") as f:
f.write(file_content)
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
managedfolder = project.create_managed_folder("titi")
count = len(managedfolder.list_contents()['items'])
with open(stuff, "r") as f:
managedfolder.put_file('stuff', f)
eq_(count + 1, len(managedfolder.list_contents()['items']))
with closing(managedfolder.get_file('stuff')) as s:
c = s.raw.read()
eq_(file_content, c)
try:
# climb the file hierarchy up to the dip_home
managedfolder.delete_file('../../../shared-secret.txt')
raise AssertionError('Access outside folder should fail')
finally:
managedfolder.delete()
def create_delete_connection_test():
client = DSSClient(host, apiKey)
count = len(client.list_connections())
connection = client.create_connection("toto", "HDFS")
eq_(count + 1, len(client.list_connections()))
connection.delete()
eq_(count, len(client.list_connections()))
def create_delete_group_test():
client = DSSClient(host, apiKey)
count = len(client.list_groups())
group = client.create_group("toto")
eq_(count + 1, len(client.list_groups()))
group.delete()
eq_(count, len(client.list_groups()))
def project_create_delete_test():
client = DSSClient(host, apiKey)
count = len(client.list_project_keys())
p = client.create_project("toto", "name of toto", "me")
eq_(count + 1, len(client.list_project_keys()))
p.delete()
eq_(count, len(client.list_project_keys()))
def get_client_as(self):
"""
Gets a :class:`dataikuapi.DSSClient` that has the permissions of this user.
This allows administrators to impersonate actions on behalf of other users, in order to perform
actions on their behalf
"""
from dataikuapi.dssclient import DSSClient
if self.client.api_key is not None:
return DSSClient(self.client.host,
self.client.api_key,
extra_headers={"X-DKU-ProxyUser": self.login})
elif self.client.internal_ticket is not None:
return DSSClient(self.client.host,
internal_ticket=self.client.internal_ticket,
extra_headers={"X-DKU-ProxyUser": self.login})
else:
raise ValueError("Don't know how to proxy this client")
def dataset_data_test():
client = DSSClient(host, apiKey)
p = client.get_project(testProjectKey)
d = p.get_dataset(testDataset)
counter = 0
for r in d.iter_rows():
counter = counter + 1
if counter > 5:
break
eq_(6, counter)
def create_delete_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
count = len(project.list_managed_folders())
managedfolder = project.create_managed_folder("titi")
eq_(count + 1, len(project.list_managed_folders()))
managedfolder.delete()
eq_(count, len(project.list_managed_folders()))
def create_delete_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
count = len(project.list_datasets())
dataset = project.create_dataset("titi", "Filesystem")
eq_(count + 1, len(project.list_datasets()))
dataset.delete()
eq_(count, len(project.list_datasets()))
def create_delete_user_test():
client = DSSClient(host, apiKey)
count = len(client.list_users())
user = client.create_user("toto",
"password",
"display name of toto",
groups=['a', 'b'])
eq_(count + 1, len(client.list_users()))
user.delete()
eq_(count, len(client.list_users()))
def get_set_group_test():
client = DSSClient(host, apiKey)
group = client.create_group("toto")
desc = group.get_definition()
desc['description'] = 'here'
group.set_definition(desc)
desc2 = group.get_definition()
eq_('here', desc2['description'])
group.delete()
def get_set_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
managedfolder = project.create_managed_folder("titi")
definition = managedfolder.get_definition()
definition['description'] = 'describe me!'
managedfolder.set_definition(definition)
definition2 = managedfolder.get_definition()
eq_('describe me!', definition2['description'])
managedfolder.delete()
def get_set_connection_test():
client = DSSClient(host, apiKey)
connection = client.create_connection("toto", "HDFS")
desc = connection.get_definition()
desc['usableBy'] = 'ALLOWED'
desc['allowedGroups'] = ['a', 'b']
connection.set_definition(desc)
desc2 = connection.get_definition()
eq_('ALLOWED', desc2['usableBy'])
connection.delete()
def get_set_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
dataset = project.create_dataset("titi", "Filesystem")
definition = dataset.get_definition()
definition['managed'] = True
dataset.set_definition(definition)
definition2 = dataset.get_definition()
eq_(True, definition2['managed'])
dataset.delete()
def get_set_user_test():
client = DSSClient(host, apiKey)
user = client.create_user("toto",
"password",
"display name of toto",
groups=['a', 'b'])
desc = user.get_definition()
desc['displayName'] = 'tata'
user.set_definition(desc)
desc2 = user.get_definition()
eq_('tata', desc2['displayName'])
user.delete()
def project_permissions_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
perms = project.get_permissions()
ok_(perms is not None)
project.set_permissions(perms)
def list_groups_test():
client = DSSClient(host, apiKey)
groups = client.list_groups()
ok_(len(groups) > 0)
def list_connections_test():
client = DSSClient(host, apiKey)
connections = client.list_connections()
ok_(len(connections) > 0)
def list_jobs_test():
client = DSSClient(host, apiKey)
jobs = client.get_project(testProjectKey).list_jobs()
ok_(len(jobs) > 0)
def project_metadata_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
meta = project.get_metadata()
ok_(meta is not None)
project.set_metadata(meta)
def clear_partitions_test():
client = DSSClient(host, apiKey)
dataset = client.get_project(testPartitionedProjectKey).get_dataset(
testDropPartitionedDataset)
dataset.clear(json.dumps([testDropPartition]))
def list_partitions_test():
client = DSSClient(host, apiKey)
dataset = client.get_project(testPartitionedProjectKey).get_dataset(
testPartitionedDataset)
ok_(len(dataset.list_partitions()) > 0)
def clear_test():
client = DSSClient(host, apiKey)
dataset = client.get_project(testProjectKey).get_dataset(testClearDataset)
dataset.clear()
def list_projects_test():
client = DSSClient(host, apiKey)
projects = client.list_project_keys()
ok_(len(projects) > 0)
def list_datasets_test():
client = DSSClient(host, apiKey)
datasets = client.get_project(testProjectKey).list_datasets()
ok_(len(datasets) > 0)
def list_users_test():
client = DSSClient(host, apiKey)
users = client.list_users()
ok_(len(users) > 0)
def sync_metastore_test():
client = DSSClient(host, apiKey)
dataset = client.get_project(testProjectKey).get_dataset(testHiveDataset)
dataset.synchronize_hive_metastore()