def job_status_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
jobs = project.list_jobs()
for job in jobs:
status = project.get_job(job['def']['id']).get_status()
ok_(status is not None)
def job_log_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
jobs = project.list_jobs()
for job in jobs:
log = project.get_job(job['def']['id']).get_log()
ok_(log is not None)
def updload_replace_delete_test():
temp_folder = tempfile.mkdtemp()
stuff = osp.join(temp_folder, "test.txt")
stuffb = osp.join(temp_folder, "testb.txt")
with open(stuff, "w") as f:
f.write('some contents\n on several\nlines')
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
managedfolder = project.create_managed_folder("titi")
count = len(managedfolder.list_contents()['items'])
with open(stuff, "r") as f:
managedfolder.put_file('stuff', f)
eq_(count + 1, len(managedfolder.list_contents()['items']))
with open(stuffb, "w") as f:
with closing(managedfolder.get_file('stuff')) as s:
f.write(s.raw.read())
eq_(True, filecmp.cmp(stuff, stuffb))
managedfolder.delete_file('stuff')
eq_(count, len(managedfolder.list_contents()['items']))
managedfolder.delete()
def delete_contents_outside_test():
temp_folder = tempfile.mkdtemp()
file_content = 'some contents\n on several\nlines'
stuff = osp.join(temp_folder, "test.txt")
with open(stuff, "w") as f:
f.write(file_content)
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
managedfolder = project.create_managed_folder("titi")
count = len(managedfolder.list_contents()['items'])
with open(stuff, "r") as f:
managedfolder.put_file('stuff', f)
eq_(count + 1, len(managedfolder.list_contents()['items']))
with closing(managedfolder.get_file('stuff')) as s:
c = s.raw.read()
eq_(file_content, c)
try:
# climb the file hierarchy up to the dip_home
managedfolder.delete_file('../../../shared-secret.txt')
raise AssertionError('Access outside folder should fail')
finally:
managedfolder.delete()
def job_status_test(): client = DSSClient(host, apiKey) project = client.get_project(testProjectKey) jobs = project.list_jobs() for job in jobs: status = project.get_job(job['def']['id']).get_status() ok_(status is not None)
def job_log_test(): client = DSSClient(host, apiKey) project = client.get_project(testProjectKey) jobs = project.list_jobs() for job in jobs: log = project.get_job(job['def']['id']).get_log() ok_(log is not None)
def job_start_abort_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
count = len(client.get_project(testProjectKey).list_jobs())
job = project.start_job({
"initiator":
"me",
"name":
"some job",
"triggeredFrom":
"API",
"type":
"NON_RECURSIVE_FORCED_BUILD",
"outputs": [{
"type": "DATASET",
"projectKey": testProjectKey,
"id": testDataset,
"partition": "NP"
}]
})
job.abort()
eq_(count + 1, len(client.get_project(testProjectKey).list_jobs()))
def project_create_delete_test():
client = DSSClient(host, apiKey)
count = len(client.list_project_keys())
p = client.create_project("toto","name of toto", "me")
eq_(count + 1, len(client.list_project_keys()))
p.delete()
eq_(count, len(client.list_project_keys()))
def create_delete_group_test():
client = DSSClient(host, apiKey)
count = len(client.list_groups())
group = client.create_group("toto")
eq_(count + 1, len(client.list_groups()))
group.delete()
eq_(count, len(client.list_groups()))
def create_delete_connection_test():
client = DSSClient(host, apiKey)
count = len(client.list_connections())
connection = client.create_connection("toto", "HDFS")
eq_(count + 1, len(client.list_connections()))
connection.delete()
eq_(count, len(client.list_connections()))
def create_delete_user_test():
client = DSSClient(host, apiKey)
count = len(client.list_users())
user = client.create_user("toto", "password", "display name of toto", groups=['a','b'])
eq_(count + 1, len(client.list_users()))
user.delete()
eq_(count, len(client.list_users()))
def create_delete_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
count = len(project.list_managed_folders())
managedfolder = project.create_managed_folder("titi")
eq_(count + 1, len(project.list_managed_folders()))
managedfolder.delete()
eq_(count, len(project.list_managed_folders()))
def create_delete_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
count = len(project.list_datasets())
dataset = project.create_dataset("titi", "Filesystem")
eq_(count + 1, len(project.list_datasets()))
dataset.delete()
eq_(count, len(project.list_datasets()))
def dataset_data_test(): client = DSSClient(host, apiKey) p = client.get_project(testProjectKey) d = p.get_dataset(testDataset) counter = 0 for r in d.iter_rows(): counter = counter + 1 if counter > 5: break eq_(6, counter)
def create_delete_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
count = len(project.list_datasets())
dataset = project.create_dataset("titi", "Filesystem")
eq_(count + 1, len(project.list_datasets()))
dataset.delete()
eq_(count, len(project.list_datasets()))
def dataset_data_test():
client = DSSClient(host, apiKey)
p = client.get_project(testProjectKey)
d = p.get_dataset(testDataset)
counter = 0
for r in d.iter_rows():
counter = counter + 1
if counter > 5:
break
eq_(6, counter)
def get_set_group_test():
client = DSSClient(host, apiKey)
group = client.create_group("toto")
desc = group.get_definition()
desc['description'] = 'here'
group.set_definition(desc)
desc2 = group.get_definition()
eq_('here', desc2['description'])
group.delete()
def get_set_group_test():
client = DSSClient(host, apiKey)
group = client.create_group("toto")
desc = group.get_definition()
desc['description'] = 'here'
group.set_definition(desc)
desc2 = group.get_definition()
eq_('here', desc2['description'])
group.delete()
def get_set_user_test():
client = DSSClient(host, apiKey)
user = client.create_user("toto", "password", "display name of toto", groups=['a','b'])
desc = user.get_definition()
desc['displayName'] = 'tata'
user.set_definition(desc)
desc2 = user.get_definition()
eq_('tata', desc2['displayName'])
user.delete()
def get_set_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
dataset = project.create_dataset("titi", "Filesystem")
definition = dataset.get_definition()
definition['managed'] = True
dataset.set_definition(definition)
definition2 = dataset.get_definition()
eq_(True, definition2['managed'])
dataset.delete()
def get_set_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
dataset = project.create_dataset("titi", "Filesystem")
definition = dataset.get_definition()
definition['managed'] = True
dataset.set_definition(definition)
definition2 = dataset.get_definition()
eq_(True, definition2['managed'])
dataset.delete()
def get_set_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
managedfolder = project.create_managed_folder("titi")
definition = managedfolder.get_definition()
definition['description'] = 'describe me!'
managedfolder.set_definition(definition)
definition2 = managedfolder.get_definition()
eq_('describe me!', definition2['description'])
managedfolder.delete()
def get_set_connection_test():
client = DSSClient(host, apiKey)
connection = client.create_connection("toto", "HDFS")
desc = connection.get_definition()
desc['usableBy'] = 'ALLOWED'
desc['allowedGroups'] = ['a', 'b']
connection.set_definition(desc)
desc2 = connection.get_definition()
eq_('ALLOWED', desc2['usableBy'])
connection.delete()
def get_set_connection_test():
client = DSSClient(host, apiKey)
connection = client.create_connection("toto", "HDFS")
desc = connection.get_definition()
desc['usableBy'] = 'ALLOWED'
desc['allowedGroups'] = ['a','b']
connection.set_definition(desc)
desc2 = connection.get_definition()
eq_('ALLOWED', desc2['usableBy'])
connection.delete()
def get_set_user_test():
client = DSSClient(host, apiKey)
user = client.create_user("toto",
"password",
"display name of toto",
groups=['a', 'b'])
desc = user.get_definition()
desc['displayName'] = 'tata'
user.set_definition(desc)
desc2 = user.get_definition()
eq_('tata', desc2['displayName'])
user.delete()
def get_client_as(self):
"""
Gets a :class:`dataikuapi.DSSClient` that has the permissions of this user.
This allows administrators to impersonate actions on behalf of other users, in order to perform
actions on their behalf
"""
from dataikuapi.dssclient import DSSClient
if self.client.api_key is not None:
return DSSClient(self.client.host,
self.client.api_key,
extra_headers={"X-DKU-ProxyUser": self.login})
elif self.client.internal_ticket is not None:
return DSSClient(self.client.host,
internal_ticket=self.client.internal_ticket,
extra_headers={"X-DKU-ProxyUser": self.login})
else:
raise ValueError("Don't know how to proxy this client")
def job_start_abort_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
count = len(client.get_project(testProjectKey).list_jobs())
job = project.start_job({
"initiator" : "me",
"name" : "some job",
"triggeredFrom" : "API",
"type" : "NON_RECURSIVE_FORCED_BUILD",
"outputs" : [{
"type" : "DATASET",
"projectKey" : testProjectKey,
"id" : testDataset,
"partition" : "NP"
}]
})
job.abort()
eq_(count + 1, len(client.get_project(testProjectKey).list_jobs()))
def create_delete_connection_test():
client = DSSClient(host, apiKey)
count = len(client.list_connections())
connection = client.create_connection("toto", "HDFS")
eq_(count + 1, len(client.list_connections()))
connection.delete()
eq_(count, len(client.list_connections()))
def create_delete_group_test():
client = DSSClient(host, apiKey)
count = len(client.list_groups())
group = client.create_group("toto")
eq_(count + 1, len(client.list_groups()))
group.delete()
eq_(count, len(client.list_groups()))
def project_create_delete_test():
client = DSSClient(host, apiKey)
count = len(client.list_project_keys())
p = client.create_project("toto", "name of toto", "me")
eq_(count + 1, len(client.list_project_keys()))
p.delete()
eq_(count, len(client.list_project_keys()))
def create_delete_user_test():
client = DSSClient(host, apiKey)
count = len(client.list_users())
user = client.create_user("toto",
"password",
"display name of toto",
groups=['a', 'b'])
eq_(count + 1, len(client.list_users()))
user.delete()
eq_(count, len(client.list_users()))
def list_jobs_test(): client = DSSClient(host, apiKey) jobs = client.get_project(testProjectKey).list_jobs() ok_(len(jobs) > 0)
def list_projects_test():
client = DSSClient(host, apiKey)
projects = client.list_project_keys()
ok_(len(projects) > 0)
def project_permissions_test(): client = DSSClient(host, apiKey) project = client.get_project(testProjectKey) perms = project.get_permissions() ok_(perms is not None) project.set_permissions(perms)
def list_groups_test():
client = DSSClient(host, apiKey)
groups = client.list_groups()
ok_(len(groups) > 0)
def project_metadata_test(): client = DSSClient(host, apiKey) project = client.get_project(testProjectKey) meta = project.get_metadata() ok_(meta is not None) project.set_metadata(meta)
def list_users_test():
client = DSSClient(host, apiKey)
users = client.list_users()
ok_(len(users) > 0)
def clear_test():
client = DSSClient(host, apiKey)
dataset = client.get_project(testProjectKey).get_dataset(testClearDataset)
dataset.clear()
def sync_metastore_test():
client = DSSClient(host, apiKey)
dataset = client.get_project(testProjectKey).get_dataset(testHiveDataset)
dataset.synchronize_hive_metastore()
def list_datasets_test():
client = DSSClient(host, apiKey)
datasets = client.get_project(testProjectKey).list_datasets()
ok_(len(datasets) > 0)
def list_partitions_test(): client = DSSClient(host, apiKey) dataset = client.get_project(testPartitionedProjectKey).get_dataset(testPartitionedDataset) ok_(len(dataset.list_partitions()) > 0)
def list_projects_test(): client = DSSClient(host, apiKey) projects = client.list_project_keys() ok_(len(projects) > 0)
def list_connections_test():
client = DSSClient(host, apiKey)
connections = client.list_connections()
ok_(len(connections) > 0)
def project_permissions_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
perms = project.get_permissions()
ok_(perms is not None)
project.set_permissions(perms)
def list_jobs_test():
client = DSSClient(host, apiKey)
jobs = client.get_project(testProjectKey).list_jobs()
ok_(len(jobs) > 0)
def project_metadata_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
meta = project.get_metadata()
ok_(meta is not None)
project.set_metadata(meta)
def clear_partitions_test(): client = DSSClient(host, apiKey) dataset = client.get_project(testPartitionedProjectKey).get_dataset(testDropPartitionedDataset) dataset.clear(json.dumps([testDropPartition]))
def list_managedfolders_test():
client = DSSClient(host, apiKey)
managedfolders = client.get_project(testProjectKey).list_managed_folders()
ok_(len(managedfolders) > 0)
def list_connections_test(): client = DSSClient(host, apiKey) connections = client.list_connections() ok_(len(connections) > 0)
def list_groups_test(): client = DSSClient(host, apiKey) groups = client.list_groups() ok_(len(groups) > 0)
def list_partitions_test():
client = DSSClient(host, apiKey)
dataset = client.get_project(testPartitionedProjectKey).get_dataset(
testPartitionedDataset)
ok_(len(dataset.list_partitions()) > 0)
create_api_key()
api_key = retrieve_api_key()[1]
with open(port_file, 'r') as f:
read_data = f.read()
port = 11200
for line in read_data.split('\n'):
if 'DKU_BASE_PORT' in line :
port = int(line.replace('"','').split('=')[1])
host = socket.getfqdn()
host = 'http://' + host + ':' + str(port)
#create the client to use the rest api
client = DSSClient(host, api_key)
#define the current project
project = client.get_project(project_name)
source_folder_path = input_folder.get_path()
#Let s create the datasets corresponding to thefiles in the folder
files = os.listdir(source_folder_path)
#Extraction loop
dat = []
success = []
succes_reason = []
for file in files :
try:
def get_log_test():
client = DSSClient(host, apiKey)
log = client.get_log("error.log")
ok_(log is not None)
def clear_partitions_test():
client = DSSClient(host, apiKey)
dataset = client.get_project(testPartitionedProjectKey).get_dataset(
testDropPartitionedDataset)
dataset.clear(json.dumps([testDropPartition]))
def list_users_test(): client = DSSClient(host, apiKey) users = client.list_users() ok_(len(users) > 0)
def list_logs_test():
client = DSSClient(host, apiKey)
ok_(len(client.list_logs()) > 0)