def job_start_abort_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
count = len(client.get_project(testProjectKey).list_jobs())
job = project.start_job({
"initiator":
"me",
"name":
"some job",
"triggeredFrom":
"API",
"type":
"NON_RECURSIVE_FORCED_BUILD",
"outputs": [{
"type": "DATASET",
"projectKey": testProjectKey,
"id": testDataset,
"partition": "NP"
}]
})
job.abort()
eq_(count + 1, len(client.get_project(testProjectKey).list_jobs()))
def job_log_test(): client = DSSClient(host, apiKey) project = client.get_project(testProjectKey) jobs = project.list_jobs() for job in jobs: log = project.get_job(job['def']['id']).get_log() ok_(log is not None)
def job_status_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
jobs = project.list_jobs()
for job in jobs:
status = project.get_job(job['def']['id']).get_status()
ok_(status is not None)
def job_log_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
jobs = project.list_jobs()
for job in jobs:
log = project.get_job(job['def']['id']).get_log()
ok_(log is not None)
def job_status_test(): client = DSSClient(host, apiKey) project = client.get_project(testProjectKey) jobs = project.list_jobs() for job in jobs: status = project.get_job(job['def']['id']).get_status() ok_(status is not None)
def delete_contents_outside_test():
temp_folder = tempfile.mkdtemp()
file_content = 'some contents\n on several\nlines'
stuff = osp.join(temp_folder, "test.txt")
with open(stuff, "w") as f:
f.write(file_content)
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
managedfolder = project.create_managed_folder("titi")
count = len(managedfolder.list_contents()['items'])
with open(stuff, "r") as f:
managedfolder.put_file('stuff', f)
eq_(count + 1, len(managedfolder.list_contents()['items']))
with closing(managedfolder.get_file('stuff')) as s:
c = s.raw.read()
eq_(file_content, c)
try:
# climb the file hierarchy up to the dip_home
managedfolder.delete_file('../../../shared-secret.txt')
raise AssertionError('Access outside folder should fail')
finally:
managedfolder.delete()
def updload_replace_delete_test():
temp_folder = tempfile.mkdtemp()
stuff = osp.join(temp_folder, "test.txt")
stuffb = osp.join(temp_folder, "testb.txt")
with open(stuff, "w") as f:
f.write('some contents\n on several\nlines')
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
managedfolder = project.create_managed_folder("titi")
count = len(managedfolder.list_contents()['items'])
with open(stuff, "r") as f:
managedfolder.put_file('stuff', f)
eq_(count + 1, len(managedfolder.list_contents()['items']))
with open(stuffb, "w") as f:
with closing(managedfolder.get_file('stuff')) as s:
f.write(s.raw.read())
eq_(True, filecmp.cmp(stuff, stuffb))
managedfolder.delete_file('stuff')
eq_(count, len(managedfolder.list_contents()['items']))
managedfolder.delete()
def create_delete_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
count = len(project.list_managed_folders())
managedfolder = project.create_managed_folder("titi")
eq_(count + 1, len(project.list_managed_folders()))
managedfolder.delete()
eq_(count, len(project.list_managed_folders()))
def create_delete_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
count = len(project.list_datasets())
dataset = project.create_dataset("titi", "Filesystem")
eq_(count + 1, len(project.list_datasets()))
dataset.delete()
eq_(count, len(project.list_datasets()))
def dataset_data_test():
client = DSSClient(host, apiKey)
p = client.get_project(testProjectKey)
d = p.get_dataset(testDataset)
counter = 0
for r in d.iter_rows():
counter = counter + 1
if counter > 5:
break
eq_(6, counter)
def create_delete_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
count = len(project.list_datasets())
dataset = project.create_dataset("titi", "Filesystem")
eq_(count + 1, len(project.list_datasets()))
dataset.delete()
eq_(count, len(project.list_datasets()))
def dataset_data_test(): client = DSSClient(host, apiKey) p = client.get_project(testProjectKey) d = p.get_dataset(testDataset) counter = 0 for r in d.iter_rows(): counter = counter + 1 if counter > 5: break eq_(6, counter)
def job_start_abort_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
count = len(client.get_project(testProjectKey).list_jobs())
job = project.start_job({
"initiator" : "me",
"name" : "some job",
"triggeredFrom" : "API",
"type" : "NON_RECURSIVE_FORCED_BUILD",
"outputs" : [{
"type" : "DATASET",
"projectKey" : testProjectKey,
"id" : testDataset,
"partition" : "NP"
}]
})
job.abort()
eq_(count + 1, len(client.get_project(testProjectKey).list_jobs()))
def get_set_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
dataset = project.create_dataset("titi", "Filesystem")
definition = dataset.get_definition()
definition['managed'] = True
dataset.set_definition(definition)
definition2 = dataset.get_definition()
eq_(True, definition2['managed'])
dataset.delete()
def get_set_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
managedfolder = project.create_managed_folder("titi")
definition = managedfolder.get_definition()
definition['description'] = 'describe me!'
managedfolder.set_definition(definition)
definition2 = managedfolder.get_definition()
eq_('describe me!', definition2['description'])
managedfolder.delete()
def get_set_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
dataset = project.create_dataset("titi", "Filesystem")
definition = dataset.get_definition()
definition['managed'] = True
dataset.set_definition(definition)
definition2 = dataset.get_definition()
eq_(True, definition2['managed'])
dataset.delete()
def clear_partitions_test():
client = DSSClient(host, apiKey)
dataset = client.get_project(testPartitionedProjectKey).get_dataset(
testDropPartitionedDataset)
dataset.clear(json.dumps([testDropPartition]))
def list_partitions_test():
client = DSSClient(host, apiKey)
dataset = client.get_project(testPartitionedProjectKey).get_dataset(
testPartitionedDataset)
ok_(len(dataset.list_partitions()) > 0)
def clear_test():
client = DSSClient(host, apiKey)
dataset = client.get_project(testProjectKey).get_dataset(testClearDataset)
dataset.clear()
def sync_metastore_test():
client = DSSClient(host, apiKey)
dataset = client.get_project(testProjectKey).get_dataset(testHiveDataset)
dataset.synchronize_hive_metastore()
def clear_partitions_test(): client = DSSClient(host, apiKey) dataset = client.get_project(testPartitionedProjectKey).get_dataset(testDropPartitionedDataset) dataset.clear(json.dumps([testDropPartition]))
def list_datasets_test():
client = DSSClient(host, apiKey)
datasets = client.get_project(testProjectKey).list_datasets()
ok_(len(datasets) > 0)
def project_permissions_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
perms = project.get_permissions()
ok_(perms is not None)
project.set_permissions(perms)
def project_metadata_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
meta = project.get_metadata()
ok_(meta is not None)
project.set_metadata(meta)
def project_permissions_test(): client = DSSClient(host, apiKey) project = client.get_project(testProjectKey) perms = project.get_permissions() ok_(perms is not None) project.set_permissions(perms)
def clear_test(): client = DSSClient(host, apiKey) dataset = client.get_project(testProjectKey).get_dataset(testClearDataset) dataset.clear()
def list_managedfolders_test():
client = DSSClient(host, apiKey)
managedfolders = client.get_project(testProjectKey).list_managed_folders()
ok_(len(managedfolders) > 0)
def list_partitions_test(): client = DSSClient(host, apiKey) dataset = client.get_project(testPartitionedProjectKey).get_dataset(testPartitionedDataset) ok_(len(dataset.list_partitions()) > 0)
with open(port_file, 'r') as f:
read_data = f.read()
port = 11200
for line in read_data.split('\n'):
if 'DKU_BASE_PORT' in line :
port = int(line.replace('"','').split('=')[1])
host = socket.getfqdn()
host = 'http://' + host + ':' + str(port)
#create the client to use the rest api
client = DSSClient(host, api_key)
#define the current project
project = client.get_project(project_name)
source_folder_path = input_folder.get_path()
#Let s create the datasets corresponding to thefiles in the folder
files = os.listdir(source_folder_path)
#Extraction loop
dat = []
success = []
succes_reason = []
for file in files :
try:
dataset = project.create_dataset(file.encode('ascii', 'ignore').decode('ascii').split('.')[0] # dot is not allowed in dataset names
,'Filesystem'
, params={
def list_jobs_test():
client = DSSClient(host, apiKey)
jobs = client.get_project(testProjectKey).list_jobs()
ok_(len(jobs) > 0)
def project_metadata_test(): client = DSSClient(host, apiKey) project = client.get_project(testProjectKey) meta = project.get_metadata() ok_(meta is not None) project.set_metadata(meta)
def sync_metastore_test(): client = DSSClient(host, apiKey) dataset = client.get_project(testProjectKey).get_dataset(testHiveDataset) dataset.synchronize_hive_metastore()
def list_jobs_test(): client = DSSClient(host, apiKey) jobs = client.get_project(testProjectKey).list_jobs() ok_(len(jobs) > 0)
def list_datasets_test(): client = DSSClient(host, apiKey) datasets = client.get_project(testProjectKey).list_datasets() ok_(len(datasets) > 0)
