예제 #1
0
def job_start_abort_test():
    client = DSSClient(host, apiKey)
    project = client.get_project(testProjectKey)

    count = len(client.get_project(testProjectKey).list_jobs())

    job = project.start_job({
        "initiator":
        "me",
        "name":
        "some job",
        "triggeredFrom":
        "API",
        "type":
        "NON_RECURSIVE_FORCED_BUILD",
        "outputs": [{
            "type": "DATASET",
            "projectKey": testProjectKey,
            "id": testDataset,
            "partition": "NP"
        }]
    })
    job.abort()

    eq_(count + 1, len(client.get_project(testProjectKey).list_jobs()))
def job_log_test():
	client = DSSClient(host, apiKey)
	project = client.get_project(testProjectKey)
	jobs = project.list_jobs()
	for job in jobs:
		log = project.get_job(job['def']['id']).get_log()
		ok_(log is not None)
예제 #3
0
def job_status_test():
    client = DSSClient(host, apiKey)
    project = client.get_project(testProjectKey)
    jobs = project.list_jobs()
    for job in jobs:
        status = project.get_job(job['def']['id']).get_status()
        ok_(status is not None)
예제 #4
0
def job_log_test():
    client = DSSClient(host, apiKey)
    project = client.get_project(testProjectKey)
    jobs = project.list_jobs()
    for job in jobs:
        log = project.get_job(job['def']['id']).get_log()
        ok_(log is not None)
def job_status_test():
	client = DSSClient(host, apiKey)
	project = client.get_project(testProjectKey)
	jobs = project.list_jobs()
	for job in jobs:
		status = project.get_job(job['def']['id']).get_status()
		ok_(status is not None)
def delete_contents_outside_test():
    temp_folder = tempfile.mkdtemp()
    file_content = 'some contents\n on several\nlines'
    stuff = osp.join(temp_folder, "test.txt")
    with open(stuff, "w") as f:
        f.write(file_content)
    
    client = DSSClient(host, apiKey)
    project = client.get_project(testProjectKey)
    managedfolder = project.create_managed_folder("titi")

    count = len(managedfolder.list_contents()['items'])
    with open(stuff, "r") as f:
        managedfolder.put_file('stuff', f)
    eq_(count + 1, len(managedfolder.list_contents()['items']))
        
    with closing(managedfolder.get_file('stuff')) as s:
        c = s.raw.read()
    eq_(file_content, c)
    
    try:
        # climb the file hierarchy up to the dip_home
        managedfolder.delete_file('../../../shared-secret.txt')
        raise AssertionError('Access outside folder should fail')        
    finally:
        managedfolder.delete()    
           
def updload_replace_delete_test():
    temp_folder = tempfile.mkdtemp()
    stuff = osp.join(temp_folder, "test.txt")
    stuffb = osp.join(temp_folder, "testb.txt")
    with open(stuff, "w") as f:
        f.write('some contents\n on several\nlines')
    
    client = DSSClient(host, apiKey)
    project = client.get_project(testProjectKey)
    managedfolder = project.create_managed_folder("titi")

    count = len(managedfolder.list_contents()['items'])
    with open(stuff, "r") as f:
        managedfolder.put_file('stuff', f)
    eq_(count + 1, len(managedfolder.list_contents()['items']))
        
    with open(stuffb, "w") as f:
        with closing(managedfolder.get_file('stuff')) as s:
            f.write(s.raw.read())
    
    eq_(True, filecmp.cmp(stuff, stuffb))

    managedfolder.delete_file('stuff')
    eq_(count, len(managedfolder.list_contents()['items']))

    managedfolder.delete()    
def create_delete_test():
    client = DSSClient(host, apiKey)
    project = client.get_project(testProjectKey)
    count = len(project.list_managed_folders())

    managedfolder = project.create_managed_folder("titi")
    eq_(count + 1, len(project.list_managed_folders()))
    
    managedfolder.delete()    
    eq_(count, len(project.list_managed_folders()))
def create_delete_test():
	client = DSSClient(host, apiKey)
	project = client.get_project(testProjectKey)
	count = len(project.list_datasets())

	dataset = project.create_dataset("titi", "Filesystem")
	eq_(count + 1, len(project.list_datasets()))
	
	dataset.delete()	
	eq_(count, len(project.list_datasets()))
def dataset_data_test():
    client = DSSClient(host, apiKey)
    p = client.get_project(testProjectKey)
    d = p.get_dataset(testDataset)
    counter = 0
    for r in d.iter_rows():
        counter = counter + 1
        if counter > 5:
            break
    eq_(6, counter)
def create_delete_test():
    client = DSSClient(host, apiKey)
    project = client.get_project(testProjectKey)
    count = len(project.list_datasets())

    dataset = project.create_dataset("titi", "Filesystem")
    eq_(count + 1, len(project.list_datasets()))

    dataset.delete()
    eq_(count, len(project.list_datasets()))
def dataset_data_test():
	client = DSSClient(host, apiKey)
	p = client.get_project(testProjectKey)
	d = p.get_dataset(testDataset)
	counter = 0
	for r in d.iter_rows():
		counter = counter + 1
		if counter > 5:
			break
	eq_(6, counter)
예제 #13
0
def job_start_abort_test():
	client = DSSClient(host, apiKey)
	project = client.get_project(testProjectKey)
	
	count = len(client.get_project(testProjectKey).list_jobs())
	
	job = project.start_job({
		"initiator" : "me",
		"name" : "some job",
		"triggeredFrom" : "API",
		"type" : "NON_RECURSIVE_FORCED_BUILD",
		"outputs" : [{
						"type" : "DATASET",
						"projectKey" : testProjectKey,
						"id" : testDataset,
						"partition" : "NP"
					}]
	})
	job.abort()

	eq_(count + 1, len(client.get_project(testProjectKey).list_jobs()))
def get_set_test():
	client = DSSClient(host, apiKey)
	project = client.get_project(testProjectKey)
	dataset = project.create_dataset("titi", "Filesystem")
	
	definition = dataset.get_definition()
	definition['managed'] = True
	dataset.set_definition(definition)
	definition2 = dataset.get_definition()

	eq_(True, definition2['managed'])
	
	dataset.delete()	
def get_set_test():
    client = DSSClient(host, apiKey)
    project = client.get_project(testProjectKey)
    managedfolder = project.create_managed_folder("titi")
    
    definition = managedfolder.get_definition()
    definition['description'] = 'describe me!'
    managedfolder.set_definition(definition)
    definition2 = managedfolder.get_definition()

    eq_('describe me!', definition2['description'])
    
    managedfolder.delete()    
def get_set_test():
    client = DSSClient(host, apiKey)
    project = client.get_project(testProjectKey)
    dataset = project.create_dataset("titi", "Filesystem")

    definition = dataset.get_definition()
    definition['managed'] = True
    dataset.set_definition(definition)
    definition2 = dataset.get_definition()

    eq_(True, definition2['managed'])

    dataset.delete()
def clear_partitions_test():
    client = DSSClient(host, apiKey)
    dataset = client.get_project(testPartitionedProjectKey).get_dataset(
        testDropPartitionedDataset)
    dataset.clear(json.dumps([testDropPartition]))
def list_partitions_test():
    client = DSSClient(host, apiKey)
    dataset = client.get_project(testPartitionedProjectKey).get_dataset(
        testPartitionedDataset)
    ok_(len(dataset.list_partitions()) > 0)
def clear_test():
    client = DSSClient(host, apiKey)
    dataset = client.get_project(testProjectKey).get_dataset(testClearDataset)
    dataset.clear()
def sync_metastore_test():
    client = DSSClient(host, apiKey)
    dataset = client.get_project(testProjectKey).get_dataset(testHiveDataset)
    dataset.synchronize_hive_metastore()
def clear_partitions_test():
	client = DSSClient(host, apiKey)
	dataset = client.get_project(testPartitionedProjectKey).get_dataset(testDropPartitionedDataset)
	dataset.clear(json.dumps([testDropPartition]))
def list_datasets_test():
    client = DSSClient(host, apiKey)
    datasets = client.get_project(testProjectKey).list_datasets()
    ok_(len(datasets) > 0)
예제 #23
0
def project_permissions_test():
    client = DSSClient(host, apiKey)
    project = client.get_project(testProjectKey)
    perms = project.get_permissions()
    ok_(perms is not None)
    project.set_permissions(perms)
예제 #24
0
def project_metadata_test():
    client = DSSClient(host, apiKey)
    project = client.get_project(testProjectKey)
    meta = project.get_metadata()
    ok_(meta is not None)
    project.set_metadata(meta)
def project_permissions_test():
	client = DSSClient(host, apiKey)
	project = client.get_project(testProjectKey)
	perms = project.get_permissions()
	ok_(perms is not None)
	project.set_permissions(perms)
def clear_test():
	client = DSSClient(host, apiKey)
	dataset = client.get_project(testProjectKey).get_dataset(testClearDataset)
	dataset.clear()
def list_managedfolders_test():
    client = DSSClient(host, apiKey)
    managedfolders = client.get_project(testProjectKey).list_managed_folders()
    ok_(len(managedfolders) > 0)    
def list_partitions_test():
	client = DSSClient(host, apiKey)
	dataset = client.get_project(testPartitionedProjectKey).get_dataset(testPartitionedDataset)
	ok_(len(dataset.list_partitions()) > 0)
예제 #29
0
with open(port_file, 'r') as f:
    read_data = f.read()
    port = 11200
    for line in read_data.split('\n'):
        if 'DKU_BASE_PORT' in line :
            port = int(line.replace('"','').split('=')[1])

host = socket.getfqdn()
host = 'http://' + host + ':' + str(port)

#create the client to use the rest api
client = DSSClient(host, api_key)

#define the current project
project = client.get_project(project_name)

source_folder_path = input_folder.get_path()

#Let s create the datasets corresponding to thefiles in the folder
files = os.listdir(source_folder_path)

#Extraction loop
dat = []
success = []
succes_reason = []
for file in files :
    try:
        dataset = project.create_dataset(file.encode('ascii', 'ignore').decode('ascii').split('.')[0]  # dot is not allowed in dataset names
            	,'Filesystem'
            	, params={
예제 #30
0
def list_jobs_test():
    client = DSSClient(host, apiKey)
    jobs = client.get_project(testProjectKey).list_jobs()
    ok_(len(jobs) > 0)
def project_metadata_test():
	client = DSSClient(host, apiKey)
	project = client.get_project(testProjectKey)
	meta = project.get_metadata()
	ok_(meta is not None)
	project.set_metadata(meta)
def sync_metastore_test():
	client = DSSClient(host, apiKey)
	dataset = client.get_project(testProjectKey).get_dataset(testHiveDataset)
	dataset.synchronize_hive_metastore()
예제 #33
0
def list_jobs_test():
	client = DSSClient(host, apiKey)
	jobs = client.get_project(testProjectKey).list_jobs()
	ok_(len(jobs) > 0)
def list_datasets_test():
	client = DSSClient(host, apiKey)
	datasets = client.get_project(testProjectKey).list_datasets()
	ok_(len(datasets) > 0)