Example #1
0
    def test_Jobs_submitRDBJob(self):

        rdbComputeDomain = Jobs.getRDBComputeDomainFromName(
            Jobs_RDBComputeDomainName)

        jobId = Jobs.submitRDBQueryJob(Jobs_SqlQuery, rdbComputeDomain,
                                       Jobs_DatabaseContextName,
                                       Jobs_QueryResultsFile, Jobs_Alias)

        jobStatus = Jobs.waitForJob(jobId)
        self.assertTrue(jobStatus == Jobs.getJobStatus(jobId))
        self.assertTrue(jobStatus.get('status') == 32)

        job = Jobs.getJobDescription(jobId)
        self.assertTrue(job.get('username') == Authentication_loginName)
        self.assertTrue(job.get('rdbDomainName') == Jobs_RDBComputeDomainName)
        self.assertTrue(
            job.get('databaseContextName') == Jobs_DatabaseContextName)
        self.assertTrue(job.get('inputSql') == Jobs_SqlQuery)
        self.assertTrue(job.get('submitterDID') == Jobs_Alias)

        fileService = Files.getFileServiceFromName(Jobs_FileServiceName)
        jobDirectory = job.get('resultsFolderURI')
        relativePath = jobDirectory.split(
            'scratch/')[1] + Jobs_QueryResultsFile + '.csv'
        string = Files.download(fileService,
                                'scratch',
                                '',
                                relativePath,
                                format="txt",
                                userVolumeOwner=Authentication_loginName)
        string.rstrip("\n")
        self.assertTrue(string, Jobs_SqlQueryResult)
Example #2
0
    def test_Files_getRootVolumesInfo(self):
        fileService = Files.getFileServiceFromName(Files_FileServiceName)
        rootVolumes = Files.getRootVolumesInfo(fileService)
        self.assertTrue(rootVolumes.__len__() > 0)
        found = False
        for rootVolume in rootVolumes:
            if rootVolume.get('rootVolumeName') == Files_RootVolumeName1:
                found = True
        self.assertTrue(found)

        found = False
        for rootVolume in rootVolumes:
            if rootVolume.get('rootVolumeName') == Files_RootVolumeName2:
                found = True
        self.assertTrue(found)
Example #3
0
 def test_Files_createUserVolume_deleteUserVolume(self):
     fileService = Files.getFileServiceFromName(Files_FileServiceName)
     Files.createUserVolume(fileService,
                            "/".join([
                                Files_RootVolumeName1,
                                Authentication_loginName,
                                Files_UserVolumeName1
                            ]),
                            quiet=False)
     Files.deleteUserVolume(fileService,
                            "/".join([
                                Files_RootVolumeName1,
                                Authentication_loginName,
                                Files_UserVolumeName1
                            ]),
                            quiet=False)
Example #4
0
    def test_Files_getFileServicesNames(self):
        fileServiceNames = Files.getFileServicesNames()
        found = False
        for fileService in fileServiceNames:
            if fileService.get('name') == Files_FileServiceName:
                found = True

        self.assertTrue(found)
    def upload_repo(file_service_name, volume_name, path_='Redshift-Resnet.tar'):
        """Archive this repo and upload it to sciserver-files"""
        tmp_dir = tempfile.TemporaryDirectory().name
        archive = zip_dir('',
                          os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'),
                          format='tar',
                          dest_dir=tmp_dir)
        logger.info('The repo is archived in {}'.format(archive))

        file_service_name = file_service_name
        file_service = sf.getFileServiceFromName(file_service_name)
        user_volumes = sf.getUserVolumesInfo(file_service)
        volume_path = user_volumes[0].get('path', 'Storage/username/persistent').rsplit('/', 1)[0] + '/' + volume_name
        logger.info('Creating {} if it doesn\'t exist.'.format(volume_path))

        sf.createUserVolume(file_service, volume_path, quiet=True)
        sf.upload(file_service,
                  path=volume_path + '/' + path_,
                  localFilePath=archive,
                  quiet=True)
        logger.info('Uploaded archived repo to sciserver-files at {}'.format(volume_path + '/' + path_))

        return volume_path + '/' + path_
Files_NewDirectoryName1 = "myNewDirectory555"
Files_NewDirectoryName2 = "myNewDirectory999"
Files_LocalFileName = "MyNewFile.txt"
Files_LocalFileContent = "#ID,Column1,Column2\n1,4.5,5.5"


# In[ ]:

token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword);


# In[ ]:

# get available FileServices

fileServices = Files.getFileServices();
print(fileServices)


# In[ ]:

# get names of file services available to the user.

fileServiceNames = Files.getFileServicesNames();
print(fileServiceNames)


# In[ ]:

# get FileService from Name
Example #7
0
    def test_Jobs_submitNotebookJob_cancel_waitForJob_getJobStatus_getJobDescription_submitShellCommandJob(
            self):

        fileService = Files.getFileServiceFromName(Jobs_FileServiceName)
        try:
            Files.deleteUserVolume(fileService, Jobs_RootVolumeName,
                                   Jobs_UserVolumeName)
        except:
            pass

        Files.createUserVolume(fileService, Jobs_RootVolumeName,
                               Jobs_UserVolumeName)
        Files.upload(fileService,
                     Jobs_RootVolumeName,
                     Jobs_UserVolumeName,
                     Jobs_DirectoryName + "/" + Jobs_NotebookName,
                     localFilePath=Jobs_NotebookName)

        dockerComputeDomain = Jobs.getDockerComputeDomainFromName(
            Jobs_DockerComputeDomainName)

        jobId_1 = Jobs.submitNotebookJob(
            '/home/idies/workspace/' + Jobs_UserVolumeName + '/' +
            Jobs_DirectoryName + '/' + Jobs_NotebookName, dockerComputeDomain,
            Jobs_DockerImageName, Jobs_UserVolumes, Jobs_DataVolumes,
            Jobs_Parameters, Jobs_Alias)
        Jobs.cancelJob(jobId_1)
        jobStatus = Jobs.getJobStatus(jobId_1)
        self.assertTrue(jobStatus.get('status') == 128)

        jobId_2 = Jobs.submitNotebookJob(Jobs_RemoteNotebookPath,
                                         dockerComputeDomain,
                                         Jobs_DockerImageName,
                                         Jobs_UserVolumes, Jobs_DataVolumes,
                                         Jobs_Parameters, Jobs_Alias)

        jobStatus = Jobs.waitForJob(jobId_2)
        self.assertTrue(jobStatus == Jobs.getJobStatus(jobId_2))
        self.assertTrue(jobStatus.get('status') == 32)

        job = Jobs.getJobDescription(jobId_2)
        self.assertTrue(job.get('username') == Authentication_loginName)
        self.assertTrue(job.get('dockerImageName') == Jobs_DockerImageName)
        self.assertTrue(job.get('scriptURI') == Jobs_RemoteNotebookPath)
        self.assertTrue(job.get('submitterDID') == Jobs_Alias)

        jobDirectory = job.get('resultsFolderURI')
        relativePath = jobDirectory.split(
            'scratch/')[1] + Jobs_NoteBookOutPutFile
        string = Files.download(fileService,
                                'scratch',
                                '',
                                relativePath,
                                format="txt",
                                userVolumeOwner=Authentication_loginName)
        string.rstrip("\n")
        self.assertTrue(string, job.get('resultsFolderURI'))

        jobs = Jobs.getJobsList(top=2)
        found = False
        for job in jobs:
            if jobId_1 == job.get("id"):
                found = True
        self.assertTrue(found)

        found = False
        for job in jobs:
            if jobId_2 == job.get("id"):
                found = True
        self.assertTrue(found)

        jobId = Jobs.submitShellCommandJob(Jobs_ShellCommand,
                                           dockerComputeDomain,
                                           Jobs_DockerImageName,
                                           Jobs_UserVolumes, Jobs_DataVolumes,
                                           Jobs_Alias)

        jobStatus = Jobs.waitForJob(jobId)
        self.assertTrue(jobStatus == Jobs.getJobStatus(jobId))
        self.assertTrue(jobStatus.get('status') == 32)

        job = Jobs.getJobDescription(jobId)
        self.assertTrue(job.get('username') == Authentication_loginName)
        self.assertTrue(job.get('dockerImageName') == Jobs_DockerImageName)
        self.assertTrue(job.get('command') == Jobs_ShellCommand)
        self.assertTrue(job.get('submitterDID') == Jobs_Alias)

        jobDirectory = job.get('resultsFolderURI')
        relativePath = jobDirectory.split('scratch/')[1] + "command.txt"
        string = Files.download(fileService,
                                'scratch',
                                '',
                                relativePath,
                                format="txt",
                                userVolumeOwner=Authentication_loginName)
        string.rstrip("\n")
        self.assertTrue(string, job.get('resultsFolderURI'))

        Files.deleteUserVolume(fileService, Jobs_RootVolumeName,
                               Jobs_UserVolumeName)
Example #8
0
    def test_Files_createDir_upload_dirList_download_download_shareUserVolume(
            self):

        try:
            fileService = Files.getFileServiceFromName(Files_FileServiceName)
            os.remove(Files_LocalFileName)
            Files.deleteUserVolume(fileService,
                                   Files_RootVolumeName1,
                                   Files_UserVolumeName1,
                                   quiet=True)
            Files.deleteUserVolume(fileService,
                                   Files_RootVolumeName1,
                                   Files_UserVolumeName2,
                                   quiet=True)
        except:
            pass

        try:
            fileService = Files.getFileServiceFromName(Files_FileServiceName)

            Files.createUserVolume(fileService,
                                   "/".join([
                                       Files_RootVolumeName1,
                                       Authentication_loginName,
                                       Files_UserVolumeName1
                                   ]),
                                   quiet=False)
            Files.createUserVolume(fileService,
                                   "/".join([
                                       Files_RootVolumeName1,
                                       Authentication_loginName,
                                       Files_UserVolumeName2
                                   ]),
                                   quiet=False)

            Files.createDir(
                fileService, "/".join([
                    Files_RootVolumeName1, Authentication_loginName,
                    Files_UserVolumeName1, Files_NewDirectoryName1
                ]))
            Files.createDir(
                fileService, "/".join([
                    Files_RootVolumeName2, Authentication_loginName,
                    Files_UserVolumeName2, Files_NewDirectoryName2
                ]))

            dirList = Files.dirList(fileService,
                                    "/".join([
                                        Files_RootVolumeName1,
                                        Authentication_loginName,
                                        Files_UserVolumeName1,
                                        Files_NewDirectoryName1
                                    ]),
                                    level=2)
            self.assertTrue(
                dirList.get('root').get('name') == Files_NewDirectoryName1)

            Files.upload(fileService,
                         "/".join([
                             Files_RootVolumeName1, Authentication_loginName,
                             Files_UserVolumeName1, Files_NewDirectoryName1,
                             Files_LocalFileName
                         ]),
                         data=Files_LocalFileContent)

            dirList = Files.dirList(fileService,
                                    "/".join([
                                        Files_RootVolumeName1,
                                        Authentication_loginName,
                                        Files_UserVolumeName1,
                                        Files_NewDirectoryName1
                                    ]),
                                    level=2)
            self.assertTrue(
                dirList.get('root').get('files')[0].get('name') ==
                Files_LocalFileName)

            Files.download(fileService,
                           "/".join([
                               Files_RootVolumeName1, Authentication_loginName,
                               Files_UserVolumeName1, Files_NewDirectoryName1,
                               Files_LocalFileName
                           ]),
                           localFilePath=Files_LocalFileName)

            with open(Files_LocalFileName, 'r') as myfile:
                downloadedFileContent = myfile.read()
                assert (downloadedFileContent == Files_LocalFileContent)

            Files.delete(
                fileService, "/".join([
                    Files_RootVolumeName1, Authentication_loginName,
                    Files_UserVolumeName1, Files_NewDirectoryName1,
                    Files_LocalFileName
                ]))

            dirList = Files.dirList(fileService,
                                    "/".join([
                                        Files_RootVolumeName1,
                                        Authentication_loginName,
                                        Files_UserVolumeName1,
                                        Files_NewDirectoryName1
                                    ]),
                                    level=2)
            self.assertIsNone(dirList.get('root').get('files'))

            Files.upload(fileService,
                         "/".join([
                             Files_RootVolumeName1, Authentication_loginName,
                             Files_UserVolumeName1, Files_NewDirectoryName1,
                             Files_LocalFileName
                         ]),
                         localFilePath=Files_LocalFileName,
                         quiet=False)

            Files.move(
                fileService, "/".join([
                    Files_RootVolumeName1, Authentication_loginName,
                    Files_UserVolumeName1, Files_NewDirectoryName1,
                    Files_LocalFileName
                ]), fileService, "/".join([
                    Files_RootVolumeName2, Authentication_loginName,
                    Files_UserVolumeName2, Files_NewDirectoryName2,
                    Files_LocalFileName
                ]))

            Files.shareUserVolume(
                fileService,
                "/".join([
                    Files_RootVolumeName2, Authentication_loginName,
                    Files_UserVolumeName2
                ]),
                sharedWith=Authentication_login_sharedWithName,
                type="USER",
                allowedActions=["read"])

            token1 = Authentication.login(
                Authentication_login_sharedWithName,
                Authentication_login_sharedWithPassword)

            string = Files.download(fileService,
                                    "/".join([
                                        Files_RootVolumeName2,
                                        Authentication_loginName,
                                        Files_UserVolumeName2,
                                        Files_NewDirectoryName2,
                                        Files_LocalFileName
                                    ]),
                                    format="txt")

            self.assertTrue(string, Files_LocalFileContent)

            token1 = Authentication.login(Authentication_loginName,
                                          Authentication_loginPassword)

        finally:
            try:
                os.remove(Files_LocalFileName)
                Files.deleteUserVolume(fileService,
                                       "/".join([
                                           Files_RootVolumeName1,
                                           Authentication_loginName,
                                           Files_UserVolumeName1
                                       ]),
                                       quiet=True)
                Files.deleteUserVolume(fileService,
                                       "/".join([
                                           Files_RootVolumeName1,
                                           Authentication_loginName,
                                           Files_UserVolumeName2
                                       ]),
                                       quiet=True)
            except:
                pass
Example #9
0
 def test_Files_getUserVolumesInfo(self):
     fileService = Files.getFileServiceFromName(Files_FileServiceName)
     userVolumesInfo = Files.getUserVolumesInfo(fileService)
     self.assertTrue(userVolumesInfo.__len__() > 0)
Example #10
0
 def test_Files_getFileServiceFromName(self):
     fileService = Files.getFileServiceFromName(Files_FileServiceName)
     self.assertTrue(fileService.get('name') == Files_FileServiceName)
Example #11
0
 def test_Files_getFileServicesNames(self):
     fileServiceNames = Files.getFileServicesNames()
     self.assertTrue(fileServiceNames.__len__() > 0)