def test_Jobs_submitRDBJob(self):
rdbComputeDomain = Jobs.getRDBComputeDomainFromName(
Jobs_RDBComputeDomainName)
jobId = Jobs.submitRDBQueryJob(Jobs_SqlQuery, rdbComputeDomain,
Jobs_DatabaseContextName,
Jobs_QueryResultsFile, Jobs_Alias)
jobStatus = Jobs.waitForJob(jobId)
self.assertTrue(jobStatus == Jobs.getJobStatus(jobId))
self.assertTrue(jobStatus.get('status') == 32)
job = Jobs.getJobDescription(jobId)
self.assertTrue(job.get('username') == Authentication_loginName)
self.assertTrue(job.get('rdbDomainName') == Jobs_RDBComputeDomainName)
self.assertTrue(
job.get('databaseContextName') == Jobs_DatabaseContextName)
self.assertTrue(job.get('inputSql') == Jobs_SqlQuery)
self.assertTrue(job.get('submitterDID') == Jobs_Alias)
fileService = Files.getFileServiceFromName(Jobs_FileServiceName)
jobDirectory = job.get('resultsFolderURI')
relativePath = jobDirectory.split(
'scratch/')[1] + Jobs_QueryResultsFile + '.csv'
string = Files.download(fileService,
'scratch',
'',
relativePath,
format="txt",
userVolumeOwner=Authentication_loginName)
string.rstrip("\n")
self.assertTrue(string, Jobs_SqlQueryResult)
def test_Files_getRootVolumesInfo(self):
fileService = Files.getFileServiceFromName(Files_FileServiceName)
rootVolumes = Files.getRootVolumesInfo(fileService)
self.assertTrue(rootVolumes.__len__() > 0)
found = False
for rootVolume in rootVolumes:
if rootVolume.get('rootVolumeName') == Files_RootVolumeName1:
found = True
self.assertTrue(found)
found = False
for rootVolume in rootVolumes:
if rootVolume.get('rootVolumeName') == Files_RootVolumeName2:
found = True
self.assertTrue(found)
def test_Files_createUserVolume_deleteUserVolume(self):
fileService = Files.getFileServiceFromName(Files_FileServiceName)
Files.createUserVolume(fileService,
"/".join([
Files_RootVolumeName1,
Authentication_loginName,
Files_UserVolumeName1
]),
quiet=False)
Files.deleteUserVolume(fileService,
"/".join([
Files_RootVolumeName1,
Authentication_loginName,
Files_UserVolumeName1
]),
quiet=False)
def test_Files_getFileServicesNames(self):
fileServiceNames = Files.getFileServicesNames()
found = False
for fileService in fileServiceNames:
if fileService.get('name') == Files_FileServiceName:
found = True
self.assertTrue(found)
def upload_repo(file_service_name, volume_name, path_='Redshift-Resnet.tar'):
"""Archive this repo and upload it to sciserver-files"""
tmp_dir = tempfile.TemporaryDirectory().name
archive = zip_dir('',
os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'),
format='tar',
dest_dir=tmp_dir)
logger.info('The repo is archived in {}'.format(archive))
file_service_name = file_service_name
file_service = sf.getFileServiceFromName(file_service_name)
user_volumes = sf.getUserVolumesInfo(file_service)
volume_path = user_volumes[0].get('path', 'Storage/username/persistent').rsplit('/', 1)[0] + '/' + volume_name
logger.info('Creating {} if it doesn\'t exist.'.format(volume_path))
sf.createUserVolume(file_service, volume_path, quiet=True)
sf.upload(file_service,
path=volume_path + '/' + path_,
localFilePath=archive,
quiet=True)
logger.info('Uploaded archived repo to sciserver-files at {}'.format(volume_path + '/' + path_))
return volume_path + '/' + path_
Files_NewDirectoryName1 = "myNewDirectory555" Files_NewDirectoryName2 = "myNewDirectory999" Files_LocalFileName = "MyNewFile.txt" Files_LocalFileContent = "#ID,Column1,Column2\n1,4.5,5.5" # In[ ]: token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword); # In[ ]: # get available FileServices fileServices = Files.getFileServices(); print(fileServices) # In[ ]: # get names of file services available to the user. fileServiceNames = Files.getFileServicesNames(); print(fileServiceNames) # In[ ]: # get FileService from Name
def test_Jobs_submitNotebookJob_cancel_waitForJob_getJobStatus_getJobDescription_submitShellCommandJob(
self):
fileService = Files.getFileServiceFromName(Jobs_FileServiceName)
try:
Files.deleteUserVolume(fileService, Jobs_RootVolumeName,
Jobs_UserVolumeName)
except:
pass
Files.createUserVolume(fileService, Jobs_RootVolumeName,
Jobs_UserVolumeName)
Files.upload(fileService,
Jobs_RootVolumeName,
Jobs_UserVolumeName,
Jobs_DirectoryName + "/" + Jobs_NotebookName,
localFilePath=Jobs_NotebookName)
dockerComputeDomain = Jobs.getDockerComputeDomainFromName(
Jobs_DockerComputeDomainName)
jobId_1 = Jobs.submitNotebookJob(
'/home/idies/workspace/' + Jobs_UserVolumeName + '/' +
Jobs_DirectoryName + '/' + Jobs_NotebookName, dockerComputeDomain,
Jobs_DockerImageName, Jobs_UserVolumes, Jobs_DataVolumes,
Jobs_Parameters, Jobs_Alias)
Jobs.cancelJob(jobId_1)
jobStatus = Jobs.getJobStatus(jobId_1)
self.assertTrue(jobStatus.get('status') == 128)
jobId_2 = Jobs.submitNotebookJob(Jobs_RemoteNotebookPath,
dockerComputeDomain,
Jobs_DockerImageName,
Jobs_UserVolumes, Jobs_DataVolumes,
Jobs_Parameters, Jobs_Alias)
jobStatus = Jobs.waitForJob(jobId_2)
self.assertTrue(jobStatus == Jobs.getJobStatus(jobId_2))
self.assertTrue(jobStatus.get('status') == 32)
job = Jobs.getJobDescription(jobId_2)
self.assertTrue(job.get('username') == Authentication_loginName)
self.assertTrue(job.get('dockerImageName') == Jobs_DockerImageName)
self.assertTrue(job.get('scriptURI') == Jobs_RemoteNotebookPath)
self.assertTrue(job.get('submitterDID') == Jobs_Alias)
jobDirectory = job.get('resultsFolderURI')
relativePath = jobDirectory.split(
'scratch/')[1] + Jobs_NoteBookOutPutFile
string = Files.download(fileService,
'scratch',
'',
relativePath,
format="txt",
userVolumeOwner=Authentication_loginName)
string.rstrip("\n")
self.assertTrue(string, job.get('resultsFolderURI'))
jobs = Jobs.getJobsList(top=2)
found = False
for job in jobs:
if jobId_1 == job.get("id"):
found = True
self.assertTrue(found)
found = False
for job in jobs:
if jobId_2 == job.get("id"):
found = True
self.assertTrue(found)
jobId = Jobs.submitShellCommandJob(Jobs_ShellCommand,
dockerComputeDomain,
Jobs_DockerImageName,
Jobs_UserVolumes, Jobs_DataVolumes,
Jobs_Alias)
jobStatus = Jobs.waitForJob(jobId)
self.assertTrue(jobStatus == Jobs.getJobStatus(jobId))
self.assertTrue(jobStatus.get('status') == 32)
job = Jobs.getJobDescription(jobId)
self.assertTrue(job.get('username') == Authentication_loginName)
self.assertTrue(job.get('dockerImageName') == Jobs_DockerImageName)
self.assertTrue(job.get('command') == Jobs_ShellCommand)
self.assertTrue(job.get('submitterDID') == Jobs_Alias)
jobDirectory = job.get('resultsFolderURI')
relativePath = jobDirectory.split('scratch/')[1] + "command.txt"
string = Files.download(fileService,
'scratch',
'',
relativePath,
format="txt",
userVolumeOwner=Authentication_loginName)
string.rstrip("\n")
self.assertTrue(string, job.get('resultsFolderURI'))
Files.deleteUserVolume(fileService, Jobs_RootVolumeName,
Jobs_UserVolumeName)
def test_Files_createDir_upload_dirList_download_download_shareUserVolume(
self):
try:
fileService = Files.getFileServiceFromName(Files_FileServiceName)
os.remove(Files_LocalFileName)
Files.deleteUserVolume(fileService,
Files_RootVolumeName1,
Files_UserVolumeName1,
quiet=True)
Files.deleteUserVolume(fileService,
Files_RootVolumeName1,
Files_UserVolumeName2,
quiet=True)
except:
pass
try:
fileService = Files.getFileServiceFromName(Files_FileServiceName)
Files.createUserVolume(fileService,
"/".join([
Files_RootVolumeName1,
Authentication_loginName,
Files_UserVolumeName1
]),
quiet=False)
Files.createUserVolume(fileService,
"/".join([
Files_RootVolumeName1,
Authentication_loginName,
Files_UserVolumeName2
]),
quiet=False)
Files.createDir(
fileService, "/".join([
Files_RootVolumeName1, Authentication_loginName,
Files_UserVolumeName1, Files_NewDirectoryName1
]))
Files.createDir(
fileService, "/".join([
Files_RootVolumeName2, Authentication_loginName,
Files_UserVolumeName2, Files_NewDirectoryName2
]))
dirList = Files.dirList(fileService,
"/".join([
Files_RootVolumeName1,
Authentication_loginName,
Files_UserVolumeName1,
Files_NewDirectoryName1
]),
level=2)
self.assertTrue(
dirList.get('root').get('name') == Files_NewDirectoryName1)
Files.upload(fileService,
"/".join([
Files_RootVolumeName1, Authentication_loginName,
Files_UserVolumeName1, Files_NewDirectoryName1,
Files_LocalFileName
]),
data=Files_LocalFileContent)
dirList = Files.dirList(fileService,
"/".join([
Files_RootVolumeName1,
Authentication_loginName,
Files_UserVolumeName1,
Files_NewDirectoryName1
]),
level=2)
self.assertTrue(
dirList.get('root').get('files')[0].get('name') ==
Files_LocalFileName)
Files.download(fileService,
"/".join([
Files_RootVolumeName1, Authentication_loginName,
Files_UserVolumeName1, Files_NewDirectoryName1,
Files_LocalFileName
]),
localFilePath=Files_LocalFileName)
with open(Files_LocalFileName, 'r') as myfile:
downloadedFileContent = myfile.read()
assert (downloadedFileContent == Files_LocalFileContent)
Files.delete(
fileService, "/".join([
Files_RootVolumeName1, Authentication_loginName,
Files_UserVolumeName1, Files_NewDirectoryName1,
Files_LocalFileName
]))
dirList = Files.dirList(fileService,
"/".join([
Files_RootVolumeName1,
Authentication_loginName,
Files_UserVolumeName1,
Files_NewDirectoryName1
]),
level=2)
self.assertIsNone(dirList.get('root').get('files'))
Files.upload(fileService,
"/".join([
Files_RootVolumeName1, Authentication_loginName,
Files_UserVolumeName1, Files_NewDirectoryName1,
Files_LocalFileName
]),
localFilePath=Files_LocalFileName,
quiet=False)
Files.move(
fileService, "/".join([
Files_RootVolumeName1, Authentication_loginName,
Files_UserVolumeName1, Files_NewDirectoryName1,
Files_LocalFileName
]), fileService, "/".join([
Files_RootVolumeName2, Authentication_loginName,
Files_UserVolumeName2, Files_NewDirectoryName2,
Files_LocalFileName
]))
Files.shareUserVolume(
fileService,
"/".join([
Files_RootVolumeName2, Authentication_loginName,
Files_UserVolumeName2
]),
sharedWith=Authentication_login_sharedWithName,
type="USER",
allowedActions=["read"])
token1 = Authentication.login(
Authentication_login_sharedWithName,
Authentication_login_sharedWithPassword)
string = Files.download(fileService,
"/".join([
Files_RootVolumeName2,
Authentication_loginName,
Files_UserVolumeName2,
Files_NewDirectoryName2,
Files_LocalFileName
]),
format="txt")
self.assertTrue(string, Files_LocalFileContent)
token1 = Authentication.login(Authentication_loginName,
Authentication_loginPassword)
finally:
try:
os.remove(Files_LocalFileName)
Files.deleteUserVolume(fileService,
"/".join([
Files_RootVolumeName1,
Authentication_loginName,
Files_UserVolumeName1
]),
quiet=True)
Files.deleteUserVolume(fileService,
"/".join([
Files_RootVolumeName1,
Authentication_loginName,
Files_UserVolumeName2
]),
quiet=True)
except:
pass
def test_Files_getUserVolumesInfo(self):
fileService = Files.getFileServiceFromName(Files_FileServiceName)
userVolumesInfo = Files.getUserVolumesInfo(fileService)
self.assertTrue(userVolumesInfo.__len__() > 0)
def test_Files_getFileServiceFromName(self):
fileService = Files.getFileServiceFromName(Files_FileServiceName)
self.assertTrue(fileService.get('name') == Files_FileServiceName)
def test_Files_getFileServicesNames(self):
fileServiceNames = Files.getFileServicesNames()
self.assertTrue(fileServiceNames.__len__() > 0)