def _download_from_scidrive(self,
table_name,
save_to,
max_tries=20,
verbose=True):
"""Check if the table is ready to download and download it once ready"""
tgt_file_path = CASJOBS_CONTAINER + '/' + table_name + '_{}'.format(
self.uname) + '.csv'
count = 0
while True:
list_of_files = sd.directoryList(CASJOBS_CONTAINER)['contents']
for file in list_of_files:
if file['path'].strip() == tgt_file_path:
sd.download(tgt_file_path,
localFilePath='{0}/{1}.csv'.format(
save_to, table_name))
is_deleted = sd.delete(tgt_file_path)
print(
'Successfully downloaded the table csv, deleted in SciDrive: ',
is_deleted)
return
count += 1
if count > max_tries:
break
if verbose:
print('Waiting')
time.sleep(10)
print('could not download file')
def _create_container(self):
"""Clear the scidrive container"""
print('Creating a new container path')
try:
sd.createContainer(CASJOBS_CONTAINER)
except:
pass
time.sleep(3)
def test_SciDrive_publicUrl(self):
try:
responseDelete = SciDrive.delete(SciDrive_Directory)
except:
pass
responseCreate = SciDrive.createContainer(SciDrive_Directory)
url = SciDrive.publicUrl(SciDrive_Directory)
responseDelete = SciDrive.delete(SciDrive_Directory)
isUrl = url.startswith("http")
self.assertEqual(responseCreate, True)
self.assertEqual(isUrl, True)
self.assertEqual(responseDelete, True)
def testSciDrive(df,file_name, token):
df.to_csv(file_name, sep=',')
f=open(file_name,'rb')
data=f.read()
f.close()
container="test_"+str(randint(0,1000000))
SciDrive.createContainer(container,token=token)
sdFile= container+"/"+file_name
SciDrive.upload(sdFile, data)
data1 = SciDrive.download(sdFile,token)
print data1.read()
def uploadSync(payload, target):
if os.path.isfile(str(payload)):
#target.ensureDirExists(str(target))
SciDrive.upload(os.path.join(str(target), os.path.basename(str(payload))), localFilePath=str(payload))
print("Uploading file {0} to SciDrive:{1}".format(os.path.join(str(target),os.path.basename(str(payload))),str(target)))
elif os.path.isdir(str(payload)):
recursiveDirSync(payload, target)
else:
print('This is supposed to be unreachable code - toUploadPath failed in checking input path')
exit()
def test_SciDrive_createContainer_directoryList_delete(self):
try:
responseDelete = SciDrive.delete(SciDrive_Directory)
except:
pass
try:
responseCreate = SciDrive.createContainer(SciDrive_Directory)
self.assertEqual(responseCreate, True)
dirList = SciDrive.directoryList(SciDrive_Directory)
self.assertTrue(dirList["path"].__contains__(SciDrive_Directory))
finally:
responseDelete = SciDrive.delete(SciDrive_Directory)
self.assertEqual(responseDelete, True)
def frame_from_drive(path, grouped=True, replace_9=False):
'''Create DataFrame from SciDrive csv at given path.'''
try:
csv_url = SciDrive.publicUrl(f'metis_project_3/{path}.csv')
except Exception as err:
if str(err) == 'User token is not defined. First log into SciServer.':
token = login()
csv_url = SciDrive.publicUrl(f'metis_project_3/{path}.csv')
else:
raise err
if replace_9:
df = pd.read_csv(csv_url, index_col=0, na_values='-9.999')
else:
df = pd.read_csv(csv_url, index_col=0)
if grouped:
df = grouped_frame(df)
return df
def recursiveDirSync(payload, target):
if os.path.isfile(str(payload)):
print('Error: the directory upload function has been called on a file path {0}'.format(payload))
exit()
elif os.path.isdir(str(payload)):
#target.ensureDirExists(str(target))
dirpath, dirname, filename = next(os.walk(str(payload)))
current = os.path.join(str(target),os.path.basename(dirpath))
if len(filename) == 0:
# problem: this is the most obvious reason for SciDrive.createContainer to fail
# however it could also fail for authentication reasons, connection reasons, etc.
# this problem can be solved when ensureDirExists is functional
try:
print("Creating directory {0} on SciDrive".format(current))
SciDrive.createContainer(current)
except:
print("The folder {0} already exists in SciDrive".format(current))
else:
for local_file in filename:
print("Uploading file {0} to SciDrive:{1}...".format(os.path.join(dirpath,local_file), os.path.join(str(current), local_file)))
try:
SciDrive.upload(os.path.join(str(current), local_file), localFilePath=os.path.join(dirpath,local_file))
except:
print("Upload failed, trying again...")
time.sleep(2)
try:
SciDrive.upload(os.path.join(str(current), local_file), localFilePath=os.path.join(dirpath,local_file))
except:
print("Upload failed twice, trying one more time...")
time.sleep(2)
try:
SciDrive.upload(os.path.join(str(current), local_file), localFilePath=os.path.join(dirpath,local_file))
except:
print("Upload failed three times. Exiting...")
for sub_dir in dirname:
recursiveDirSync(os.path.join(dirpath, sub_dir), current)
else:
print('Input path {0} could not properly be checked'.format(payload))
exit()
def test_SciDrive_upload_download_delete(self):
try:
if (sys.version_info > (3, 0)): #python3
file = open(SciDrive_FileName, "w")
else: #python2
file = open(SciDrive_FileName, "wb")
file.write(SciDrive_FileContent)
file.close()
responseUpload = SciDrive.upload(path=SciDrive_Directory + "/" +
SciDrive_FileName,
localFilePath=SciDrive_FileName)
stringio = SciDrive.download(path=SciDrive_Directory + "/" +
SciDrive_FileName,
format="StringIO")
fileContent = stringio.read()
responseDelete = SciDrive.delete(SciDrive_Directory)
self.assertEqual(responseUpload["path"],
SciDrive_Directory + "/" + SciDrive_FileName)
self.assertEqual(fileContent, SciDrive_FileContent)
self.assertEqual(responseDelete, True)
responseUpload = SciDrive.upload(path=SciDrive_Directory + "/" +
SciDrive_FileName,
data=SciDrive_FileContent)
fileContent = SciDrive.download(path=SciDrive_Directory + "/" +
SciDrive_FileName,
format="text")
responseDelete = SciDrive.delete(SciDrive_Directory)
self.assertEqual(responseUpload["path"],
SciDrive_Directory + "/" + SciDrive_FileName)
self.assertEqual(fileContent, SciDrive_FileContent)
self.assertEqual(responseDelete, True)
finally:
try:
os.remove(SciDrive_FileName)
except:
pass
# *******************************************************************************************************
# SciDrive section:
# *******************************************************************************************************
# In[ ]:
#help(SciDrive)
# In[ ]:
#list content and metadata of top level directory in SciDrive
dirList = SciDrive.directoryList("")
print(dirList)
# In[ ]:
#define name of directory to be created in SciDrive:
SciDrive_Directory = "SciScriptPython"
#define name, path and content of a file to be first created and then uploaded into SciDrive:
SciDrive_FileName = "TestFile.csv"
SciDrive_FilePath = "./TestFile.csv"
SciDrive_FileContent = "Column1,Column2\n4.5,5.5\n"
# In[ ]: