def getJobStatus(jobId):
"""
Shows the status of a job submitted to CasJobs.
:param jobId: id of job (integer)
:return: Returns a dictionary object containing the job status and related metadata. The "Status" field can be equal to 0 (Ready), 1 (Started), 2 (Canceling), 3(Canceled), 4 (Failed) or 5 (Finished). If jobId is the empty string, then returns a list with the statuses of all previous jobs.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the CasJobs API returns an error.
:example: status = CasJobs.getJobStatus(CasJobs.submitJob("select 1"))
.. seealso:: CasJobs.submitJob, CasJobs.waitForJob, CasJobs.cancelJob.
"""
token = Authentication.getToken()
if token is not None and token != "":
taskName = "";
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.CasJobs.getJobStatus"
else:
taskName = "SciScript-Python.CasJobs.getJobStatus"
QueryUrl = Config.CasJobsRESTUri + "/jobs/" + str(jobId) + "?TaskName=" + taskName
headers={'X-Auth-Token': token,'Content-Type': 'application/json'}
postResponse =requests.get(QueryUrl,headers=headers)
if postResponse.status_code != 200:
raise Exception("Error when getting the status of job " + str(jobId) + ".\nHttp Response from CasJobs API returned status code " + str(postResponse.status_code) + ":\n" + postResponse.content.decode());
return json.loads(postResponse.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
def delete(fileService, path, quiet=True):
"""
Deletes a directory or file in the File System.
:param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices().
:param path: String defining the path (in the remote fileService) of the file or directory to be deleted, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/fileToBeDeleted.txt.
:param quiet: If set to False, it will throw an error if the file does not exist. If set to True. it will not throw an error.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error.
:example: fileServices = Files.getFileServices(); isDeleted = Files.delete("/myUselessFile.txt","persistent","myUserName", fileServices[0]);
.. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList
"""
token = Authentication.getToken()
if token is not None and token != "":
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.Files.delete"
else:
taskName = "SciScript-Python.Files.delete"
(rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path);
url = __getFileServiceAPIUrl(fileService) + "api/data/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName
headers = {'X-Auth-Token': token}
res = requests.delete(url, headers=headers)
if res.status_code >= 200 and res.status_code < 300:
pass;
else:
raise Exception("Error when deleting '" + str(path) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode());
else:
raise Exception("User token is not defined. First log into SciServer.")
def cancelJob(jobId):
"""
Cancels a job already submitted.
:param jobId: id of job (integer)
:return: Returns True if the job was canceled successfully.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the CasJobs API returns an error.
:example: response = CasJobs.cancelJob(CasJobs.submitJob("select 1"))
.. seealso:: CasJobs.submitJob, CasJobs.waitForJob.
"""
token = Authentication.getToken()
if token is not None and token != "":
taskName = "";
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.CasJobs.cancelJob"
else:
taskName = "SciScript-Python.CasJobs.cancelJob"
QueryUrl = Config.CasJobsRESTUri + "/jobs/" + str(jobId) + "?TaskName=" + taskName
headers={'X-Auth-Token': token,'Content-Type': 'application/json'}
response =requests.delete(QueryUrl,headers=headers)
if response.status_code != 200:
raise Exception("Error when canceling job " + str(jobId) + ".\nHttp Response from CasJobs API returned status code " + str(response.status_code) + ":\n" + response.content.decode());
return True;#json.loads(response.content)
else:
raise Exception("User token is not defined. First log into SciServer.")
def cancelJob(jobId):
"""
Cancels the execution of a job.
:param jobId: Id of the job (integer)
:raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error.
:example: job = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); isCanceled = Jobs.cancelJob(job.get('id'));
.. seealso:: Jobs.submitNotebookJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains.
"""
token = Authentication.getToken()
if token is not None and token != "":
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.Jobs.cancelJob"
else:
taskName = "SciScript-Python.Jobs.cancelJob"
url = Config.RacmApiURL + "/jobm/rest/jobs/" + str(
jobId) + "/cancel?TaskName=" + taskName
headers = {'X-Auth-Token': token, "Content-Type": "application/json"}
res = requests.post(url, headers=headers, stream=True)
if res.status_code != 200:
raise Exception(
"Error when getting from JOBM API the job status of jobId=" +
str(jobId) +
".\nHttp Response from JOBM API returned status code " +
str(res.status_code) + ":\n" + res.content.decode())
else:
pass
else:
raise Exception("User token is not defined. First log into SciServer.")
def getPandasDataFrameFromQuery(queryString, context="MyDB"):
"""
Executes a casjobs quick query and returns the result as a pandas dataframe object with an index (http://pandas.pydata.org/pandas-docs/stable/).
:param queryString: sql query (string)
:param context: database context (string)
:return: Returns a Pandas dataframe containing the results table.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the CasJobs API returns an error.
:example: df = CasJobs.getPandasDataFrameFromQuery("select 1 as foo", context="MyDB")
.. seealso:: CasJobs.submitJob, CasJobs.getJobStatus, CasJobs.executeQuery, CasJobs.writeFitsFileFromQuery, CasJobs.getNumpyArrayFromQuery
"""
try:
if Config.isSciServerComputeEnvironment():
task.name = "Compute.SciScript-Python.CasJobs.getPandasDataFrameFromQuery"
else:
task.name = "SciScript-Python.CasJobs.getPandasDataFrameFromQuery"
cvsResponse = executeQuery(queryString, context=context,format="readable")
#if the index column is not specified then it will add it's own column which causes problems when uploading the transformed data
dataFrame = pandas.read_csv(cvsResponse, index_col=None)
return dataFrame
except Exception as e:
raise e
def writeFitsFileFromQuery(fileName, queryString, context="MyDB"):
"""
Executes a quick CasJobs query and writes the result to a local Fits file (http://www.stsci.edu/institute/software_hardware/pyfits).
:param fileName: path to the local Fits file to be created (string)
:param queryString: sql query (string)
:param context: database context (string)
:return: Returns True if the fits file was created successfully.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the CasJobs API returns an error.
:example: CasJobs.writeFitsFileFromQuery("/home/user/myFile.fits","select 1 as foo")
.. seealso:: CasJobs.submitJob, CasJobs.getJobStatus, CasJobs.executeQuery, CasJobs.getPandasDataFrameFromQuery, CasJobs.getNumpyArrayFromQuery
"""
try:
if Config.isSciServerComputeEnvironment():
task.name = "Compute.SciScript-Python.CasJobs.writeFitsFileFromQuery"
else:
task.name = "SciScript-Python.CasJobs.writeFitsFileFromQuery"
bytesio = executeQuery(queryString, context=context, format="fits")
theFile = open(fileName, "w+b")
theFile.write(bytesio.read())
theFile.close()
return True
except Exception as e:
raise e
def uploadPandasDataFrameToTable(dataFrame, tableName, context="MyDB"):
"""
Uploads a pandas dataframe object into a CasJobs table. If the dataframe contains a named index, then the index will be uploaded as a column as well.
:param dataFrame: Pandas data frame containg the data (pandas.core.frame.DataFrame)
:param tableName: name of CasJobs table to be created.
:param context: database context (string)
:return: Returns True if the dataframe was uploaded successfully.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the CasJobs API returns an error.
:example: response = CasJobs.uploadPandasDataFrameToTable(CasJobs.getPandasDataFrameFromQuery("select 1 as foo", context="MyDB"), "NewTableFromDataFrame")
.. seealso:: CasJobs.uploadCSVDataToTable
"""
try:
if Config.isSciServerComputeEnvironment():
task.name = "Compute.SciScript-Python.CasJobs.uploadPandasDataFrameToTable"
else:
task.name = "SciScript-Python.CasJobs.uploadPandasDataFrameToTable"
if dataFrame.index.name is not None and dataFrame.index.name != "":
sio = dataFrame.to_csv().encode("utf8")
else:
sio = dataFrame.to_csv(index_label=False, index=False).encode("utf8")
return uploadCSVDataToTable(sio, tableName, context)
except Exception as e:
raise e
def getNumpyArrayFromQuery(queryString, context="MyDB"):
"""
Executes a casjobs query and returns the results table as a Numpy array (http://docs.scipy.org/doc/numpy/).
:param queryString: sql query (string)
:param context: database context (string)
:return: Returns a Numpy array storing the results table.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the CasJobs API returns an error.
:example: array = CasJobs.getNumpyArrayFromQuery("select 1 as foo", context="MyDB")
.. seealso:: CasJobs.submitJob, CasJobs.getJobStatus, CasJobs.executeQuery, CasJobs.writeFitsFileFromQuery, CasJobs.getPandasDataFrameFromQuery
"""
try:
if Config.isSciServerComputeEnvironment():
task.name = "Compute.SciScript-Python.CasJobs.getNumpyArrayFromQuery"
else:
task.name = "SciScript-Python.CasJobs.getNumpyArrayFromQuery"
dataFrame = getPandasDataFrameFromQuery(queryString, context)
return dataFrame.as_matrix()
except Exception as e:
raise e
def getTables(context="MyDB"):
"""
Gets the names, size and creation date of all tables in a database context that the user has access to.
:param context: database context (string)
:return: The result is a json object with format [{"Date":seconds,"Name":"TableName","Rows":int,"Size",int},..]
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the CasJobs API returns an error.
:example: tables = CasJobs.getTables("MyDB")
.. seealso:: CasJobs.getSchemaName
"""
token = Authentication.getToken()
if token is not None and token != "":
taskName = "";
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.CasJobs.getTables"
else:
taskName = "SciScript-Python.CasJobs.getTables"
TablesUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/Tables" + "?TaskName=" + taskName
headers={'X-Auth-Token': token,'Content-Type': 'application/json'}
getResponse = requests.get(TablesUrl,headers=headers)
if getResponse.status_code != 200:
raise Exception("Error when getting table description from database context " + str(context) + ".\nHttp Response from CasJobs API returned status code " + str(getResponse.status_code) + ":\n" + getResponse.content.decode());
jsonResponse = json.loads(getResponse.content.decode())
return jsonResponse
else:
raise Exception("User token is not defined. First log into SciServer.")
def setToken(_token):
"""
Sets the SciServer authentication token of the user in the variable Authentication.token.value, as well as in the python instance argument variable "--ident".
:param _token: Sciserver's authentication token for the user (string)
:example: Authentication.setToken('myToken')
.. seealso:: Authentication.getKeystoneUserWithToken, Authentication.login, Authentication.getToken, Authentication.token.
"""
if _token is None:
warnings.warn("Authentication token is being set with a None value.",
Warning,
stacklevel=2)
if _token == "":
warnings.warn("Authentication token is being set as an empty string.",
Warning,
stacklevel=2)
if Config.isSciServerComputeEnvironment():
warnings.warn(
"Authentication token cannot be set to arbitary value when inside SciServer-Compute environment.",
Warning,
stacklevel=2)
else:
token.value = _token
found = False
ident = identArgIdentifier()
for arg in sys.argv:
if (arg.startswith(ident)):
sys.argv.remove(arg)
sys.argv.append(ident + _token)
found = True
if not found:
sys.argv.append(ident + _token)
def getRDBComputeDomains():
"""
Gets a list of all registered Relational Database (RDB) compute domains that the user has access to.
:return: a list of dictionaries, each one containing the definition of an RDB compute domain.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error.
:example: rdbComputeDomains = Jobs.getRDBComputeDomains();
.. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob
"""
token = Authentication.getToken()
if token is not None and token != "":
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.Jobs.getRDBComputeDomains"
else:
taskName = "SciScript-Python.Jobs.getRDBComputeDomains"
url = Config.RacmApiURL + "/jobm/rest/computedomains/rdb?TaskName=" + taskName
headers = {'X-Auth-Token': token, "Content-Type": "application/json"}
res = requests.get(url, headers=headers, stream=True)
if res.status_code != 200:
raise Exception(
"Error when getting RDB Compute Domains from JOBM API.\nHttp Response from JOBM API returned status code "
+ str(res.status_code) + ":\n" + res.content.decode())
else:
return json.loads(res.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
def getSchemaName():
"""
Returns the WebServiceID that identifies the schema for a user in MyScratch database with CasJobs.
:return: WebServiceID of the user (string).
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the CasJobs API returns an error.
:example: wsid = CasJobs.getSchemaName()
.. seealso:: CasJobs.getTables.
"""
token = Authentication.getToken()
if token is not None and token != "":
keystoneUserId = Authentication.getKeystoneUserWithToken(token).id
taskName = ""
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.CasJobs.getSchemaName"
else:
taskName = "SciScript-Python.CasJobs.getSchemaName"
usersUrl = Config.CasJobsRESTUri + "/users/" + keystoneUserId + "?TaskName=" + taskName
headers={'X-Auth-Token': token,'Content-Type': 'application/json'}
getResponse = requests.get(usersUrl,headers=headers)
if getResponse.status_code != 200:
raise Exception("Error when getting schema name. Http Response from CasJobs API returned status code " + str(getResponse.status_code) + ":\n" + getResponse.content.decode());
jsonResponse = json.loads(getResponse.content.decode())
return "wsid_" + str(jsonResponse["WebServicesId"])
else:
raise Exception("User token is not defined. First log into SciServer.")
def download(path, format="text", localFilePath=""):
"""
Downloads a file (directory) from SciDrive into the local file system, or returns the file conetent as an object in several formats.
:param path: path of the file (or directory) in SciDrive.
:param format: type of the returned object. Can be "StringIO" (data.StringIO object containing readable text), "BytesIO" (data.BytesIO object containing readable binary data), "response" ( the HTTP response as an object of class requests.Response) or "text" (a text string). If the parameter 'localFilePath' is defined, then the 'format' parameter is not used and the file is downloaded to the local file system instead.
:param localFilePath: local path of the file to be downloaded. If 'localFilePath' is defined, then the 'format' parameter is not used.
:return: If the 'localFilePath' parameter is defined, then it will return True when the file is downloaded successfully in the local file system. If the 'localFilePath' is not defined, then the type of the returned object depends on the value of the 'format' parameter (either data.StringIO, data.BytesIO, requests.Response or string).
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SciDrive API returns an error.
:example: csvString = SciDrive.download("path/to/SciDrive/file.csv", format="text");
.. seealso:: SciDrive.upload
"""
token = Authentication.getToken()
if token is not None and token != "":
taskName = ""
if Config.isSciServerComputeEnvironment():
task.name = "Compute.SciScript-Python.SciDrive.download"
else:
task.name = "SciScript-Python.SciDrive.download"
fileUrl = publicUrl(path)
res = requests.get(fileUrl, stream=True)
if res.status_code != 200:
raise Exception(
"Error when downloading SciDrive file " + str(path) +
".\nHttp Response from SciDrive API returned status code " +
str(res.status_code) + ":\n" + res.content.decode())
if localFilePath is not None and localFilePath != "":
bytesio = BytesIO(res.content)
theFile = open(localFilePath, "w+b")
theFile.write(bytesio.read())
theFile.close()
return True
else:
if format is not None and format != "":
if format == "StringIO":
return StringIO(res.content.decode())
if format == "text":
return res.content.decode()
elif format == "BytesIO":
return BytesIO(res.content)
elif format == "response":
return res
else:
raise Exception(
"Unknown format '" + format +
"' when trying to download SciDrive file " +
str(path) + ".\n")
else:
raise Exception("Wrong format parameter value\n")
else:
raise Exception("User token is not defined. First log into SciServer.")
def getToken():
"""
Returns the SciServer authentication token of the user. First, will try to return Authentication.token.value.
If Authentication.token.value is not set, Authentication.getToken will try to return the token value in the python instance argument variable "--ident".
If this variable does not exist, will try to return the token stored in Config.KeystoneTokenFilePath. Will return a None value if all previous steps fail.
:return: authentication token (string)
:example: token = Authentication.getToken()
.. seealso:: Authentication.getKeystoneUserWithToken, Authentication.login, Authentication.setToken, Authentication.token.
"""
try:
if Config.isSciServerComputeEnvironment():
tokenFile = Config.KeystoneTokenPath; # '/home/idies/keystone.token'
if os.path.isfile(tokenFile):
with open(tokenFile, 'r') as f:
_token = f.read().rstrip('\n')
if _token is not None and _token != "":
token.value = _token;
found = False
ident = identArgIdentifier()
for arg in sys.argv:
if (arg.startswith(ident)):
sys.argv.remove(arg)
sys.argv.append(ident + _token)
found = True
if not found:
sys.argv.append(ident + _token)
return _token
else:
warnings.warn("In Authentication.getToken: Cannot find token in system token file " + str(Config.KeystoneTokenPath) + ".", Warning, stacklevel=2)
return None;
else:
warnings.warn("In Authentication.getToken: Cannot find system token file " + str(Config.KeystoneTokenPath) + ".", Warning, stacklevel=2)
return None;
else:
if token.value is not None:
return token.value
else:
_token = None
ident = identArgIdentifier()
for arg in sys.argv:
if (arg.startswith(ident)):
_token = arg[len(ident):]
if _token is not None and _token != "":
token.value = _token
return _token
else:
warnings.warn("In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.", Warning, stacklevel=2)
return None;
except Exception as e:
raise e;
def login(UserName=None, Password=None):
"""
Logs the user into SciServer and returns the authentication token.
This function is useful when SciScript-Python library methods are executed outside the SciServer-Compute environment.
In this case, the session authentication token does not exist (and therefore can't be automatically recognized),
so the user has to use Authentication.login in order to log into SciServer manually and get the authentication token.
Authentication.login also sets the token value in the python instance argument variable "--ident", and as the local object Authentication.token (of class Authentication.Token).
:param UserName: name of the user (string). If not set, then a user name prompt will show.
:param Password: password of the user (string). If not set, then a password prompt will show.
:return: authentication token (string)
:raises: Throws an exception if the HTTP request to the Authentication URL returns an error.
:example: token1 = Authentication.login(); token2 = Authentication.login('loginName','loginPassword');
.. seealso:: Authentication.getKeystoneUserWithToken, Authentication.getToken, Authentication.setToken, Authentication.token.
"""
taskName = ""
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.Authentication.Login"
else:
taskName = "SciScript-Python.Authentication.Login"
loginURL = Config.AuthenticationURL + "?TaskName=" + taskName
userName = UserName
password = Password
if userName is None:
userName = getpass.getpass(prompt="Enter SciServer user name: ")
if password is None:
password = getpass.getpass(prompt="Enter SciServer password: "******"auth": {
"identity": {
"password": {
"user": {
"name": userName,
"password": password
}
}
}
}
}
data = json.dumps(authJson).encode()
headers = {'Content-Type': "application/json"}
postResponse = requests.post(loginURL, data=data, headers=headers)
if postResponse.status_code != 200:
raise Exception(
"Error when logging in. Http Response from the Authentication API returned status code "
+ str(postResponse.status_code) + ":\n" +
postResponse.content.decode())
_token = postResponse.headers['X-Subject-Token']
setToken(_token)
return _token
def rectangularSearch(min_ra, max_ra, min_dec, max_dec, coordType="equatorial", whichPhotometry="optical", limit="10", dataRelease=None):
"""
Runs a query in the SDSS database that searches for all objects within a certain rectangular box defined on the the sky, and retrieves the result table as a Panda's dataframe.\n
:param min_ra: Minimum value of Right Ascension coordinate that defines the box boundaries on the sky.\n
:param max_ra: Maximum value of Right Ascension coordinate that defines the box boundaries on the sky.\n
:param min_dec: Minimum value of Declination coordinate that defines the box boundaries on the sky.\n
:param max_dec: Maximum value of Declination coordinate that defines the box boundaries on the sky.\n
:param coordType: Type of celestial coordinate system. Can be set to "equatorial" or "galactic".\n
:param whichPhotometry: Type of retrieved data. Can be set to "optical" or "infrared".\n
:param limit: Maximum number of rows in the result table (string). If set to "0", then the function will return all rows.\n
:param dataRelease: SDSS data release string. Example: dataRelease='DR13'. Default value already set in SciServer.Config.DataRelease
:return: Returns the results table as a Pandas data frame.
:raises: Throws an exception if the HTTP request to the SkyServer API returns an error.
:example: df = SkyServer.rectangularSearch(min_ra=258.2, max_ra=258.3, min_dec=64,max_dec=64.1)
.. seealso:: SkyServer.sqlSearch, SkyServer.radialSearch.
"""
if(dataRelease):
if dataRelease != "":
url = Config.SkyServerWSurl + '/' + dataRelease + '/SkyServerWS/SearchTools/RectangularSearch?'
else:
url = Config.SkyServerWSurl + '/SkyServerWS/SearchTools/RectangularSearch?'
else:
if Config.DataRelease != "":
url = Config.SkyServerWSurl + '/' + Config.DataRelease + '/SkyServerWS/SearchTools/RectangularSearch?'
else:
url = Config.SkyServerWSurl + '/SkyServerWS/SearchTools/RectangularSearch?'
url = url + 'format=csv&'
url = url + 'min_ra=' + str(min_ra) + '&'
url = url + 'max_ra=' + str(max_ra) + '&'
url = url + 'min_dec=' + str(min_dec) + '&'
url = url + 'max_dec=' + str(max_dec) + '&'
url = url + 'coordType=' + coordType + '&'
url = url + 'whichPhotometry=' + whichPhotometry + '&'
url = url + 'limit=' + limit + '&'
if Config.isSciServerComputeEnvironment():
url = url + "TaskName=Compute.SciScript-Python.SkyServer.rectangularSearch&"
else:
url = url + "TaskName=SciScript-Python.SkyServer.rectangularSearch&"
#url = urllib.quote_plus(url)
acceptHeader = "text/plain"
headers = {'Content-Type': 'application/json', 'Accept': acceptHeader}
token = Authentication.getToken()
if token is not None and token != "":
headers['X-Auth-Token'] = token
response = requests.get(url,headers=headers, stream=True)
if response.status_code != 200:
raise Exception("Error when executing a rectangular search.\nHttp Response from SkyServer API returned status code " + str(response.status_code) + ":\n" + response.content.decode());
r=response.content.decode();
return pandas.read_csv(StringIO(r), comment='#', index_col=None)
def move(fileService, path, destinationFileService, destinationPath, replaceExisting=True, doCopy=True):
"""
Moves or copies a file or folder.
:param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices().
:param path: String defining the origin path (in the remote fileService) of the file or directory to be copied/moved, starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/fileToBeMoved.txt or dataVolume/fileToBeMoved.txt
:param destinationFileService: name of fileService (string), or object (dictionary) that defines a destination file service (where the file is moved/copied into). A list of these kind of objects available to the user is returned by the function Files.getFileServices().
:param destinationRelativePath: String defining the destination path (in the remote destinationFileService) of the file or directory to be copied/moved, starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/recentlyMovedFile.txt or dataVolume/recentlyMovedFile.txt}
:param replaceExisting: If set to False, it will throw an error if the file already exists, If set to True, it will not throw and eeror in that case.
:param doCopy: if set to True, then it will copy the file or folder. If set to False, then the file or folder will be moved.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error.
:example: fileServices = Files.getFileServices(); Files.move(fileServices[0], "Storage/myUserName/persistent/myFile.txt", fileServices[0], "Storage/myUserName/persistent/myFile2.txt")
.. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.upload, Files.dirList
"""
token = Authentication.getToken()
if token is not None and token != "":
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.Files.Move"
else:
taskName = "SciScript-Python.Files.Move"
if type(fileService) == str:
destinationFileServiceName = destinationFileService
fileService = getFileServiceFromName(fileService)
else:
destinationFileServiceName = destinationFileService['name']
if type(destinationFileService) == str:
destinationFileService = getFileServiceFromName(destinationFileService)
(topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService);
(destinationTopVolume, destinationUserVolumeOwner, destinationUserVolume, destinationRelativePath, isDestinationTopVolumeARootVolume) = splitPath(destinationPath, destinationFileService);
if isTopVolumeARootVolume:
url = __getFileServiceAPIUrl(fileService) + "api/data/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?replaceExisting=" + str(replaceExisting) + "&doCopy=" + str(doCopy) + "&TaskName=" + taskName;
jsonDict = {'destinationPath': destinationRelativePath,
'destinationRootVolume': destinationTopVolume,
'destinationUserVolume': destinationUserVolume,
'destinationOwnerName': destinationUserVolumeOwner,
'destinationFileService': destinationFileServiceName};
else:
url = __getFileServiceAPIUrl(fileService) + "api/data/" + topVolume + "/" + relativePath + "?replaceExisting=" + str(replaceExisting) + "&doCopy=" + str(doCopy) + "&TaskName=" + taskName;
jsonDict = {'destinationPath': destinationRelativePath,
'destinationDataVolume': destinationTopVolume,
'destinationFileService': destinationFileServiceName};
headers = {'X-Auth-Token': token, "Content-Type": "application/json"}
res = requests.put(url, stream=True, headers=headers, json=jsonDict)
if res.status_code < 200 or res.status_code >= 300:
raise Exception("Error when moving '" + str(path) + "' in file service '" + str(fileService.get("name")) + "' to '" + str(destinationPath) + "' in file service '" + str(destinationFileService.get("name")) + "'. \nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode());
else:
raise Exception("User token is not defined. First log into SciServer.")
def shareUserVolume(fileService,
path,
sharedWith,
allowedActions,
type="USER"):
"""
Shares a user volume with another user or group
:param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices().
:param path: String defining the path (in the remote fileService) of the user volume to be shared, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume.
:param sharedWith: name (string) of user or group that the user volume is shared with.
:param allowedActions: array of strings defining actions the user or group is allowed to do with respect to the shared user volume. E.g.: ["read","write","grant","delete"]. The "grant" action means that the user or group can also share the user volume with another user or group. The "delete" action meand ability to delete the user volume (use with care).
:param type: type (string) of the entity defined by the "sharedWith" parameter. Can be set to "USER" or "GROUP".
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error.
:example: fileServices = Files.getFileServices(); Files.shareUserVolume(getFileServices[0], "Storage/myUserName/myUSerVolume", "userName", ["read","write"], type="USER");
.. seealso:: Files.getFileServices(), Files.getFilrmsieServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList
"""
token = Authentication.getToken()
if token is not None and token != "":
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.Files.ShareUserVolume"
else:
taskName = "SciScript-Python.Files.ShareUserVolume"
if type(fileService) == str:
fileService = getFileServiceFromName(fileService)
(rootVolume, userVolumeOwner, userVolume, relativePath,
isTopVolumeARootVolume) = splitPath(path, fileService)
data = [{
'name': sharedWith,
'type': type,
'allowedActions': allowedActions
}]
body = json.dumps(data).encode()
url = __getFileServiceAPIUrl(
fileService
) + "api/share/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?TaskName=" + taskName
headers = {'X-Auth-Token': token, 'Content-Type': 'application/json'}
res = requests.patch(url, headers=headers, data=body)
if res.status_code >= 200 and res.status_code < 300:
pass
else:
raise Exception(
"Error when sharing userVolume '" + str(path) +
"' in file service '" + str(fileService.get('name')) +
"'.\nHttp Response from FileService API returned status code "
+ str(res.status_code) + ":\n" + res.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
def getJobsList(top=10, open=None, start=None, end=None, type='all'):
"""
Gets the list of Jobs submitted by the user.
:param top: top number of jobs (integer) returned. If top=None, then all jobs are returned.
:param open: If set to 'True', then only returns jobs that have not finished executing and wrapped up (status <= FINISHED). If set to 'False' then only returnes jobs that are still running. If set to 'None', then returns both finished and unfinished jobs.
:param start: The earliest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to 'None', then there is no lower bound on date.
:param end: The latest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to 'None', then there is no upper bound on date.
:param type: type (string) of jobs returned. Can take values of 'rdb' (for returning only relational database jobs), 'docker' (for returning only Docker jobs) and 'all' (all job types are returned).
:return: a list of dictionaries, each one containing the definition of a submitted job.
:raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error.
:example: jobs = Jobs.getJobsList(top=2);
.. seealso:: Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob
"""
token = Authentication.getToken()
if token is not None and token != "":
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.Jobs.getJobsList"
else:
taskName = "SciScript-Python.Jobs.getJobsList"
topString = ("top=" + str(top) + "&" if top != None else "")
startString = ("start=" + str(start) + "&" if start != None else "")
endString = ("end=" + str(end) + "&" if end != None else "")
if open is None:
openString = ""
else:
if open is True:
openString = "open=true&"
else:
openString = "open=false&"
url = Config.RacmApiURL + "/jobm/rest/jobs?"
if (type == 'rdb'):
url = Config.RacmApiURL + "/jobm/rest/rdbjobs?"
if (type == 'docker'):
url = Config.RacmApiURL + "/jobm/rest/dockerjobs?"
url = url + topString + startString + endString + "TaskName=" + taskName
headers = {'X-Auth-Token': token, "Content-Type": "application/json"}
res = requests.get(url, headers=headers, stream=True)
if res.status_code != 200:
raise Exception(
"Error when getting list of jobs from JOBM API.\nHttp Response from JOBM API returned status code "
+ str(res.status_code) + ":\n" + res.content.decode())
else:
return json.loads(res.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
def dirList(fileService, path, level=1, options=''):
"""
Lists the contents of a directory.
:param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices().
:param path: String defining the path (in the remote file service) of the directory to be listed, starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/directoryToBeListed or dataVolume/directoryToBeListed
:param level: amount (int) of listed directory levels that are below or at the same level to that of the relativePath.
:param options: string of file filtering options.
:return: dictionary containing the directory listing.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error.
:example: fileServices = Files.getFileServices(); dirs = Files.dirList(fileServices[0], "Storage/myUserName/persistent/", level=2);
.. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.createDir
"""
token = Authentication.getToken()
if token is not None and token != "":
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.Files.dirList"
else:
taskName = "SciScript-Python.Files.dirList"
if type(fileService) == str:
fileService = getFileServiceFromName(fileService)
(topVolume, userVolumeOwner, userVolume, relativePath,
isTopVolumeARootVolume) = splitPath(path, fileService)
if isTopVolumeARootVolume:
url = __getFileServiceAPIUrl(
fileService
) + "api/jsontree/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?options=" + options + "&level=" + str(
level) + "&TaskName=" + taskName
else:
url = __getFileServiceAPIUrl(
fileService
) + "api/jsontree/" + topVolume + "/" + relativePath + "?options=" + options + "&level=" + str(
level) + "&TaskName=" + taskName
headers = {'X-Auth-Token': token}
res = requests.get(url, headers=headers)
if res.status_code >= 200 and res.status_code < 300:
return json.loads(res.content.decode())
else:
raise Exception(
"Error when listing contents of '" + str(path) +
"'.\nHttp Response from FileService API returned status code "
+ str(res.status_code) + ":\n" + res.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
def uploadTable(uploadData, tableName, datasetName="MyDB", format="csv"):
"""
Uploads a data table into a database (more info in http://www.voservices.net/skyquery).
:param uploadData: data table, for now accepted in CSV string format.
:param tableName: name of table (string) within dataset.
:param datasetName: name of dataset or database context (string).
:param format: format of the 'data' parameter. Set to 'csv' for now.
:return: returns True if the table was uploaded successfully.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error.
:example: result = SkyQuery.uploadTable("Column1,Column2\\n4.5,5.5\\n", tableName="myTable", datasetName="MyDB", format="csv")
.. seealso:: SkyQuery.listQueues, SkyQuery.listAllDatasets, SkyQuery.getDatasetInfo, SkyQuery.listDatasetTables, SkyQuery.getTableInfo, SkyQuery.getTable, SkyQuery.submitJob
"""
token = Authentication.getToken()
if token is not None and token != "":
taskName = ""
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.SkyQuery.uploadTable"
else:
taskName = "SciScript-Python.SkyQuery.uploadTable"
url = Config.SkyQueryUrl + '/Data.svc/' + datasetName + '/' + tableName + "?TaskName=" + taskName
ctype = ""
if format == "csv":
ctype = 'text/csv'
else:
raise Exception("Unknown format '" + format +
"' when trying to upload data in SkyQuery.\n")
headers = {'Content-Type': ctype, 'Accept': 'application/json'}
headers['X-Auth-Token'] = token
#url = urllib.quote_plus(url)
response = requests.put(url,
data=uploadData,
headers=headers,
stream=True)
if response.status_code == 200:
return (True)
else:
raise Exception(
"Error when uploading data to table " + str(tableName) +
" in dataset " + str(datasetName) +
".\nHttp Response from SkyQuery API returned status code " +
str(response.status_code) + ":\n" + response.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
def uploadCSVDataToTable(csvData, tableName, context="MyDB"):
"""
Uploads CSV data into a CasJobs table.
:param csvData: a CSV table in string format.
:param tableName: name of CasJobs table to be created.
:param context: database context (string)
:return: Returns True if the csv data was uploaded successfully.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the CasJobs API returns an error.
:example: csv = CasJobs.getPandasDataFrameFromQuery("select 1 as foo", context="MyDB").to_csv().encode("utf8"); response = CasJobs.uploadCSVDataToTable(csv, "NewTableFromDataFrame")
.. seealso:: CasJobs.uploadPandasDataFrameToTable
"""
token = Authentication.getToken()
if token is not None and token != "":
#if (Config.executeMode == "debug"):
# print "Uploading ", sys.getsizeof(CVSdata), "bytes..."
taskName = ""
if task.name is not None:
taskName = task.name
task.name = None
else:
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.CasJobs.uploadCSVDataToTable"
else:
taskName = "SciScript-Python.CasJobs.uploadCSVDataToTable"
tablesUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/Tables/" + tableName + "?TaskName=" + taskName
headers = {}
headers['X-Auth-Token'] = token
postResponse = requests.post(tablesUrl,
data=csvData,
headers=headers,
stream=True)
if postResponse.status_code != 200:
raise Exception(
"Error when uploading CSV data into CasJobs table " +
tableName +
".\nHttp Response from CasJobs API returned status code " +
str(postResponse.status_code) + ":\n" +
postResponse.content.decode())
return True
else:
raise Exception("User token is not defined. First log into SciServer.")
def upload(fileService, path, data="", localFilePath=None, quiet=True):
"""
Uploads data or a local file into a path defined in the file system.
:param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices().
:param path: path (in the remote file service) to the destination file (string), starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/destinationFile.txt or dataVolume/destinationFile.txt
:param data: string containing data to be uploaded, in case localFilePath is not set.
:param localFilePath: path to a local file to be uploaded (string),
:param userVolumeOwner: name (string) of owner of the userVolume. Can be left undefined if requester is the owner of the user volume.
:param quiet: If set to False, it will throw an error if the file already exists. If set to True. it will not throw an error.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error.
:example: fileServices = Files.getFileServices(); Files.upload(fileServices[0], "myRootVolume/myUserName/myUserVolume/myUploadedFile.txt", None, localFilePath="/myFile.txt");
.. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.download, Files.dirList
"""
token = Authentication.getToken()
if token is not None and token != "":
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.Files.UploadFile"
else:
taskName = "SciScript-Python.Files.UploadFile"
if type(fileService) == str:
fileService = getFileServiceFromName(fileService)
(topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService);
if isTopVolumeARootVolume:
url = __getFileServiceAPIUrl(fileService) + "api/file/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName
else:
url = __getFileServiceAPIUrl(fileService) + "api/file/" + topVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName
headers = {'X-Auth-Token': token}
if localFilePath is not None and localFilePath != "":
with open(localFilePath, "rb") as file:
res = requests.put(url, data=file, headers=headers, stream=True)
else:
if data != None:
res = requests.put(url, data=data, headers=headers, stream=True)
else:
raise Exception("Error: No local file or data specified for uploading.");
if res.status_code >= 200 and res.status_code < 300:
pass;
else:
raise Exception("Error when uploading file to '" + str(path) + "' in file service '" + str(fileService.get('name')) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode());
else:
raise Exception("User token is not defined. First log into SciServer.")
def createDir(fileService, path, quiet=True):
"""
Create a directory.
:param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices().
:param path: path (in the remote file service) to the directory (string), starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/directory or dataVolume/directory
:param quiet: If set to False, it will throw an error if the directory already exists. If set to True. it will not throw an error.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error.
:example: fileServices = Files.getFileServices(); Files.createDir(fileServices[0], "Storage/myUserName/persistent/myNewDir");
.. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList
"""
token = Authentication.getToken()
if token is not None and token != "":
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.Files.createDir"
else:
taskName = "SciScript-Python.Files.createDir"
if type(fileService) == str:
fileService = getFileServiceFromName(fileService)
(topVolume, userVolumeOwner, userVolume, relativePath,
isTopVolumeARootVolume) = splitPath(path, fileService)
if isTopVolumeARootVolume:
url = __getFileServiceAPIUrl(
fileService
) + "api/folder/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(
quiet) + "&TaskName=" + taskName
else:
url = __getFileServiceAPIUrl(
fileService
) + "api/folder/" + topVolume + "/" + relativePath + "?quiet=" + str(
quiet) + "&TaskName=" + taskName
headers = {'X-Auth-Token': token}
res = requests.put(url, headers=headers)
if res.status_code >= 200 and res.status_code < 300:
pass
else:
raise Exception(
"Error when creating directory '" + str(path) +
"' in file service '" + str(fileService.get('name')) +
"'.\nHttp Response from FileService API returned status code "
+ str(res.status_code) + ":\n" + res.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
def upload(path, data="", localFilePath=""):
"""
Uploads data or a local file into a SciDrive directory.
:param path: desired file path in SciDrive (string).
:param data: data to be uploaded into SciDrive. If the 'localFilePath' parameter is set, then the local file will be uploaded instead.
:param localFilePath: path to the local file to be uploaded (string).
:return: Returns an object with the attributes of the uploaded file.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SciDrive API returns an error.
:example: response = SciDrive.upload("/SciDrive/path/to/file.csv", localFilePath="/local/path/to/file.csv")
.. seealso:: SciDrive.createContainer
"""
token = Authentication.getToken()
if token is not None and token != "":
taskName = ""
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.SciDrive.upload"
else:
taskName = "SciScript-Python.SciDrive.upload"
url = Config.SciDriveHost + '/vospace-2.0/1/files_put/dropbox/' + path + "?TaskName=" + taskName
headers = {'X-Auth-Token': token}
if (localFilePath != ""):
with open(localFilePath, "rb") as file:
res = requests.put(url,
data=file,
headers=headers,
stream=True)
else:
res = requests.put(url, data=data, headers=headers, stream=True)
if res.status_code != 200:
if (localFilePath != None):
raise Exception(
"Error when uploading local file " + str(localFilePath) +
" to SciDrive path " + str(path) +
".\nHttp Response from SciDrive API returned status code "
+ str(res.status_code) + ":\n" + res.content.decode())
else:
raise Exception(
"Error when uploading data to SciDrive path " + str(path) +
".\nHttp Response from SciDrive API returned status code "
+ str(res.status_code) + ":\n" + res.content.decode())
return json.loads(res.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
def cancelJob(jobId):
"""
Cancels a single job (more info in http://www.voservices.net/skyquery).
:param jobId: the ID of the job, which is obtained at the moment of submitting the job.
:return: Returns True if the job was cancelled successfully.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error.
:example: isCanceled = SkyQuery.cancelJob(SkyQuery.submitJob("select 1 as foo"))
.. seealso:: SkyQuery.submitJob, SkyQuery.getJobStatus
"""
token = Authentication.getToken()
if token is not None and token != "":
taskName = ""
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.SkyQuery.cancelJob"
else:
taskName = "SciScript-Python.SkyQuery.cancelJob"
statusURL = Config.SkyQueryUrl + '/Jobs.svc/jobs/' + jobId + "?TaskName=" + taskName
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json'
}
headers['X-Auth-Token'] = token
response = requests.delete(statusURL, headers=headers)
if response.status_code == 200:
#r = response.json()
#try:
# status = r['queryJob']["status"]
# if status == "canceled":
# return True;
# else:
# return False;
#except:
# return False;
return True
else:
raise Exception(
"Error when cancelling job " + str(jobId) +
".\nHttp Response from SkyQuery API returned status code " +
str(response.status_code) + ":\n" + response.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
def getFileServices(verbose=True):
"""
Gets the definitions of file services that a user is able to access. A FileService represents a file system that contains root volumes accessible to the user for public/private data storage. Within each rootVolume, users can create sharable userVolumes for storing files.
:param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False).
:return: list of dictionaries, where each dictionary represents the description of a FileService that the user is able to access.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error.
:example: fileServices = Files.getFileServices();
.. seealso:: Files.getFileServiceFromName
"""
token = Authentication.getToken()
if token is not None and token != "":
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.Files.getFileServices"
else:
taskName = "SciScript-Python.Files.getFileServices"
url = Config.RacmApiURL + "/storem/fileservices?TaskName="+taskName;
headers = {'X-Auth-Token': token}
res = requests.get(url, headers=headers)
if res.status_code >= 200 and res.status_code < 300:
fileServices = [];
fileServicesAPIs = json.loads(res.content.decode())
for fileServicesAPI in fileServicesAPIs:
url = fileServicesAPI.get("apiEndpoint")
name = fileServicesAPI.get("name")
url = url + "api/volumes/?TaskName="+taskName;
try:
res = requests.get(url, headers=headers)
except:
if verbose:
warnings.warn("Error when getting definition of FileService named '" + name + "' with API URL '" + fileServicesAPI.get("apiEndpoint") + "'. This FileService might be not available", Warning, stacklevel=2)
if res.status_code >= 200 and res.status_code < 300:
fileServices.append(json.loads(res.content.decode()));
else:
if verbose:
warnings.warn("Error when getting definition of FileService named '" + name + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode(),Warning, stacklevel=2)
return fileServices;
else:
raise Exception("Error when getting the list of FileServices.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode());
else:
raise Exception("User token is not defined. First log into SciServer.")
def getKeystoneUserWithToken(token):
"""
Returns the name and Keystone id of the user corresponding to the specified token.
:param token: Sciserver's authentication token (string) for the user.
:return: Returns a KeystoneUser object, which stores the name and id of the user.
:raises: Throws an exception if the HTTP request to the Authentication URL returns an error.
:example: token = Authentication.getKeystoneUserWithToken(Authentication.getToken())
.. seealso:: Authentication.getToken, Authentication.login, Authentication.setToken.
"""
if keystoneUser.token == token and keystoneUser.token is not None:
return keystoneUser
else:
taskName = ""
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.Authentication.getKeystoneUserWithToken"
else:
taskName = "SciScript-Python.Authentication.getKeystoneUserWithToken"
loginURL = Config.AuthenticationURL
if ~loginURL.endswith("/"):
loginURL = loginURL + "/"
loginURL = loginURL + token + "?TaskName=" + taskName
getResponse = requests.get(loginURL)
if getResponse.status_code != 200:
raise Exception(
"Error when getting the keystone user with token " +
str(token) +
".\nHttp Response from the Authentication API returned status code "
+ str(getResponse.status_code) + ":\n" +
getResponse.content.decode())
responseJson = json.loads((getResponse.content.decode()))
ksu = KeystoneUser()
ksu.userName = responseJson["token"]["user"]["name"]
ksu.id = responseJson["token"]["user"]["id"]
keystoneUser.token = token
keystoneUser.userName = ksu.userName
keystoneUser.id = ksu.id
return ksu
def sqlSearch(sql, dataRelease=None):
"""
Executes a SQL query to the SDSS database, and retrieves the result table as a dataframe. Maximum number of rows retrieved is set currently to 500,000.
:param sql: a string containing the sql query
:param dataRelease: SDSS data release (string). E.g, 'DR13'. Default value already set in SciServer.Config.DataRelease
:return: Returns the results table as a Pandas data frame.
:raises: Throws an exception if the HTTP request to the SkyServer API returns an error.
:example: df = SkyServer.sqlSearch(sql="select 1")
.. seealso:: CasJobs.executeQuery, CasJobs.submitJob.
"""
if (dataRelease):
if dataRelease != "":
url = Config.SkyServerWSurl + '/' + dataRelease + '/SkyServerWS/SearchTools/SqlSearch?'
else:
url = Config.SkyServerWSurl + '/SkyServerWS/SearchTools/SqlSearch?'
else:
if Config.DataRelease != "":
url = Config.SkyServerWSurl + '/' + Config.DataRelease + '/SkyServerWS/SearchTools/SqlSearch?'
else:
url = Config.SkyServerWSurl + '/SkyServerWS/SearchTools/SqlSearch?'
url = url + 'format=csv&'
url = url + 'cmd=' + sql + '&'
if Config.isSciServerComputeEnvironment():
url = url + "TaskName=Compute.SciScript-Python.SkyServer.sqlSearch&"
else:
url = url + "TaskName=SciScript-Python.SkyServer.sqlSearch&"
#url = urllib.quote_plus(url)
acceptHeader = "text/plain"
headers = {'Content-Type': 'application/json', 'Accept': acceptHeader}
token = Authentication.getToken()
if token is not None and token != "":
headers['X-Auth-Token'] = token
response = requests.get(url, headers=headers, stream=True)
if response.status_code != 200:
raise Exception(
"Error when executing a sql query.\nHttp Response from SkyServer API returned status code "
+ str(response.status_code) + ":\n" + response.content.decode())
r = response.content.decode()
return pandas.read_csv(StringIO(r), comment='#', index_col=None)
def getTable(tableName, datasetName="MyDB", top=None):
"""
Returns a dataset table as a pandas DataFrame (more info in http://www.voservices.net/skyquery).
:param tableName: name of table (string) within dataset.
:param datasetName: name of dataset or database context (string).
:param top: number of top rows retrieved (integer).
:return: returns the table as a Pandas dataframe.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error.
:example: table = SkyQuery.getTable("myTable", datasetName="MyDB", top=10)
.. seealso:: SkyQuery.listQueues, SkyQuery.listAllDatasets, SkyQuery.getDatasetInfo, SkyQuery.listDatasetTables, SkyQuery.getTableInfo, SkyQuery.dropTable, SkyQuery.submitJob
"""
token = Authentication.getToken()
if token is not None and token != "":
taskName = ""
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.SkyQuery.getTable"
else:
taskName = "SciScript-Python.SkyQuery.getTable"
url = Config.SkyQueryUrl + '/Data.svc/' + datasetName + '/' + tableName + "?TaskName=" + taskName
if top != None and top != "":
url = url + '?top=' + str(top)
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json'
}
headers['X-Auth-Token'] = token
response = requests.get(url, headers=headers, stream=True)
if response.status_code == 200:
return (pandas.read_csv(StringIO(response.content.decode()),
sep="\t"))
else:
raise Exception(
"Error when getting table " + str(tableName) +
" from dataset " + str(datasetName) +
".\nHttp Response from SkyQuery API returned status code " +
str(response.status_code) + ":\n" + response.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
