def getSchemaName():
"""
Returns the WebServiceID that identifies the schema for a user in MyScratch database with CasJobs.
:return: WebServiceID of the user (string).
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the CasJobs API returns an error.
:example: wsid = CasJobs.getSchemaName()
.. seealso:: CasJobs.getTables.
"""
token = Authentication.getToken()
if token is not None and token != "":
keystoneUserId = Authentication.getKeystoneUserWithToken(token).id
usersUrl = Config.CasJobsRESTUri + "/users/" + keystoneUserId
headers = {'X-Auth-Token': token, 'Content-Type': 'application/json'}
getResponse = requests.get(usersUrl, headers=headers)
if getResponse.status_code != 200:
raise Exception(
"Error when getting schema name. Http Response from CasJobs API returned status code "
+ str(getResponse.status_code) + ":\n" +
getResponse.content.decode())
jsonResponse = json.loads(getResponse.content.decode())
return "wsid_" + str(jsonResponse["WebServicesId"])
else:
raise Exception("User token is not defined. First log into SciServer.")
def retrieve_sdss_photometry(ra: float, dec: float):
"""
Retrieve SDSS photometry for a given field, in a 0.2 x 0.2 degree box centred on the passed coordinates
coordinates. (Note - the width of the box is in RA degrees, not corrected for spherical distortion)
:param ra: Right Ascension of the centre of the desired field, in degrees.
:param dec: Declination of the centre of the desired field, in degrees.
:return: Retrieved photometry table, as a pandas dataframe, if successful; if not, None.
"""
try:
from SciServer import Authentication, CasJobs
except ImportError:
print("It seems that SciScript/SciServer is not installed, or not accessible to this environment. "
"\nIf you wish to automatically download SDSS data, please install "
"\nSciScript (https://github.com/sciserver/SciScript-Python); "
"\notherwise, retrieve the data manually from "
"\nhttp://skyserver.sdss.org/dr16/en/tools/search/sql.aspx")
return None
print(f"Querying SDSS DR16 archive for field centring on RA={ra}, DEC={dec}")
user = keys['sciserver_user']
password = keys["sciserver_pwd"]
Authentication.login(UserName=user, Password=password)
# Construct an SQL query to send to SciServer
query = "SELECT objid,ra,dec"
for f in sdss_filters:
query += f",psfMag_{f},psfMagErr_{f},fiberMag_{f},fiberMagErr_{f},fiber2Mag_{f},fiber2MagErr_{f},petroMag_{f},petroMagErr_{f} "
query += "FROM PhotoObj "
query += f"WHERE ra BETWEEN {ra - 0.1} AND {ra + 0.1} "
query += f"AND dec BETWEEN {dec - 0.1} AND {dec + 0.1} "
print(f"Retrieving photometry from SDSS DR16 via SciServer for field at {ra}, {dec}...")
df = CasJobs.executeQuery(sql=query, context='DR16')
if len(df.index) == 0:
df = None
return df
def login(username, password):
token1 = Authentication.login(username, password)
user = Authentication.getKeystoneUserWithToken(token1)
print("userName="******"id=" + user.id)
iden = Authentication.identArgIdentifier()
print("ident=" + iden)
def __setupserver(self):
print("Setting up SkyServer...")
Authentication_loginName = self.username
Authentication_loginPassword = self.password
token1 = Authentication.login(Authentication_loginName,
Authentication_loginPassword)
token2 = Authentication.getToken()
token3 = Authentication.getKeystoneToken()
token4 = Authentication.token.value
user = Authentication.getKeystoneUserWithToken(token1)
iden = Authentication.identArgIdentifier()
def getSchemaName(token=""):
if token == "":
userToken = Authentication.getToken()
else:
userToken = token
keystoneUserId = Authentication.getKeystoneUserWithToken(userToken).id
usersUrl = Config.CasJobsRESTUri + "/users/" + keystoneUserId
headers = {"X-Auth-Token": userToken, "Content-Type": "application/json"}
getResponse = requests.get(usersUrl, headers=headers)
jsonResponse = json.loads(getResponse.content.decode())
return "wsid_" + str(jsonResponse["WebServicesId"])
def move(fileService, path, destinationFileService, destinationPath, replaceExisting=True, doCopy=True):
"""
Moves or copies a file or folder.
:param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices().
:param path: String defining the origin path (in the remote fileService) of the file or directory to be copied/moved, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/fileToBeMoved.txt.
:param destinationFileService: name of fileService (string), or object (dictionary) that defines a destination file service (where the file is moved/copied into). A list of these kind of objects available to the user is returned by the function Files.getFileServices().
:param destinationRelativePath: String defining the destination path (in the remote destinationFileService) of the file or directory to be copied/moved, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/recentlyMovedFile.txt.
:param replaceExisting: If set to False, it will throw an error if the file already exists, If set to True, it will not throw and eeror in that case.
:param doCopy: if set to True, then it will copy the file or folder. If set to False, then the file or folder will be moved.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error.
:example: fileServices = Files.getFileServices(); isDownloaded = Files.upload("/myUploadedFile.txt","persistent","myUserName", fileServices[0], localFilePath="/myDownloadedFile.txt");
.. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.upload, Files.dirList
"""
token = Authentication.getToken()
if token is not None and token != "":
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.Files.Move"
else:
taskName = "SciScript-Python.Files.Move"
(rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path);
(destinationRootVolume, destinationUserVolumeOwner, destinationUserVolume, destinationRelativePath) = splitPath(destinationPath);
if userVolumeOwner is None:
userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName;
if destinationUserVolumeOwner is None:
destinationUserVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName;
if type(fileService) == str:
fileService = getFileServiceFromName(fileService)
if type(destinationFileService) == str:
destinationFileService = getFileServiceFromName(destinationFileService)
url = __getFileServiceAPIUrl(fileService) + "api/data/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?replaceExisting=" + str(replaceExisting) + "&doCopy=" + str(doCopy) + "&TaskName=" + taskName;
headers = {'X-Auth-Token': token, "Content-Type": "application/json"}
if type(destinationFileService) == dict:
destinationFileService = destinationFileService['name']
jsonDict = {'destinationPath': destinationRelativePath, 'destinationRootVolume': destinationRootVolume, 'destinationUserVolume':destinationUserVolume, 'destinationOwnerName': destinationUserVolumeOwner, 'destinationFileService': destinationFileService};
data = json.dumps(jsonDict).encode()
res = requests.put(url, stream=True, headers=headers, json=jsonDict)
if res.status_code < 200 or res.status_code >= 300:
raise Exception("Error when moving '" + str(path) + "' in file service '" + fileService.get("name") + "' to '" + str(path) + "' in file service '" + destinationFileService.get("name)") + "'. \nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode());
else:
raise Exception("User token is not defined. First log into SciServer.")
def listTableColumns(tableName, datasetName="MyDB"):
"""
Returns a list of all columns in a table belonging to a particular dataset (more info in http://www.voservices.net/skyquery).
:param tableName: name of table (string) within dataset.
:param datasetName: name of dataset (string).
:return: returns a list containing the columns and associated descriptions.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error.
:example: columns = SkyQuery.listTableColumns("myTable", datasetName="MyDB")
.. seealso:: SkyQuery.listQueues, SkyQuery.listAllDatasets, SkyQuery.getDatasetInfo, SkyQuery.listDatasetTables, SkyQuery.getTableInfo, SkyQuery.getTable, SkyQuery.dropTable
"""
token = Authentication.getToken()
if token is not None and token != "":
url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName + '/tables/' + tableName + '/columns'
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json'
}
headers['X-Auth-Token'] = token
response = requests.get(url, headers=headers)
if response.status_code == 200:
r = response.json()
return (r['columns'])
else:
raise Exception(
"Error when listing columns of table " + str(tableName) +
" in dataset " + str(datasetName) +
".\nHttp Response from SkyQuery API returned status code " +
str(response.status_code) + ":\n" + response.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
def dropTable(tableName, datasetName="MyDB"):
"""
Drops (deletes) a table from the user database (more info in http://www.voservices.net/skyquery).
:param tableName: name of table (string) within dataset.
:param datasetName: name of dataset or database context (string).
:return: returns True if the table was deleted successfully.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error.
:example: result = SkyQuery.dropTable("myTable", datasetName="MyDB")
.. seealso:: SkyQuery.listQueues, SkyQuery.listAllDatasets, SkyQuery.getDatasetInfo, SkyQuery.listDatasetTables, SkyQuery.getTableInfo, SkyQuery.getTable, SkyQuery.submitJob
"""
token = Authentication.getToken()
if token is not None and token != "":
url = Config.SkyQueryUrl + '/Data.svc/' + datasetName + '/' + tableName
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json'
}
headers['X-Auth-Token'] = token
response = requests.delete(url, headers=headers)
if response.status_code == 200:
return (True)
else:
raise Exception(
"Error when dropping table " + str(tableName) +
" from dataset " + str(datasetName) +
".\nHttp Response from SkyQuery API returned status code " +
str(response.status_code) + ":\n" + response.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
def publicUrl(path):
"""
Gets the public URL of a file (or directory) in SciDrive.
:param path: path of the file (or directory) in SciDrive.
:return: URL of a file in SciDrive (string).
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SciDrive API returns an error.
:example: url = SciDrive.publicUrl("path/to/SciDrive/file.csv")
.. seealso:: SciDrive.upload
"""
token = Authentication.getToken()
if token is not None and token != "":
url = Config.SciDriveHost + '/vospace-2.0/1/media/sandbox/' + str(path)
headers = {'X-Auth-Token': token}
res = requests.get(url, headers=headers)
if res.status_code != 200:
raise Exception(
"Error when getting the public URL of SciDrive file " +
str(path) +
".\nHttp Response from SciDrive API returned status code " +
str(res.status_code) + ":\n" + res.content.decode())
jsonRes = json.loads(res.content.decode())
fileUrl = jsonRes["url"]
return (fileUrl)
else:
raise Exception("User token is not defined. First log into SciServer.")
def getJobStatus(jobId):
"""
Shows the status of a job submitted to CasJobs.
:param jobId: id of job (integer)
:return: Returns a dictionary object containing the job status and related metadata. The "Status" field can be equal to 0 (Ready), 1 (Started), 2 (Canceling), 3(Canceled), 4 (Failed) or 5 (Finished). If jobId is the empty string, then returns a list with the statuses of all previous jobs.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the CasJobs API returns an error.
:example: status = CasJobs.getJobStatus(CasJobs.submitJob("select 1"))
.. seealso:: CasJobs.submitJob, CasJobs.waitForJob, CasJobs.cancelJob.
"""
token = Authentication.getToken()
if token is not None and token != "":
QueryUrl = Config.CasJobsRESTUri + "/jobs/" + str(jobId)
headers = {'X-Auth-Token': token, 'Content-Type': 'application/json'}
postResponse = requests.get(QueryUrl, headers=headers)
if postResponse.status_code != 200:
raise Exception(
"Error when getting the status of job " + str(jobId) +
".\nHttp Response from CasJobs API returned status code " +
str(postResponse.status_code) + ":\n" +
postResponse.content.decode())
return json.loads(postResponse.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
def getQueueInfo(queue):
"""
Returns information about a particular job queue (more info in http://www.voservices.net/skyquery).
:param queue: queue name (string)
:return: a dictionary containing information associated to the queue.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error.
:example: queueInfo = SkyQuery.getQueueInfo('quick')
.. seealso:: SkyQuery.listQueues, SkyQuery.submitJob, SkyQuery.getJobStatus
"""
token = Authentication.getToken()
if token is not None and token != "":
jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json'
}
headers['X-Auth-Token'] = token
response = requests.get(jobsURL, headers=headers)
if response.status_code == 200:
r = response.json()
return (r['queue'])
else:
raise Exception(
"Error when getting queue info of " + str(queue) +
".\nHttp Response from SkyQuery API returned status code " +
str(response.status_code) + ":\n" + response.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
def listJobs(queue="quick"):
"""
Lists the jobs in the queue in descending order by submission time. Only jobs of the authenticated user are listed (more info in http://www.voservices.net/skyquery).
:param queue: queue name (string). Can be set to 'quick' for a quick job, or 'long' for a long job.
:return: returns job definitions as a list object.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error.
:example: jobsList = SkyQuery.listJobs('quick')
.. seealso:: SkyQuery.getJobStatus, SkyQuery.listQueues
"""
token = Authentication.getToken()
if token is not None and token != "":
jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + '/jobs?'
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json'
}
headers['X-Auth-Token'] = token
response = requests.get(jobsURL, headers=headers)
if response.status_code == 200:
r = response.json()
return (r['jobs'])
else:
raise Exception(
"Error when listing jobs on queue " + str(queue) +
".\nHttp Response from SkyQuery API returned status code " +
str(response.status_code) + ":\n" + response.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
def submitJob(query,token = '', queue='quick'):
"""
Submits a new job.
Parameters:
query: sql query
token: SciServer's authentication token.
queue: can be set to 'quick' for a quick job, or 'long' for a long job.
Output: returns the jobID unique identifier of the job.
"""
jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + '/jobs'
headers = {'Content-Type': 'application/json','Accept': 'application/json'}
if (token == ""):
headers['X-Auth-Token']= Authentication.getToken()
else:
headers['X-Auth-Token']= token
body={"queryJob":{"query":query}}
data=json.dumps(body).encode()
response = requests.post(jobsURL,data=data,headers=headers)
if response.status_code == 200:
r = response.json()
return(r['queryJob']['guid'])
else:
raise NameError(response.json())
def uploadCVSDataToTable(CVSdata, tableName, context="MyDB", token=""):
"""Uploads cvs data into casjobs. data should support the buffered interface (it should have a read() method).
https://docs.python.org/3/library/urllib.request.html
Returns the output from casjobs in string form."""
if Config.executeMode == "debug":
print "Uploading ", sys.getsizeof(CVSdata), "bytes..."
tablesUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/Tables/" + tableName
headers = {}
if token == "":
headers["X-Auth-Token"] = Authentication.getToken()
else:
headers["X-Auth-Token"] = token
try:
postResponse = requests.post(tablesUrl, data=CVSdata, headers=headers)
if Config.executeMode == "debug":
print "uploadCVSDataFrameToTable POST response: ", postResponse.status_code, postResponse.reason
return postResponse.content.decode()
except Exception as error:
if Config.executeMode == "debug":
print "There was a problem uploading the data. Exception message: ", error.message
raise
def createContainer(path):
"""
Creates a container (directory) in SciDrive
:param path: path of the directory in SciDrive.
:return: Returns True if the container (directory) was created successfully.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SciDrive API returns an error.
:example: response = SciDrive.createContainer("MyDirectory")
.. seealso:: SciDrive.upload.
"""
token = Authentication.getToken()
if token is not None and token != "":
containerBody = (
'<vos:node xmlns:xsi="http://www.w3.org/2001/thisSchema-instance" '
'xsi:type="vos:ContainerNode" xmlns:vos="http://www.ivoa.net/xml/VOSpace/v2.0" '
'uri="vos://' + Config.SciDriveHost + '!vospace/' + path + '">'
'<vos:properties/><vos:accepts/><vos:provides/><vos:capabilities/>'
'</vos:node>')
url = Config.SciDriveHost + '/vospace-2.0/nodes/' + path
data = str.encode(containerBody)
headers = {'X-Auth-Token': token, 'Content-Type': 'application/xml'}
res = requests.put(url, data=data, headers=headers)
if res.status_code < 200 or res.status_code >= 300:
raise Exception(
"Error when creating SciDrive container at " + str(path) +
".\nHttp Response from SciDrive API returned status code " +
str(res.status_code) + ":\n" + res.content.decode())
return True
else:
raise Exception("User token is not defined. First log into SciServer.")
def getDatasetInfo(datasetName="MyDB"):
"""
Gets information related to a particular dataset (more info in http://www.voservices.net/skyquery).
:param datasetName: name of dataset (string).
:return: returns a dictionary containing the dataset information.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error.
:example: info = SkyQuery.getDatasetInfo("MyDB")
.. seealso:: SkyQuery.listQueues, SkyQuery.listAllDatasets, SkyQuery.listDatasetTables, SkyQuery.getTableInfo, SkyQuery.listTableColumns, SkyQuery.getTable, SkyQuery.dropTable
"""
token = Authentication.getToken()
if token is not None and token != "":
schemaURL = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json'
}
headers['X-Auth-Token'] = token
response = requests.get(schemaURL, headers=headers)
if response.status_code == 200:
return (response.json())
else:
raise Exception(
"Error when getting info from dataset " + str(datasetName) +
".\nHttp Response from SkyQuery API returned status code " +
str(response.status_code) + ":\n" + response.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
def getTables(context="MyDB"):
"""
Gets the names, size and creation date of all tables in a database context that the user has access to.
:param context: database context (string)
:return: The result is a json object with format [{"Date":seconds,"Name":"TableName","Rows":int,"Size",int},..]
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the CasJobs API returns an error.
:example: tables = CasJobs.getTables("MyDB")
.. seealso:: CasJobs.getSchemaName
"""
token = Authentication.getToken()
if token is not None and token != "":
TablesUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/Tables"
headers = {'X-Auth-Token': token, 'Content-Type': 'application/json'}
getResponse = requests.get(TablesUrl, headers=headers)
if getResponse.status_code != 200:
raise Exception(
"Error when getting table description from database context " +
str(context) +
".\nHttp Response from CasJobs API returned status code " +
str(getResponse.status_code) + ":\n" +
getResponse.content.decode())
jsonResponse = json.loads(getResponse.content.decode())
return jsonResponse
else:
raise Exception("User token is not defined. First log into SciServer.")
def listAllDatasets():
"""
Lists all available datasets (more info in http://www.voservices.net/skyquery).
:return: returns dataset definitions as a list object.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error.
:example: datasets = SkyQuery.listAllDatasets()
.. seealso:: SkyQuery.listQueues, SkyQuery.getDatasetInfo, SkyQuery.listDatasetTables, SkyQuery.getTableInfo, SkyQuery.listTableColumns, SkyQuery.getTable, SkyQuery.dropTable
"""
token = Authentication.getToken()
if token is not None and token != "":
schemaURL = Config.SkyQueryUrl + '/Schema.svc/datasets'
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json'
}
headers['X-Auth-Token'] = token
response = requests.get(schemaURL, headers=headers)
if response.status_code == 200:
r = response.json()
return (r['datasets'])
else:
raise Exception(
"Error when listing all datasets.\nHttp Response from SkyQuery API returned status code "
+ str(response.status_code) + ":\n" +
response.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
def cancelJob(jobId):
"""
Cancels a job already submitted.
:param jobId: id of job (integer)
:return: Returns True if the job was canceled successfully.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the CasJobs API returns an error.
:example: response = CasJobs.cancelJob(CasJobs.submitJob("select 1"))
.. seealso:: CasJobs.submitJob, CasJobs.waitForJob.
"""
token = Authentication.getToken()
if token is not None and token != "":
QueryUrl = Config.CasJobsRESTUri + "/jobs/" + str(jobId)
headers = {'X-Auth-Token': token, 'Content-Type': 'application/json'}
response = requests.delete(QueryUrl, headers=headers)
if response.status_code != 200:
raise Exception(
"Error when canceling job " + str(jobId) +
".\nHttp Response from CasJobs API returned status code " +
str(response.status_code) + ":\n" + response.content.decode())
return True
#json.loads(response.content)
else:
raise Exception("User token is not defined. First log into SciServer.")
def getJobStatus(jobId):
"""
Gets the status of a job, as well as other related metadata (more info in http://www.voservices.net/skyquery).
:param jobId: the ID of the job (string), which is obtained at the moment of submitting the job.
:return: a dictionary with the job status and other related metadata.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error.
:example: status = SkyQuery.getJobStatus(SkyQuery.submitJob("select 1 as foo"))
.. seealso:: SkyQuery.submitJob, SkyQuery.cancelJob
"""
token = Authentication.getToken()
if token is not None and token != "":
statusURL = Config.SkyQueryUrl + '/Jobs.svc/jobs/' + str(jobId)
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json'
}
headers['X-Auth-Token'] = token
response = requests.get(statusURL, headers=headers)
if response.status_code == 200:
r = response.json()
return (r['queryJob'])
else:
raise Exception(
"Error when getting the job status of job " + str(jobId) +
".\nHttp Response from SkyQuery API returned status code " +
str(response.status_code) + ":\n" + response.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
def listQueues():
"""
Returns a list of all available job queues and related metadata (more info in http://www.voservices.net/skyquery).
:return: a list of all available job queues and related metadata.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error.
:example: queueList = SkyQuery.listQueues()
.. seealso:: SkyQuery.getQueueInfo, SkyQuery.submitJob, SkyQuery.getJobStatus
"""
token = Authentication.getToken()
if token is not None and token != "":
jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues'
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json'
}
headers['X-Auth-Token'] = token
response = requests.get(jobsURL, headers=headers)
if response.status_code == 200:
r = response.json()
return (r['queues'])
else:
raise Exception(
"Error when listing queues.\nHttp Response from SkyQuery API returned status code "
+ str(response.status_code) + ":\n" +
response.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
def delete(fileService, path, quiet=True):
"""
Deletes a directory or file in the File System.
:param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices().
:param path: String defining the path (in the remote fileService) of the file or directory to be deleted, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/fileToBeDeleted.txt.
:param quiet: If set to False, it will throw an error if the file does not exist. If set to True. it will not throw an error.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error.
:example: fileServices = Files.getFileServices(); isDeleted = Files.delete("/myUselessFile.txt","persistent","myUserName", fileServices[0]);
.. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList
"""
token = Authentication.getToken()
if token is not None and token != "":
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.Files.delete"
else:
taskName = "SciScript-Python.Files.delete"
(rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path);
url = __getFileServiceAPIUrl(fileService) + "api/data/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName
headers = {'X-Auth-Token': token}
res = requests.delete(url, headers=headers)
if res.status_code >= 200 and res.status_code < 300:
pass;
else:
raise Exception("Error when deleting '" + str(path) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode());
else:
raise Exception("User token is not defined. First log into SciServer.")
def directoryList(path=""):
"""
Gets the contents and metadata of a SciDrive directory (or file).
:param path: path of the directory (or file ) in SciDrive.
:return: a dictionary containing info and metadata of the directory (or file).
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SciDrive API returns an error.
:example: dirList = SciDrive.directoryList("path/to/SciDrive/directory")
.. seealso:: SciDrive.upload, SciDrive.download
"""
token = Authentication.getToken()
if token is not None and token != "":
url = Config.SciDriveHost + "/vospace-2.0/1/metadata/sandbox/" + str(
path) + "?list=True&path=" + str(path)
headers = {'X-Auth-Token': token}
res = requests.get(url, headers=headers)
if res.status_code != 200:
raise Exception(
"Error when getting the public URL of SciDrive file " +
str(path) +
".\nHttp Response from SciDrive API returned status code " +
str(res.status_code) + ":\n" + res.content.decode())
jsonRes = json.loads(res.content.decode())
return (jsonRes)
else:
raise Exception("User token is not defined. First log into SciServer.")
def cancelJob(jobId):
"""
Cancels the execution of a job.
:param jobId: Id of the job (integer)
:raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error.
:example: job = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); isCanceled = Jobs.cancelJob(job.get('id'));
.. seealso:: Jobs.submitNotebookJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains.
"""
token = Authentication.getToken()
if token is not None and token != "":
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.Jobs.cancelJob"
else:
taskName = "SciScript-Python.Jobs.cancelJob"
url = Config.RacmApiURL + "/jobm/rest/jobs/" + str(
jobId) + "/cancel?TaskName=" + taskName
headers = {'X-Auth-Token': token, "Content-Type": "application/json"}
res = requests.post(url, headers=headers, stream=True)
if res.status_code != 200:
raise Exception(
"Error when getting from JOBM API the job status of jobId=" +
str(jobId) +
".\nHttp Response from JOBM API returned status code " +
str(res.status_code) + ":\n" + res.content.decode())
else:
pass
else:
raise Exception("User token is not defined. First log into SciServer.")
def sqlSearch(sql, limit="10", token=""):
"""Runs a SQL query against the SDSS database. If a token is supplied, then it will run on behalf of the token's user.\n
'sql': a string containing the sql query\n
'limit': maximum number of rows in the result table (string). If set to '0', then the function will return all rows.\n
'token': Sciserver's authentication token for the user.\n
"""
url = Config.SkyServerWSurl + '/' + Config.DataRelease + '/SearchTools/SqlSearch?'
url = url + 'format=csv&'
url = url + 'cmd=' + sql + '&'
url = url + 'limit=' + limit + '&'
url = urllib.quote_plus(url)
acceptHeader = "text/plain"
headers = {'Content-Type': 'application/json', 'Accept': acceptHeader}
if (token != ""):
headers['X-Auth-Token'] = token
else:
Token = ""
try:
Token = Authentication.getToken()
except:
Token = ""
if(Token != ""):
headers['X-Auth-Token'] = Token
try:
response = requests.get(url,headers=headers)
if response.status_code != 200:
return {"Error":{"ErrorCode":response.status_code,"Message":response.content.decode()}}
r=response.content.decode();
return pandas.read_csv(StringIO(r), comment='#')
except requests.exceptions.RequestException as e:
return e
def getRDBComputeDomains():
"""
Gets a list of all registered Relational Database (RDB) compute domains that the user has access to.
:return: a list of dictionaries, each one containing the definition of an RDB compute domain.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error.
:example: rdbComputeDomains = Jobs.getRDBComputeDomains();
.. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob
"""
token = Authentication.getToken()
if token is not None and token != "":
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.Jobs.getRDBComputeDomains"
else:
taskName = "SciScript-Python.Jobs.getRDBComputeDomains"
url = Config.RacmApiURL + "/jobm/rest/computedomains/rdb?TaskName=" + taskName
headers = {'X-Auth-Token': token, "Content-Type": "application/json"}
res = requests.get(url, headers=headers, stream=True)
if res.status_code != 200:
raise Exception(
"Error when getting RDB Compute Domains from JOBM API.\nHttp Response from JOBM API returned status code "
+ str(res.status_code) + ":\n" + res.content.decode())
else:
return json.loads(res.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
def download(path, format="text", localFilePath=""):
"""
Downloads a file (directory) from SciDrive into the local file system, or returns the file conetent as an object in several formats.
:param path: path of the file (or directory) in SciDrive.
:param format: type of the returned object. Can be "StringIO" (data.StringIO object containing readable text), "BytesIO" (data.BytesIO object containing readable binary data), "response" ( the HTTP response as an object of class requests.Response) or "text" (a text string). If the parameter 'localFilePath' is defined, then the 'format' parameter is not used and the file is downloaded to the local file system instead.
:param localFilePath: local path of the file to be downloaded. If 'localFilePath' is defined, then the 'format' parameter is not used.
:return: If the 'localFilePath' parameter is defined, then it will return True when the file is downloaded successfully in the local file system. If the 'localFilePath' is not defined, then the type of the returned object depends on the value of the 'format' parameter (either data.StringIO, data.BytesIO, requests.Response or string).
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SciDrive API returns an error.
:example: csvString = SciDrive.download("path/to/SciDrive/file.csv", format="text");
.. seealso:: SciDrive.upload
"""
token = Authentication.getToken()
if token is not None and token != "":
taskName = ""
if Config.isSciServerComputeEnvironment():
task.name = "Compute.SciScript-Python.SciDrive.download"
else:
task.name = "SciScript-Python.SciDrive.download"
fileUrl = publicUrl(path)
res = requests.get(fileUrl, stream=True)
if res.status_code != 200:
raise Exception(
"Error when downloading SciDrive file " + str(path) +
".\nHttp Response from SciDrive API returned status code " +
str(res.status_code) + ":\n" + res.content.decode())
if localFilePath is not None and localFilePath != "":
bytesio = BytesIO(res.content)
theFile = open(localFilePath, "w+b")
theFile.write(bytesio.read())
theFile.close()
return True
else:
if format is not None and format != "":
if format == "StringIO":
return StringIO(res.content.decode())
if format == "text":
return res.content.decode()
elif format == "BytesIO":
return BytesIO(res.content)
elif format == "response":
return res
else:
raise Exception(
"Unknown format '" + format +
"' when trying to download SciDrive file " +
str(path) + ".\n")
else:
raise Exception("Wrong format parameter value\n")
else:
raise Exception("User token is not defined. First log into SciServer.")
def rectangularSearch(min_ra, max_ra, min_dec, max_dec, coordType="equatorial", whichPhotometry="optical", limit="10", dataRelease=None):
"""
Runs a query in the SDSS database that searches for all objects within a certain rectangular box defined on the the sky, and retrieves the result table as a Panda's dataframe.\n
:param min_ra: Minimum value of Right Ascension coordinate that defines the box boundaries on the sky.\n
:param max_ra: Maximum value of Right Ascension coordinate that defines the box boundaries on the sky.\n
:param min_dec: Minimum value of Declination coordinate that defines the box boundaries on the sky.\n
:param max_dec: Maximum value of Declination coordinate that defines the box boundaries on the sky.\n
:param coordType: Type of celestial coordinate system. Can be set to "equatorial" or "galactic".\n
:param whichPhotometry: Type of retrieved data. Can be set to "optical" or "infrared".\n
:param limit: Maximum number of rows in the result table (string). If set to "0", then the function will return all rows.\n
:param dataRelease: SDSS data release string. Example: dataRelease='DR13'. Default value already set in SciServer.Config.DataRelease
:return: Returns the results table as a Pandas data frame.
:raises: Throws an exception if the HTTP request to the SkyServer API returns an error.
:example: df = SkyServer.rectangularSearch(min_ra=258.2, max_ra=258.3, min_dec=64,max_dec=64.1)
.. seealso:: SkyServer.sqlSearch, SkyServer.radialSearch.
"""
if(dataRelease):
if dataRelease != "":
url = Config.SkyServerWSurl + '/' + dataRelease + '/SkyServerWS/SearchTools/RectangularSearch?'
else:
url = Config.SkyServerWSurl + '/SkyServerWS/SearchTools/RectangularSearch?'
else:
if Config.DataRelease != "":
url = Config.SkyServerWSurl + '/' + Config.DataRelease + '/SkyServerWS/SearchTools/RectangularSearch?'
else:
url = Config.SkyServerWSurl + '/SkyServerWS/SearchTools/RectangularSearch?'
url = url + 'format=csv&'
url = url + 'min_ra=' + str(min_ra) + '&'
url = url + 'max_ra=' + str(max_ra) + '&'
url = url + 'min_dec=' + str(min_dec) + '&'
url = url + 'max_dec=' + str(max_dec) + '&'
url = url + 'coordType=' + coordType + '&'
url = url + 'whichPhotometry=' + whichPhotometry + '&'
url = url + 'limit=' + limit + '&'
if Config.isSciServerComputeEnvironment():
url = url + "TaskName=Compute.SciScript-Python.SkyServer.rectangularSearch&"
else:
url = url + "TaskName=SciScript-Python.SkyServer.rectangularSearch&"
#url = urllib.quote_plus(url)
acceptHeader = "text/plain"
headers = {'Content-Type': 'application/json', 'Accept': acceptHeader}
token = Authentication.getToken()
if token is not None and token != "":
headers['X-Auth-Token'] = token
response = requests.get(url,headers=headers, stream=True)
if response.status_code != 200:
raise Exception("Error when executing a rectangular search.\nHttp Response from SkyServer API returned status code " + str(response.status_code) + ":\n" + response.content.decode());
r=response.content.decode();
return pandas.read_csv(StringIO(r), comment='#', index_col=None)
def shareUserVolume(fileService,
path,
sharedWith,
allowedActions,
type="USER"):
"""
Shares a user volume with another user or group
:param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices().
:param path: String defining the path (in the remote fileService) of the user volume to be shared, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume.
:param sharedWith: name (string) of user or group that the user volume is shared with.
:param allowedActions: array of strings defining actions the user or group is allowed to do with respect to the shared user volume. E.g.: ["read","write","grant","delete"]. The "grant" action means that the user or group can also share the user volume with another user or group. The "delete" action meand ability to delete the user volume (use with care).
:param type: type (string) of the entity defined by the "sharedWith" parameter. Can be set to "USER" or "GROUP".
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error.
:example: fileServices = Files.getFileServices(); Files.shareUserVolume(getFileServices[0], "Storage/myUserName/myUSerVolume", "userName", ["read","write"], type="USER");
.. seealso:: Files.getFileServices(), Files.getFilrmsieServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList
"""
token = Authentication.getToken()
if token is not None and token != "":
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.Files.ShareUserVolume"
else:
taskName = "SciScript-Python.Files.ShareUserVolume"
if type(fileService) == str:
fileService = getFileServiceFromName(fileService)
(rootVolume, userVolumeOwner, userVolume, relativePath,
isTopVolumeARootVolume) = splitPath(path, fileService)
data = [{
'name': sharedWith,
'type': type,
'allowedActions': allowedActions
}]
body = json.dumps(data).encode()
url = __getFileServiceAPIUrl(
fileService
) + "api/share/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?TaskName=" + taskName
headers = {'X-Auth-Token': token, 'Content-Type': 'application/json'}
res = requests.patch(url, headers=headers, data=body)
if res.status_code >= 200 and res.status_code < 300:
pass
else:
raise Exception(
"Error when sharing userVolume '" + str(path) +
"' in file service '" + str(fileService.get('name')) +
"'.\nHttp Response from FileService API returned status code "
+ str(res.status_code) + ":\n" + res.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
def createDir(fileService, path, quiet=True):
"""
Create a directory.
:param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices().
:param path: path (in the remote file service) to the directory (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/directory
:param quiet: If set to False, it will throw an error if the directory already exists. If set to True. it will not throw an error.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error.
:example: fileServices = Files.getFileServices(); Files.createDir(fileServices[0], "myRootVolume","myUserVolume", "myNewDir");
.. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList
"""
token = Authentication.getToken()
if token is not None and token != "":
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.Files.createDir"
else:
taskName = "SciScript-Python.Files.createDir"
(rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path);
if not relativePath.startswith("/"):
relativePath = "/" + relativePath;
if userVolumeOwner is None:
userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName;
if type(fileService) == str:
fileService = getFileServiceFromName(fileService)
url = __getFileServiceAPIUrl(fileService) + "api/folder/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName;
headers = {'X-Auth-Token': token}
res = requests.put(url, headers=headers)
if res.status_code >= 200 and res.status_code < 300:
pass;
else:
raise Exception("Error when creating directory '" + str(path) + "' in file service '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode());
else:
raise Exception("User token is not defined. First log into SciServer.")
def getJobsList(top=10, open=None, start=None, end=None, type='all'):
"""
Gets the list of Jobs submitted by the user.
:param top: top number of jobs (integer) returned. If top=None, then all jobs are returned.
:param open: If set to 'True', then only returns jobs that have not finished executing and wrapped up (status <= FINISHED). If set to 'False' then only returnes jobs that are still running. If set to 'None', then returns both finished and unfinished jobs.
:param start: The earliest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to 'None', then there is no lower bound on date.
:param end: The latest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to 'None', then there is no upper bound on date.
:param type: type (string) of jobs returned. Can take values of 'rdb' (for returning only relational database jobs), 'docker' (for returning only Docker jobs) and 'all' (all job types are returned).
:return: a list of dictionaries, each one containing the definition of a submitted job.
:raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error.
:example: jobs = Jobs.getJobsList(top=2);
.. seealso:: Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob
"""
token = Authentication.getToken()
if token is not None and token != "":
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.Jobs.getJobsList"
else:
taskName = "SciScript-Python.Jobs.getJobsList"
topString = ("top=" + str(top) + "&" if top != None else "")
startString = ("start=" + str(start) + "&" if start != None else "")
endString = ("end=" + str(end) + "&" if end != None else "")
if open is None:
openString = ""
else:
if open is True:
openString = "open=true&"
else:
openString = "open=false&"
url = Config.RacmApiURL + "/jobm/rest/jobs?"
if (type == 'rdb'):
url = Config.RacmApiURL + "/jobm/rest/rdbjobs?"
if (type == 'docker'):
url = Config.RacmApiURL + "/jobm/rest/dockerjobs?"
url = url + topString + startString + endString + "TaskName=" + taskName
headers = {'X-Auth-Token': token, "Content-Type": "application/json"}
res = requests.get(url, headers=headers, stream=True)
if res.status_code != 200:
raise Exception(
"Error when getting list of jobs from JOBM API.\nHttp Response from JOBM API returned status code "
+ str(res.status_code) + ":\n" + res.content.decode())
else:
return json.loads(res.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
def getTables(context="MyDB"):
"""Returns all the tables from the current user in the given context.
The result is a json object with format [{"Date":seconds,"Name":"TableName","Rows":int,"Size",int},..]"""
TablesUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/Tables"
headers = {"X-Auth-Token": Authentication.getToken(), "Content-Type": "application/json"}
getResponse = requests.get(TablesUrl, headers=headers)
jsonResponse = json.loads(getResponse.content.decode())
return jsonResponse
def uploadTable(uploadData, tableName, datasetName="MyDB", format="csv"):
"""
Uploads a data table into a database (more info in http://www.voservices.net/skyquery).
:param uploadData: data table, for now accepted in CSV string format.
:param tableName: name of table (string) within dataset.
:param datasetName: name of dataset or database context (string).
:param format: format of the 'data' parameter. Set to 'csv' for now.
:return: returns True if the table was uploaded successfully.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error.
:example: result = SkyQuery.uploadTable("Column1,Column2\\n4.5,5.5\\n", tableName="myTable", datasetName="MyDB", format="csv")
.. seealso:: SkyQuery.listQueues, SkyQuery.listAllDatasets, SkyQuery.getDatasetInfo, SkyQuery.listDatasetTables, SkyQuery.getTableInfo, SkyQuery.getTable, SkyQuery.submitJob
"""
token = Authentication.getToken()
if token is not None and token != "":
taskName = ""
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.SkyQuery.uploadTable"
else:
taskName = "SciScript-Python.SkyQuery.uploadTable"
url = Config.SkyQueryUrl + '/Data.svc/' + datasetName + '/' + tableName + "?TaskName=" + taskName
ctype = ""
if format == "csv":
ctype = 'text/csv'
else:
raise Exception("Unknown format '" + format +
"' when trying to upload data in SkyQuery.\n")
headers = {'Content-Type': ctype, 'Accept': 'application/json'}
headers['X-Auth-Token'] = token
#url = urllib.quote_plus(url)
response = requests.put(url,
data=uploadData,
headers=headers,
stream=True)
if response.status_code == 200:
return (True)
else:
raise Exception(
"Error when uploading data to table " + str(tableName) +
" in dataset " + str(datasetName) +
".\nHttp Response from SkyQuery API returned status code " +
str(response.status_code) + ":\n" + response.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
def dirList(fileService, path, level=1, options=''):
"""
Lists the contents of a directory.
:param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices().
:param path: String defining the path (in the remote file service) of the directory to be listed, starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/directoryToBeListed or dataVolume/directoryToBeListed
:param level: amount (int) of listed directory levels that are below or at the same level to that of the relativePath.
:param options: string of file filtering options.
:return: dictionary containing the directory listing.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error.
:example: fileServices = Files.getFileServices(); dirs = Files.dirList(fileServices[0], "Storage/myUserName/persistent/", level=2);
.. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.createDir
"""
token = Authentication.getToken()
if token is not None and token != "":
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.Files.dirList"
else:
taskName = "SciScript-Python.Files.dirList"
if type(fileService) == str:
fileService = getFileServiceFromName(fileService)
(topVolume, userVolumeOwner, userVolume, relativePath,
isTopVolumeARootVolume) = splitPath(path, fileService)
if isTopVolumeARootVolume:
url = __getFileServiceAPIUrl(
fileService
) + "api/jsontree/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?options=" + options + "&level=" + str(
level) + "&TaskName=" + taskName
else:
url = __getFileServiceAPIUrl(
fileService
) + "api/jsontree/" + topVolume + "/" + relativePath + "?options=" + options + "&level=" + str(
level) + "&TaskName=" + taskName
headers = {'X-Auth-Token': token}
res = requests.get(url, headers=headers)
if res.status_code >= 200 and res.status_code < 300:
return json.loads(res.content.decode())
else:
raise Exception(
"Error when listing contents of '" + str(path) +
"'.\nHttp Response from FileService API returned status code "
+ str(res.status_code) + ":\n" + res.content.decode())
else:
raise Exception("User token is not defined. First log into SciServer.")
def executeQuery(queryString, context="MyDB", acceptHeader="application/json+array", token="", format="pandas"):
"""Executes a casjob query. If a token is supplied then it will execute on behalf of the token's user.
format parameter specifies the return type:
'pandas': pandas.DataFrame
'csv': a csv string
'readable' : a StringIO, readable object wrapping a csv string that can be passed into pandas.read_csv for example.
'json': a dict created from a JSON string with the Query, a Result consisting of a Columns and a Data field.
"""
if (format == "pandas") or (format =="json"):
acceptHeader="application/json+array"
elif (format == "csv") or (format == "readable"):
acceptHeader = "text/plain"
else:
return {"Error":{"Message":"Illegal format specification '"+format+"'"}}
QueryUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/query"
query = {"Query": queryString, "TaskName": "SciScript-Python.SciServer.CasJobs.executeQuery"}
data = json.dumps(query).encode()
headers = {'Content-Type': 'application/json', 'Accept': acceptHeader}
if (token == ""):
try:
headers['X-Auth-Token'] = Authentication.getToken()
except:
pass
else:
headers['X-Auth-Token'] = token
try:
postResponse = requests.post(QueryUrl,data=data,headers=headers)
if postResponse.status_code != 200:
return {"Error":{"ErrorCode":postResponse.status_code,"Message":postResponse.content.decode()}}
r=postResponse.content.decode()
if (format == "readable"):
return StringIO(r)
elif format == "pandas":
r=json.loads(r)
return pandas.DataFrame(r['Result'][0]['Data'],columns=r['Result'][0]['Columns'])
elif format == "csv":
return r
elif format == "json":
return json.loads(r)
else: # should not occur
return {"Error":{"Message":"Illegal format specification '"+format+"'"}}
except requests.exceptions.RequestException as e:
return e
def upload(fileService, path, data="", localFilePath=None, quiet=True):
"""
Uploads data or a local file into a path defined in the file system.
:param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices().
:param path: path (in the remote file service) to the destination file (string), starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/destinationFile.txt or dataVolume/destinationFile.txt
:param data: string containing data to be uploaded, in case localFilePath is not set.
:param localFilePath: path to a local file to be uploaded (string),
:param userVolumeOwner: name (string) of owner of the userVolume. Can be left undefined if requester is the owner of the user volume.
:param quiet: If set to False, it will throw an error if the file already exists. If set to True. it will not throw an error.
:raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error.
:example: fileServices = Files.getFileServices(); Files.upload(fileServices[0], "myRootVolume/myUserName/myUserVolume/myUploadedFile.txt", None, localFilePath="/myFile.txt");
.. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.download, Files.dirList
"""
token = Authentication.getToken()
if token is not None and token != "":
if Config.isSciServerComputeEnvironment():
taskName = "Compute.SciScript-Python.Files.UploadFile"
else:
taskName = "SciScript-Python.Files.UploadFile"
if type(fileService) == str:
fileService = getFileServiceFromName(fileService)
(topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService);
if isTopVolumeARootVolume:
url = __getFileServiceAPIUrl(fileService) + "api/file/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName
else:
url = __getFileServiceAPIUrl(fileService) + "api/file/" + topVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName
headers = {'X-Auth-Token': token}
if localFilePath is not None and localFilePath != "":
with open(localFilePath, "rb") as file:
res = requests.put(url, data=file, headers=headers, stream=True)
else:
if data != None:
res = requests.put(url, data=data, headers=headers, stream=True)
else:
raise Exception("Error: No local file or data specified for uploading.");
if res.status_code >= 200 and res.status_code < 300:
pass;
else:
raise Exception("Error when uploading file to '" + str(path) + "' in file service '" + str(fileService.get('name')) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode());
else:
raise Exception("User token is not defined. First log into SciServer.")
def getJobStatus(jobid):
"""Gets a casjobs job status.
Returns the dict object (https://docs.python.org/3.4/library/stdtypes.html#dict) corresponding to the json received from casjobs."""
QueryUrl = Config.CasJobsRESTUri + "/jobs/" + str(jobid)
headers = {"X-Auth-Token": Authentication.getToken(), "Content-Type": "application/json"}
try:
postResponse = requests.get(QueryUrl, headers=headers)
return json.loads(postResponse.content.decode())
except requests.exceptions.RequestException as e:
return e.code
def upload(path, data, token=""):
if token == "":
userToken = Authentication.getToken()
else:
userToken = token
url = Config.SciDriveHost + "/vospace-2.0/1/files_put/dropbox/" + path
data = data
headers = {"X-Auth-Token": userToken}
try:
res = requests.put(url, data=data, headers=headers)
if Config.executeMode == "debug":
print(res.content.decode())
except requests.exceptions.RequestException as error:
if Config.executeMode == "debug":
print(error, error.read().decode())
raise
def listAllDatasets(token = ''):
"""
lists all available datasets
"""
schemaURL = Config.SkyQueryUrl + '/Schema.svc/datasets'
headers = {'Content-Type': 'application/json','Accept': 'application/json'}
if (token == ""):
headers['X-Auth-Token']= Authentication.getToken()
else:
headers['X-Auth-Token']= token
response = requests.get(schemaURL, headers=headers)
if response.status_code == 200:
return(response.json())
else:
return(response.text)
def listDatasetTables(datasetName, token = ''):
"""
Returns a list of all tables within a dataset
"""
url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables'
headers = {'Content-Type': 'application/json','Accept': 'application/json'}
if (token == ""):
headers['X-Auth-Token']= Authentication.getToken()
else:
headers['X-Auth-Token']= token
response = requests.get(url, headers=headers)
if response.status_code == 200:
return(response.json())
else:
return(response.text)
def getDatasetInfo(datasetName, token = ''):
"""
Gets information related to a particular dataset.
"""
schemaURL = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName
headers = {'Content-Type': 'application/json','Accept': 'application/json'}
if (token == ""):
headers['X-Auth-Token']= Authentication.getToken()
else:
headers['X-Auth-Token']= token
response = requests.get(schemaURL, headers=headers)
if response.status_code == 200:
return(response.json())
else:
return(response.text)
def getTableInfo(datasetName, tableName, token = ''):
"""
Returns info about a particular table belonging to a dataset.
"""
url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables/' + tableName
headers = {'Content-Type': 'application/json','Accept': 'application/json'}
if (token == ""):
headers['X-Auth-Token']= Authentication.getToken()
else:
headers['X-Auth-Token']= token
response = requests.get(url, headers=headers)
if response.status_code == 200:
return(response.json())
else:
return(response.text)
def getTable(datasetName, tableName, top = '', token = ''):
"""
Returns a dataset table as a pandas DataFrame. 'top' parameters makes the function to return only the first 'top' rows.
"""
url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + '?top=' + top
headers = {'Content-Type': 'application/json','Accept': 'application/json'}
if (token == ""):
headers['X-Auth-Token']= Authentication.getToken()
else:
headers['X-Auth-Token']= token
response = requests.get(url, headers=headers)
if response.status_code == 200:
return(pandas.read_csv(StringIO(response.content.decode()), sep="\t"))
else:
return(response.text)
def dropTable(datasetName, tableName, token = ''):
"""
Drops (deletes) a table from the user database.
"""
url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName
headers = {'Content-Type': 'application/json','Accept': 'application/json'}
if (token == ""):
headers['X-Auth-Token']= Authentication.getToken()
else:
headers['X-Auth-Token']= token
response = requests.delete(url, headers=headers)
if response.status_code == 200:
return(response.json())
else:
return(response.text)
def rectangularSearch(min_ra, max_ra, min_dec, max_dec, coordType="equatorial", whichPhotometry="optical", limit="10", token=""):
"""Runs a query in the SDSS database that searches for all objects within a certain rectangular box defined on the the sky, and retrieves the result table as a Panda's dataframe.\n
'min_ra': Minimum value of Right Ascension coordinate that defines the box boundaries on the sky.\n
'max_ra': Maximum value of Right Ascension coordinate that defines the box boundaries on the sky.\n
'min_dec': Minimum value of Declination coordinate that defines the box boundaries on the sky.\n
'max_dec': Maximum value of Declination coordinate that defines the box boundaries on the sky.\n
'coordType': Type of celestial coordinate system. Can be set to "equatorial" or "galactic".\n
'whichPhotometry': Type of retrieved data. Can be set to "optical" or "infrared".\n
'limit': Maximum number of rows in the result table (string). If set to "0", then the function will return all rows.\n
'token': Sciserver's authentication token for the user.\n
"""
url = Config.SkyServerWSurl + '/' + Config.DataRelease + '/SearchTools/RectangularSearch?'
url = url + 'format=csv&'
url = url + 'min_ra=' + str(min_ra) + '&'
url = url + 'max_ra=' + str(max_ra) + '&'
url = url + 'min_dec=' + str(min_dec) + '&'
url = url + 'max_dec=' + str(max_dec) + '&'
url = url + 'coordType=' + coordType + '&'
url = url + 'whichPhotometry=' + whichPhotometry + '&'
url = url + 'limit=' + limit + '&'
#url = urllib.quote_plus(url)
acceptHeader = "text/plain"
headers = {'Content-Type': 'application/json', 'Accept': acceptHeader}
if (token != ""):
headers['X-Auth-Token'] = token
else:
Token = ""
try:
Token = Authentication.getToken()
except:
Token = ""
if(Token != ""):
headers['X-Auth-Token'] = Token
try:
response = requests.get(url,headers=headers)
if response.status_code != 200:
return {"Error":{"ErrorCode":response.status_code,"Message":response.content.decode()}}
r=response.content.decode();
return pandas.read_csv(StringIO(r), comment='#')
except requests.exceptions.RequestException as e:
return e
def listQueues(token = ''):
"""
Returns a list of all available job queues and related metadata.
"""
jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues'
headers = {'Content-Type': 'application/json','Accept': 'application/json'}
if (token == ""):
headers['X-Auth-Token']= Authentication.getToken()
else:
headers['X-Auth-Token']= token
response = requests.get(jobsURL, headers=headers)
if response.status_code == 200:
r = response.content
return(r)
else:
raise NameError(response.json())
def getQueueInfo(queue, token = ''):
"""
Returns information about a particular job queue.
"""
jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue
headers = {'Content-Type': 'application/json','Accept': 'application/json'}
if (token == ""):
headers['X-Auth-Token']= Authentication.getToken()
else:
headers['X-Auth-Token']= token
response = requests.get(jobsURL, headers=headers)
if response.status_code == 200:
r = response.content
return(r)
else:
raise NameError(response.json())
def listTableColumns(datasetName, tableName, token = ''):
"""
Returns a list of all columns in a table belonging to a particular dataset.
Output: json string.
"""
url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables/' + tableName + '/columns'
headers = {'Content-Type': 'application/json','Accept': 'application/json'}
if (token == ""):
headers['X-Auth-Token']= Authentication.getToken()
else:
headers['X-Auth-Token']= token
response = requests.get(url, headers=headers)
if response.status_code == 200:
return(response.json())
else:
return(response.text)
def createContainer(path, token=""):
if (token == ""):
userToken = Authentication.getToken()
else:
userToken = token;
containerBody = ('<vos:node xmlns:xsi="http://www.w3.org/2001/thisSchema-instance" '
'xsi:type="vos:ContainerNode" xmlns:vos="http://www.ivoa.net/xml/VOSpace/v2.0" '
'uri="vos://' + Config.SciDriveHost + '!vospace/' + path + '">'
'<vos:properties/><vos:accepts/><vos:provides/><vos:capabilities/>'
'</vos:node>')
url = Config.SciDriveHost + '/vospace-2.0/nodes/' + path
data = str.encode(containerBody)
headers = {'X-Auth-Token': userToken, 'Content-Type': 'application/xml'}
try:
res = requests.put(url, data=data, headers=headers)
except requests.exceptions.RequestException as error:
if (Config.executeMode == "debug"):
print error, error.read().decode()
raise
def publicUrl(path, token):
"""
retrieve public URL for file identified by path
"""
if token == "":
userToken = Authentication.getToken()
else:
userToken = token
url = Config.SciDriveHost + "/vospace-2.0/1/media/sandbox/" + path
headers = {"X-Auth-Token": userToken}
try:
res = requests.get(url, headers=headers)
except requests.exceptions.RequestException as error:
if Config.executeMode == "debug":
print(error, error.read().decode())
raise
jsonRes = json.loads(res.content.decode())
fileUrl = jsonRes["url"]
return fileUrl
def cancelJob(jobID, token = ''):
"""
Cancels a single job.
Parameters:
jobID: the ID of the job, which is obtained at the moment of submitting the job.
token: SciServer's authentication token.
"""
statusURL = Config.SkyQueryUrl + '/Jobs.svc/jobs/' + jobID
headers = {'Content-Type': 'application/json','Accept': 'application/json'}
if (token == ""):
headers['X-Auth-Token']= Authentication.getToken()
else:
headers['X-Auth-Token']= token
response = requests.delete(statusURL, headers=headers)
if response.status_code == 200:
r = response.json()
return(r['queryJob'])
else:
raise NameError(response.json())
def submitJob(queryString, context="MyDB", acceptHeader="text/plain", token=""):
"""Submits a query to the casjobs queue. If a token is supplied then it will execute on behalf of the token's user.
Returns the casjobs jobID (int)."""
QueryUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/jobs"
query = {"Query": queryString, "TaskName": "SciScript-Python.SciServer.CasJobs.submitJob"}
data = json.dumps(query).encode()
headers = {"Content-Type": "application/json", "Accept": acceptHeader}
if token == "":
headers["X-Auth-Token"] = Authentication.getToken()
else:
headers["X-Auth-Token"] = token
try:
putResponse = requests.put(QueryUrl, data=data, headers=headers)
return int(putResponse.content.decode())
except requests.exceptions.RequestException as e:
return e
def getJobStatus(jobID, token = ''):
"""
Gets the status of a job, as well as other metadata.
Parameters:
jobID: the ID of the job, which is obtained at the moment of submitting the job.
token: SciServer's authentication token.
Output: json string with the job status and other related metadata.
"""
statusURL = Config.SkyQueryUrl + '/Jobs.svc/jobs/' + jobID
headers = {'Content-Type': 'application/json','Accept': 'application/json'}
if (token == ""):
headers['X-Auth-Token']= Authentication.getToken()
else:
headers['X-Auth-Token']= token
response = requests.get(statusURL, headers=headers)
if response.status_code == 200:
r = response.json()
return(r['queryJob'])
else:
raise NameError(response.json())
def listJobs(queue, token = ''):
"""
Lists the jobs in the queue in descending order by submission time. Only jobs of the authenticated user are listed.
Parameters:
queue: can be set to 'quick' for a quick job, or 'long' for a long job.
token: SciServer's authentication token.
Output: returns the jobID unique identifier of the job.
"""
jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + '/jobs?'
headers = {'Content-Type': 'application/json','Accept': 'application/json'}
if (token == ""):
headers['X-Auth-Token']= Authentication.getToken()
else:
headers['X-Auth-Token']= token
response = requests.get(jobsURL,headers=headers)
if response.status_code == 200:
r = response.json()
return(r)
else:
raise NameError(response.json())
def getJpegImgCutout(ra, dec, scale=0.7, width=512, height=512, opt="", query="", token = ""):
"""Gets a rectangular image cutout from a region of the sky in SDSS, centered at (ra,dec). Return type is numpy.ndarray.\n
'ra': Right Ascension of the image's center.\n
'dec': Declination of the image's center.\n
'scale': scale of the image, measured in [arcsec/pix]\n
'width': Right Ascension of the image's center.\n
'ra': Right Ascension of the image's center.\n
'height': Height of the image, measured in [pix].\n
'opt': Optional drawing options, expressed as concatenation of letters (string). The letters options are\n
\t"G": Grid. Draw a N-S E-W grid through the center\n
\t"L": Label. Draw the name, scale, ra, and dec on image.\n
\t"P PhotoObj. Draw a small cicle around each primary photoObj.\n
\t"S: SpecObj. Draw a small square around each specObj.\n
\t"O": Outline. Draw the outline of each photoObj.\n
\t"B": Bounding Box. Draw the bounding box of each photoObj.\n
\t"F": Fields. Draw the outline of each field.\n
\t"M": Masks. Draw the outline of each mask considered to be important.\n
\t"Q": Plates. Draw the outline of each plate.\n
\t"I": Invert. Invert the image (B on W).\n
\t(see http://skyserver.sdss.org/dr12/en/tools/chart/chartinfo.aspx)\n
'query': Optional string. Marks with inverted triangles on the image the position of user defined objects. The (RA,Dec) coordinates of these object can be given by three means:\n
\t1) query is a SQL command of format "SELECT Id, RA, Dec, FROM Table".
\t2) query is list of objects. A header with RA and DEC columns must be included. Columns must be separated by tabs, spaces, commas or semicolons. The list may contain as many columns as wished.
\t3) query is a string following the pattern: ObjType Band (low_mag, high_mag).
\t\tObjType: S | G | P marks Stars, Galaxies or PhotoPrimary objects.\n
\t\tBand: U | G | R | I | Z | A restricts marks to objects with Band BETWEEN low_mag AND high_mag Band 'A' will mark all objects within the specified magnitude range in any band (ORs composition).\n
\tExamples:\n
\t\tS\n
\t\tS R (0.0, 23.5)\n
\t\tG A (20, 30)\n
\t\t(see http://skyserver.sdss.org/dr12/en/tools/chart/chartinfo.aspx)\n
'token': Sciserver's authentication token for the user.
"""
url = Config.SkyServerWSurl + '/' + Config.DataRelease + '/ImgCutout/getjpeg?'
url = url + 'ra=' + str(ra) + '&'
url = url + 'dec=' + str(dec) + '&'
url = url + 'scale=' + str(scale) + '&'
url = url + 'width=' + str(width) + '&'
url = url + 'height=' + str(height) + '&'
url = url + 'opt=' + opt + '&'
url = url + 'query=' + query + '&'
url = urllib.quote_plus(url)
acceptHeader = "text/plain"
headers = {'Content-Type': 'application/json', 'Accept': acceptHeader}
if (token != ""):
headers['X-Auth-Token'] = token
else:
Token = ""
try:
Token = Authentication.getToken()
except:
Token = ""
if(Token != ""):
headers['X-Auth-Token'] = Token
try:
response = requests.get(url,headers=headers)
if response.status_code != 200:
return {"Error":{"ErrorCode":response.status_code,"Message":response.content.decode()}}
return skimage.io.imread( BytesIO( response.content ) )
except requests.exceptions.RequestException as e:
return e