method='get'
#make the rest call
serverlist_result_json=callrestapi(endpoint,method)
if debug:
print(serverlist_result_json)
print('serverlist_result_json is a '+type(serverlist_result_json).__name__+' object') #serverlist_result_json is a dictionary
servers = serverlist_result_json['items']
for server in servers:
servername=server['name']
serverhost=server['host']
serverport=server['port']
serverrest=server['restPort']
print(servername+','+serverhost+','+str(serverport)+','+str(serverrest))
# connect to each CAS server
s = swat.CAS(serverhost, serverport)
# TLS relies on env var $CAS_CLIENT_SSL_CA_LIST=/opt/sas/viya/config/etc/SASSecurityCertificateFramework/cacerts/vault-ca.crt (or local location)
# else "ERROR: SSL Error: Missing CA trust list"
# get CAS_DISK_CACHE usage
s.accessControl.assumeRole(adminRole="superuser") # superuser role reqd
results = s.builtins.getCacheInfo # returns dictionary and table
# display table with CAS_DISK_CACHE usage stats
print(tabulate(results,headers='firstrow'))
def test_context_manager(self):
with swat.CAS(HOST, PORT, USER, PASSWD) as s:
self.assertTrue(len(s.serverstatus()) > 0)
with self.assertRaises(swat.SWATError):
s.serverstatus()
#!/usr/bin/env python
# encoding: utf-8
#
# Copyright © 2019, SAS Institute Inc., Cary, NC, USA. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
import swat
from sasctl import Session
from sasctl.tasks import register_model, publish_model
with swat.CAS('hostname', 5570, 'username', 'password') as cas:
# Load the regression actions in CAS
cas.loadactionset('regression')
# Upload the local CSV file to a CAS table
tbl = cas.upload('data/boston_house_prices.csv').casTable
# Model input features are everything except the target
features = tbl.columns[tbl.columns != 'medv']
# Fit a linear regression model in CAS and output an ASTORE
tbl.glm(target='medv', inputs=list(features), savestate='model_table')
astore = cas.CASTable('model_table')
# Use sasctl to connect to SAS
Session('hostname', 'username', 'password')
# Register the model in SAS Model Manager, creating the "Boston Housing"
# project if it doesn't already exist
model = register_model(astore,
def test_connection_repr(self):
s = swat.CAS(u'%s' % HOST, PORT, USER, PASSWD, name=u'\u2603')
self.assertEqual(str(s), repr(s))
def test_name(self):
user, passwd = tm.get_user_pass()
s = swat.CAS(HOST, PORT, USER, PASSWD, name='My Connection')
name = list(s.sessionid().keys())[0].split(':')[0]
self.assertEqual(s._name, name)
# packages required for clustering process
import pandas as pd
import numpy as np
import swat
import random
import itertools as ite
import time
import datetime
# connect to CAS server for loading data
# input port, username and password provided, for security, in this code, port, username and password is hidden
s = swat.CAS(host = 'cas-shared-default', port = ****, username = '******', password = '******')
# uncomment and run 2 following lines for the first run through
#distance_full = s.load_path(path = "DISTANCE_DATA.sashdat", caslib = 'public', casout = 'distance_full5', promote = 'TRUE')
#avg_full = s.load_path(path = "ZZZ_BIKES_FINAL2.sashdat", caslib = 'public', casout = 'AVG_full_final1', promote = 'TRUE')
def data_prep(distance_full, avg_full, threshold_cluster, Datetimequery):
'''
This function formats the data to be used in the clustering model
'''
distance_full = distance_full.to_frame()
avg_full = avg_full.to_frame()
avg_full_frame = avg_full[avg_full['Date_Time'] == Datetimequery]
avg_full_frame['old_index'] = avg_full_frame.index #store old index to other column to change the index
avg_full_frame = avg_full_frame.drop(columns = ['name', 'Station_Activity','Station_Freq', 'Status']) #drop unnescessary columns
avg_full_frame['demand'] = (avg_full_frame.Available_Bikes - avg_full_frame.Avg_Avail_Bikes) #calculate demand values
# adjust extreme demand values of each station (if too high) to less than or equal to the capacity of the vehicle (18)
temp_dem_list = list(avg_full_frame.demand)
for i in range(len(temp_dem_list)):
import swat
# Specify the host and port information match your site
s = swat.CAS("dsascontrol.org", 8777)
from IPython.core.display import display
result = s.table.upload(
path="/home/viya/PGV.csv",
casout={
"name":"PGV",
"replace":True}
)
PGV = s.CASTable(result.tableName)
# Display Table Info for PGV
ti = PGV.table.tableInfo().TableInfo
display(HTML('<h3>Table Information</h3>'))
display(ti.ix[:,'Name':'JavaCharSet'])
display(ti.ix[:,'ModTime':])
# Display Table Details for PGV
td = PGV.table.tableDetails().TableDetails
display(HTML('<h3>Table Details</h3>'))
display(td.ix[:,'Node':'VardataSize'])
display(td.ix[:,'Mapped':])
def test_connection_failure(self):
user, passwd = tm.get_user_pass()
with self.assertRaises(swat.SWATError):
swat.CAS(HOST, 1999, USER, PASSWD, protocol=PROTOCOL)
def test_connection_failure(self):
user, passwd = tm.get_user_pass()
with captured_stderr() as out: # noqa: F841
with self.assertRaises(swat.SWATError):
swat.CAS(re.sub(r':\d+(/|$)', r'\1', HOST), 1999,
USER, PASSWD, protocol=PROTOCOL)
import swat
import os
# lab10 environment:
# cas_shared_default_srv --> lab10-viyac1.lab.local
# CAS Server configured with it's own certificate signed by Nordic Labs
# This certificate is in the windows trust store
cas_shared_default_srv = "lab10-viyac1.lab.local"
# cas_shared_dev_srv --> lab10-viyadev.lab.local
# CS Server used the default Viya certificate
cas_shared_dev_srv = "lab10-viyadev.lab.local"
#---------------- SELECT THE CAS SERVER ----------------------
selected_cas = cas_shared_default_srv
#-------------------------------------------------------------
if (selected_cas == cas_shared_dev_srv):
os.environ[
'CAS_CLIENT_SSL_CA_LIST'] = r"/opt/sasviyarepo/lab10/viya-certificates/vault-ca.crt"
# Connect to the cas server and get a CAS Session
conn = swat.CAS(selected_cas, 5570)
print(conn)
# Get server status
print(conn.builtins.serverStatus())
# End CAS Session
r = conn.session.endSession()
print(r)
#https://sassoftware.github.io/python-swat/getting-started.html
#https://github.com/sassoftware/python-swat
import swat
sess = swat.CAS('sasserver.demo.sas.com', 5570, 'sasdemo', 'Orion123')
file_name = "viya_reg.csv"
indata = "viya_reg"
sess.loadactionset(actionset="table")
if not sess.table.tableExists(table=indata).exists:
tbl = sess.upload_file(file_name, casout={"name": indata}, promote='True')
tbl.head()
sess.close()
#%% Import Packages
import riskpy
import swat
import os
from sys import platform
import datetime as dt
import seaborn as sns
import matplotlib.pyplot as plt
from os.path import join as path
my_run = riskpy.SessionContext(session=swat.CAS(), caslib="CASUSER")
# Set output location
if platform == "win32":
# Windows...
output_dir = 'u:\\temp'
else:
# platform == "linux" or platform == "linux2" or platform == "darwin":
output_dir = '/tmp'
#%% Create market data object
market_data = riskpy.MarketData(current=path("datasources", "MarketRisk",
"current.xlsx"),
historical=path("datasources", "MarketRisk",
"history.xlsx"))
#%% Simulate Market States
market_states = riskpy.MarketStates(market_data=market_data,
as_of_date=dt.datetime(2020, 3, 2),
##
if __name__ == "__main__":
if os.environ.get('VCAP_SERVICES') is None:
# running locally, let's debug
PORT = 8989
DEBUG = True
else:
# running in cloudfoundry
PORT = int(os.getenv("PORT"))
DEBUG = False
# connect to CAS with which creds? caslib=cas_library
#
# auth using authinfo file in local filesystem will be a problem in CF
cas_session = swat.CAS(hostname="bbviya3.pre-sal.sashq-r.openstack.sas.com", port=5570,
authinfo='/home/centos/.authinfo',
caslib=cas_library, name="brad")
# load a few required CAS action sets
results = cas_session.loadactionset('astore')
results = cas_session.loadactionset('decisiontree')
if DEBUG:
print(cas_session)
# create a reference to the model table;
modeltbl = cas_session.CASTable(name=model_name, caslib=cas_library)
if DEBUG:
print(cas_session.table.fileinfo(caslib=cas_library,path="%"))
SAS SWAT is used to work with CAS objects such as data sets and models from Python.
sasctl is used to register a model into the model repository.
For more information about the SAS SWAT and sasctl packages, see the Overview section of <Developing Models Using Python>.
<Note:Currently, only the models developed in scikit-learn are supported>.
"""
import swat
from sasctl import Session
from sasctl.tasks import register_model
from sklearn.linear_model import LogisticRegression
"""
Use the CAS constructor provided by SAS SWAT to establish a connection with the CAS server.
Note: The default port for the connection between Python and SAS SWAT is 5570.
"""
conn=swat.CAS(host,port,authinfo='/path/to/.authinfo')
"""
Provide the complete path with the file name of your authinfo or netrc file.
For Example,
conn=swat.CAS('my-cas-host.com',5570,authinfo='~/.authinfo')
Only the owner of the authinfo file must have read and write accesses to the file.
"""
#Section b: Importing the analytical data set created in SAS Analytical Data Builder
#Provide the name of the CAS modeling data set in the following code snippet:
tbl = conn.CASTable(name = 'ModelingDataSet', caslib='rm_mdl')
#Note: Do not change the name of the CAS library.
""" For example,
def as_swat(self, server=None, **kwargs):
"""Create a SWAT connection to a CAS server.
Uses the authentication information from the session to establish a CAS connection using SWAT.
Parameters
----------
server : str, optional
The logical name of the CAS server, not the hostname. Defaults to "cas-shared-default".
kwargs : any
Additional arguments to pass to the `swat.CAS` constructor. Can be used to override this method's
default behavior or customize the CAS session.
Returns
-------
swat.CAS
An active SWAT connection
Raises
------
RuntimeError
If `swat` package is not available.
Examples
--------
>>> sess = Session('example.sas.com')
>>> with sess.as_swat() as conn:
... conn.listnodes()
CASResults([('nodelist', Node List
name role connected IP Address
0 example.sas.com controller Yes 127.0.0.1)])
"""
server = server or 'cas-shared-default'
if swat is None:
raise RuntimeError(
"The 'swat' package must be installed to create a SWAT connection."
)
# Construct the CAS server's URL
url = '{}://{}/{}-http/'.format(self._settings['protocol'],
self.hostname, server)
# Use this sessions info to connect to CAS unless user has explicitly give a value (even if None)
kwargs.setdefault('hostname', url)
kwargs.setdefault('username', self.username)
kwargs.setdefault('password', self._settings['password'])
orig_sslreqcert = os.environ.get('SSLREQCERT')
# If SSL connections to microservices are not being verified, don't attempt
# to verify connections to CAS - most likely certs are not in place.
if not self.verify:
os.environ['SSLREQCERT'] = 'no'
try:
cas = swat.CAS(**kwargs)
cas.setsessopt(messagelevel='warning')
finally:
# Reset environment variable to whatever it's original value was
if orig_sslreqcert:
os.environ['SSLREQCERT'] = orig_sslreqcert
return cas
def test_connect_with_bad_session(self):
user, passwd = tm.get_user_pass()
with captured_stderr() as out: # noqa: F841
with self.assertRaises(swat.SWATError):
swat.CAS(HOST, PORT, USER, PASSWD,
protocol=PROTOCOL, session='bad-session')
host = 'https://newviyawaves.sas.com/cas-shared-default-http/'
username = '******'
password = '******'
caslib = 'casuser'
table_name = 'covid19'
# check new file and update every x seconds
duration = 60 * 60 * 5
baseurl = 'https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_daily_reports/'
# In[ ]:
# Connect to CAS server
s = swat.CAS(host, port, username, password)
# In[ ]:
while True:
update = False
date = dt.datetime.today()
while update == False:
try:
url = baseurl + str(date.strftime('%m-%d-%Y')) + '.csv'
r = requests.get(url)
logging.info('Try: ', url)
r.raise_for_status()
except HTTPError:
import pandas as pd
import swat
# Session variable
_HOST = '10.96.15.151'
_PORT = '5570'
_USERNAME = '******'
_PASSWORD = '******'
_CASLIB = 'Public'
_PATH = "./Data/performances/*.csv"
_MODEL_ID = "_b377316a-ce39-4528-929a-5dd2785a2b93_champion"
# Start CAS Session
cas_session = swat.CAS(_HOST, _PORT, _USERNAME, _PASSWORD)
out = cas_session.serverstatus()
# print(out)
# For each csv, load into caslib Public with 30 secs
print("Loading Performance Table...")
for tblpath in glob.glob(_PATH):
try:
tbl_csv = os.path.basename(tblpath)
df = pd.read_csv(tblpath, sep=';', header=0)
# replace=True
# print(tbl_csv)
tblname = tbl_csv[:-4] + _MODEL_ID
perftbl = cas_session.upload_frame(df,
import swat
import sys
import pandas as pd
# Connect to an existing CAS Session passed as argument
mysession = sys.argv[1]
cashost = 'localhost'
casport = 5570
casauth = '~/.authinfo'
s = swat.CAS(cashost, casport, authinfo=casauth, session=mysession)
s.loadactionset('decisionTree')
# load my observation to CAS
tbl = pd.read_csv('./newObsToScore.csv')
score_tbl = s.upload_frame(tbl, casout=dict(name='score', replace=True))
# score my observation
s.decisionTree.dtreeScore(table={'name': 'score'},
modeltable={'name': 'DT_MODEL'},
casOut={'name': 'scored_obs'})
# save score observation to local
out = s.CASTable("scored_obs").to_frame()
out.to_csv('./newObsScored.csv')
password = '******'
astore_table = 'gb_astore'
#astore_caslib = 'public'
###################################
####### Getting astore table ######
with open("session_id.txt", "r") as f:
session_id = f.read()
f.close()
conn = swat.CAS(#'hostname.com', port=8777, protocol = 'http',
host, port = 5570, ## bug on swat 1.6.0
#caslib = 'public',
username = '******',
password = '******')#, session = session_id)
astore = conn.CASTable(astore_table)
#### coneccting from SASCTL
s = Session(host, user, password,
verify_ssl = False)
model_exists = model_repository.get_model(modelname, refresh=False)
#model_repository.delete_model(modelname)
# To install the SAS SWAT package, run either of these commands:
# pip install swat
# conda install -c sas-institute swat
# pip install https://github.com/sassoftware/python-swat/releases/download/vX.X.X/python-swat-X.X.X-platform.tar.gz
# or visit:
# https://github.com/sassoftware/python-swat
# Prior to running this program, you need to have a CAS server.
# Example of using Python to call the iml action
import swat # load the swat package
s = swat.CAS('your_host_name', SMPPort) # server='myhost'; port=12345
s.loadactionset('iml') # load the action set
# submit your program to the action
m = s.iml(code="""
c = {1, 2, 1, 3, 2, 0, 1}; /* weights */
x = {0, 2, 3, 1, 0, 2, 2}; /* data */
wtSum = c` * x; /* inner product (weighted sum) */
var = var(x); /* variance of original data */
stdX = (x-mean(x))/std(x); /* standarize data */
var2 = var(stdX); /* variance of standardized data */
print wtSum var var2;
""")
# display the results of the action
print(m)
modelname = 'python_jk_lreg'
project = 'hmeq_os'
publishdestination = 'maslocal'
data_path = './data/hmeq_score.csv'
model_filename = 'pylreg.pickle'
########
conn = swat.CAS( #host, port=8777, protocol = 'http',
'localhost',
port=5570, ## bug on swat 1.6.0
caslib='casuser',
username='******',
password='******') #, session = session_id)
s = Session(host, 'username', 's3cr3t!', verify_ssl=False)
model = pickle.load(open(model_filename, 'rb'))
ctbl = conn.CASTable(name='hmeq', caslib='public')
table = ctbl.to_frame()
### avoid using variable names with . it will have error with DS2
inputs = table.drop('BAD', axis=1)
Y = table.BAD.astype('category')
yCategory = Y.cat.categories
import swat
conn = swat.CAS("centis.example.local",
5570,
authinfo="~/_authinfo",
caslib="casuser")
conn.serverstatus()
#conn.close()
# -*- coding: utf-8 -*-
"""
Created on Thu Jun 4 15:01:52 2020
@author: edhell
"""
###
import swat
import sys
conn = swat.CAS( #'hostname.com', port=8777, protocol = 'http',
'localhost',
port=5570, ## bug on swat 1.6.0
caslib='public',
username='******',
password='******')
session_id = list(conn.session.sessionId().values())[0]
conn.sessionProp.setSessOpt(casLib="public", timeOut=3600)
tablenames = [
'hmeq', 'hmeqpr_1_1q', 'hmeqpr_2_2q', 'hmeqpr_3_3q', 'hmeqpr_4_4q'
]
tablenames[0]
tables = dict()
for i in tablenames:
def test_bad_host(self):
with self.assertRaises(swat.SWATError):
swat.CAS('junk-host-name', PORT, USER, PASSWD)
#!/usr/bin/env python
import swat
from sasctl import Session
from sasctl.tasks import register_model, publish_model
with swat.CAS('dsascontrol.org', 5570, 'robinswu', 'password') as cas:
# Load the regression actions in CAS
cas.loadactionset('regression')
# Upload the local CSV file to a CAS table
tbl = cas.upload('/home/viya/data/Los_Angeles_houses_prices.csv').casTable
# Model input features are everything except the target
features = tbl.columns[tbl.columns != 'medv']
# Fit a linear regression model in CAS and output an ASTORE
tbl.glm(target='medv', inputs=list(features), savestate='model_table')
astore = cas.CASTable('model_table')
# Use sasctl to connect to SAS
Session('dsasspre.org', 'robinswu', 'password')
# Register the model in SAS Model Manager, creating the "Boston Housing"
# project if it doesn't already exist
model = register_model(astore,
'Linear Regression',
'Los Angeles Housing',
force=True)
def test_multiple_hosts(self):
user, passwd = tm.get_user_pass()
if self.s._protocol in ['http', 'https']:
unittest.TestCase.skipTest(self, 'REST does not support multiple hosts yet')
with swat.CAS([HOST, 'foo', 'bar'], PORT, USER, PASSWD) as s:
self.assertTrue(len(s.serverstatus()) > 0)
# -*- coding: utf-8 -*-
"""
Created on Sun Dec 2 16:13:06 2018
@author: sdksbe
"""
import swat
import os
os.environ[
'CAS_CLIENT_SSL_CA_LIST'] = r"C:/dev/viyacertificates/lab01-viya34smp/vault-ca.crt"
#os.environ['REQUESTS_CA_BUNDLE']=r"C:/dev/viyacertificates/lab01-viya34smp/vault-ca.crt"
conn = swat.CAS('viya34smp.nordiclab.sashq-r.openstack.sas.com', 5570)
csvtbl = conn.read_csv(
'https://raw.githubusercontent.com/sassoftware/sas-viya-programming/master/data/cars.csv'
)
h = csvtbl.head()
print(h)
r = conn.session.endSession()
print(r)
def open_cas_connection(host, port, authinfo=".authinfo", protocol="cas"):
conn = swat.CAS(host, port, authinfo=authinfo, protocol=protocol)
return conn
def update_model_performance(data, model, label, refresh=True):
"""Upload data for calculating model performance metrics.
Model performance and data distributions can be tracked over time by
designating one or more tables that contain input data and target values.
Performance metrics can be updated by uploading a data set for a new time
period and executing the performance definition.
Parameters
----------
data : Dataframe
model : str or dict
The name or id of the model, or a dictionary representation of
the model.
label : str
The time period the data is from. Should be unique and will be
displayed on performance charts. Examples: 'Q1', '2019', 'APR2019'.
refresh : bool, optional
Whether to execute the performance definition and refresh results with
the new data.
Returns
-------
CASTable
The CAS table containing the performance data.
See Also
--------
:meth:`model_management.create_performance_definition <.ModelManagement.create_performance_definition>`
.. versionadded:: v1.3
"""
try:
import swat
except ImportError:
raise RuntimeError("The 'swat' package is required to save model "
"performance data.")
# Default to true
refresh = True if refresh is None else refresh
model_obj = mr.get_model(model)
if model_obj is None:
raise ValueError('Model %s was not found.' % model)
project = mr.get_project(model_obj.projectId)
if project.get('function', '').lower() not in ('prediction', 'classification'):
raise ValueError("Performance monitoring is currently supported for "
"regression and binary classification projects. "
"Received project with '%s' function. Should be "
"'Prediction' or 'Classification'." % project.get('function'))
if project.get('targetLevel', '').lower() not in ('interval', 'binary'):
raise ValueError("Performance monitoring is currently supported for "
"regression and binary classification projects. "
"Received project with '%s' target level. Should be "
"'Interval' or 'Binary'." % project.get('targetLevel'))
if project.get('predictionVariable', '') == '' and project.get('function',
'').lower() == 'prediction':
raise ValueError("Project '%s' does not have a prediction variable "
"specified." % project)
if project.get('eventProbabilityVariable', '') == '' and project.get(
'function', '').lower() == 'classification':
raise ValueError("Project '%s' does not have an Event Probability variable "
"specified." % project)
# Find the performance definition for the model
# As of Viya 3.4, no way to search by model or project
perf_def = None
for p in mm.list_performance_definitions():
if project.id in p.projectId:
perf_def = p
break
if perf_def is None:
raise ValueError("Unable to find a performance definition for model "
"'%s'" % model)
# Check where performance datasets should be uploaded
cas_id = perf_def['casServerId']
caslib = perf_def['dataLibrary']
table_prefix = perf_def['dataPrefix']
# All input variables must be present
missing_cols = [col for col in perf_def.inputVariables if
col not in data.columns]
if missing_cols:
raise ValueError("The following columns were expected but not found in "
"the data set: %s" % ', '.join(missing_cols))
# If CAS is not executing the model then the output variables must also be
# provided
if not perf_def.scoreExecutionRequired:
missing_cols = [col for col in perf_def.outputVariables if
col not in data.columns]
if missing_cols:
raise ValueError(
"The following columns were expected but not found in the data "
"set: %s" % ', '.join(missing_cols))
sess = current_session()
url = '{}://{}/{}-http/'.format(sess._settings['protocol'],
sess.hostname,
cas_id)
regex = r'{}_(\d+)_.*_{}'.format(table_prefix, model_obj.id)
# Save the current setting before overwriting
orig_sslreqcert = os.environ.get('SSLREQCERT')
# If SSL connections to microservices are not being verified, don't attempt
# to verify connections to CAS - most likely certs are not in place.
if not sess.verify:
os.environ['SSLREQCERT'] = 'no'
# Upload the performance data to CAS
with swat.CAS(url,
username=sess.username,
password=sess._settings['password']) as s:
s.setsessopt(messagelevel='warning')
with swat.options(exception_on_severity=2):
caslib_info = s.table.tableinfo(caslib=caslib)
all_tables = getattr(caslib_info, 'TableInfo', None)
if all_tables is not None:
# Find tables with similar names
perf_tables = all_tables.Name.str.extract(regex,
flags=re.IGNORECASE,
expand=False)
# Get last-used sequence number
last_seq = perf_tables.dropna().astype(int).max()
next_seq = 1 if math.isnan(last_seq) else last_seq + 1
else:
next_seq = 1
table_name = '{prefix}_{sequence}_{label}_{model}'.format(
prefix=table_prefix,
sequence=next_seq,
label=label,
model=model_obj.id
)
with swat.options(exception_on_severity=2):
# Table must be promoted so performance jobs can access.
result = s.upload(data, casout=dict(name=table_name,
caslib=caslib,
promote=True))
if not hasattr(result, 'casTable'):
raise RuntimeError('Unable to upload performance data to CAS.')
tbl = result.casTable
# Restore the original value
if orig_sslreqcert is not None:
os.environ['SSLREQCERT'] = orig_sslreqcert
# Execute the definition if requested
if refresh:
mm.execute_performance_definition(perf_def)
return tbl
