def refreshHisList(self):
"""
This function retrieves every histories in the server and returns a list of id
"""
print('Retrieving list of histories (trends) in server, please wait...')
self.allHistories = Histories(self)
print('Complete... Use hisAll() to check for trends or refreshHisList() to refresh the list')
print('Try hisRead() to load a bunch of trend matching criterias')
class Connect():
"""
Abstact class / Make a connection object to haystack server using requests module
A class must be made for different type of server. See NiagaraAXConnection(HaystackConnection)
"""
def __init__(self, url, username, password,**kwargs):
"""
Set local variables
Open a session object that will be used for connection, with keep-alive feature
baseURL : http://XX.XX.XX.XX/ - Server URL
queryURL : ex. for nhaystack = baseURL+haystack = http://XX.XX.XX.XX/haystack
USERNAME : used for login
PASSWORD : used for login
**kwargs :
zinc = False or True (compatibility for old device like NPM2 that cannot generate Json coding)
COOKIE : for persistent login
isConnected : flag to be used for connection related task (don't try if not connected...)
s : requests.Session() object
_filteredList : List of histories created by getFilteredHistories
timezone : timezone from site description
"""
self.baseURL = url
self.queryURL = ''
self.USERNAME = username
self.PASSWORD = password
self.COOKIE = ''
self.isConnected = False
self.s = requests.Session()
self._filteredList = []
self.timezone = 'UTC'
self._forceZincToJson = kwargs.pop('zinc','False')
def authenticate(self):
"""
This function must be overridden by specific server connection to fit particular needs (urls, other conditions)
"""
pass
def read(self,urlToGet):
if self._forceZincToJson:
return self.getZinc(urlToGet)
else:
return self.getJson(urlToGet)
def getJson(self,urlToGet):
"""
Helper for GET request. Retrieve information as json string objects
urlToGet must include only the request ex. "read?filter=site"
Queryurl (ex. http://serverIp/haystack) is already known
"""
if self.isConnected:
try:
req = self.s.get(self.queryURL + urlToGet, headers={'accept': 'application/json; charset=utf-8'})
return json.loads(req.text)
except requests.exceptions.RequestException as e:
print('Request GET error : %s' % e)
#else:
#print 'Session not connected to server, cannot make request'
else:
print('Session not connected to server, cannot make request')
def getZinc(self,urlToGet):
"""
Helper for GET request. Retrieve information as default Zinc string objects
"""
if self.isConnected:
try:
req = self.s.get(self.queryURL + urlToGet, headers={'accept': 'text/plain; charset=utf-8'})
return json.loads(zincToJson(req.text))
except requests.exceptions.RequestException as e:
print('Request GET error : %s' % e)
else:
print('Session not connected to server, cannot make request')
def postRequest(self,url,headers={'token':''}):
"""
Helper for POST request
"""
try:
req = self.s.post(url, params=headers,auth=(self.USERNAME, self.PASSWORD))
#print 'Post request response : %s' % req.status_code
#print 'POST : %s | url : %s | headers : %s | auth : %s' % (req, url, headers,self.USERNAME) Gives a 404 response but a connection ????
except requests.exceptions.RequestException as e: # This is the correct syntax
print('Request POST error : %s' % e)
def refreshHisList(self):
"""
This function retrieves every histories in the server and returns a list of id
"""
print('Retrieving list of histories (trends) in server, please wait...')
self.allHistories = Histories(self)
print('Complete... Use hisAll() to check for trends or refreshHisList() to refresh the list')
print('Try hisRead() to load a bunch of trend matching criterias')
def hisAll(self):
"""
Returns all history names and id
"""
return self.allHistories.getListofIdsAndNames()
def readAll(self,filterRequest):
"""
Returns result of filter request
"""
# Should add some verification here
req = 'read?filter=' + filterRequest
result = self.read(req)
for each in result['rows']:
print('%s' % each['dis'])
return HReadAllResult(self,result)
def hisRead(self,**kwargs):
"""
This method returns a list of history records
arguments are :
ids : a ID or a list of ID
AND_search : a list of keywords to look for in trend names
OR_search : a list of keywords to look for in trend names
rng : haystack range (today,yesterday, last24hours...
start : string representation of start time ex. '2014-01-01T00:00'
end : string representation of end time ex. '2014-01-01T00:00'
"""
self._filteredList = [] # Empty list to be returned
# Keyword Arguments
print(kwargs)
ids = kwargs.pop('id','')
AND_search = kwargs.pop('AND_search','')
OR_search = kwargs.pop('OR_search','')
rng = kwargs.pop('rng','')
start = kwargs.pop('start','')
end = kwargs.pop('end','')
takeall = kwargs.pop('all','')
# Remaining kwargs...
if kwargs: raise TypeError('Unknown argument(s) : %s' % kwargs)
# Build datetimeRange based on start and end
if start and end:
datetimeRange = start+','+end
else:
datetimeRange = rng
# Find histories matching ALL keywords in AND_search
for eachHistory in self.hisAll():
takeit = False
# Find histories matching ANY keywords in OR_search
if (AND_search != '') and all([keywords in eachHistory['name'] for keywords in AND_search]):
print('AND_search : Adding %s to recordList' % eachHistory['name'])
takeit = True
# Find histories matching ANY ID in id list
elif (OR_search != '') and any([keywords in eachHistory['name'] for keywords in OR_search]):
print('OR_search : Adding %s to recordList' % eachHistory['name'])
takeit = True
elif (ids != '') and any([id in eachHistory['id'] for id in ids]):
print('ID found : Adding %s to recordList' % eachHistory['name'])
takeit = True
elif takeall != '':
print('Adding %s to recordList' % eachHistory['name'])
takeit = True
if takeit:
self._filteredList.append(HisRecord(self,eachHistory['id'],datetimeRange))
if self._filteredList == []:
print('No trends found... sorry !')
return self._filteredList
class Connect():
"""
Abstact class / Make a connection object to haystack server using requests module
A class must be made for different type of server. See NiagaraAXConnection(HaystackConnection)
"""
def __init__(self, url, username, password, proj, **kwargs):
"""
Set local variables
Open a session object that will be used for connection, with keep-alive feature
baseURL : http://XX.XX.XX.XX/ - Server URL
queryURL : ex. for nhaystack = baseURL+haystack = http://XX.XX.XX.XX/haystack
USERNAME : used for login
PASSWORD : used for login
**kwargs :
zinc = False or True (compatibility for old equip like NPM2 that cannot generate Json coding)
COOKIE : for persistent login
isConnected : flag to be used for connection related task (don't try if not connected...)
s : requests.Session() object
_filteredList : List of histories created by getFilteredHistories
timezone : timezone from site description
"""
self.baseURL = url
self.queryURL = ''
self.USERNAME = username
self.PASSWORD = password
self.PROJECT = proj
self.COOKIE = ''
self.isConnected = False
self.s = requests.Session()
self._filteredList = []
self.timezone = 'UTC'
self._forceZincToJson = kwargs.pop('zinc', False)
def authenticate(self):
"""
This function must be overridden by specific server connection to fit particular needs (urls, other conditions)
"""
pass
def read(self, urlToGet, data=False):
if self._forceZincToJson:
return self.getZinc(urlToGet, data)
else:
return self.getJson(urlToGet, data)
def getJson(self, urlToGet, data):
"""
Helper for GET request. Retrieve information as json string objects
urlToGet must include only the request ex. "read?filter=site"
Queryurl (ex. http://serverIp/haystack) is already known
"""
if self.isConnected:
try:
if data:
req = self.s.post(self.queryURL + urlToGet, data=data,
headers={'accept': 'application/json; charset=utf-8',
'content-type': 'text/zinc'})
else:
req = self.s.get(self.queryURL + urlToGet, headers={'accept': 'application/json; charset=utf-8'})
return json.loads(req.text)
except requests.exceptions.RequestException as e:
print('Request GET error : %s' % e)
# else:
# print 'Session not connected to server, cannot make request'
else:
print('Session not connected to server, cannot make request')
def getZinc(self, urlToGet, data):
"""
Helper for GET request. Retrieve information as default Zinc string objects
"""
if self.isConnected:
try:
if data:
req = self.s.post(self.queryURL + urlToGet, data=data)
# print req.text.encode('ascii','ignore')
else:
req = self.s.get(self.queryURL + urlToGet, headers={'accept': 'text/plain; charset=utf-8'})
# print req.text.encode('ascii','ignore')
print json.loads(zincToJson(req.text))
return json.loads(zincToJson(req.text))
except requests.exceptions.RequestException as e:
print('Request GET error : %s' % e)
else:
print('Session not connected to server, cannot make request')
def postRequest(self, url, headers={'token': ''}, data=''):
"""
Helper for POST request
"""
try:
req = self.s.post(url, params=headers, auth=(self.USERNAME, self.PASSWORD), data=data)
# print 'Post request response : %s' % req.status_code
print 'POST : %s | url : %s | headers : %s | auth : %s' % (req, url, headers, self.USERNAME)
except requests.exceptions.RequestException as e: # This is the correct syntax
print('Request POST error : %s' % e)
def refreshHisList(self):
"""
This function retrieves every histories in the server and returns a list of id
"""
print('Retrieving list of histories (trends) in server, please wait...')
self.allHistories = Histories(self)
print('Complete... Use hisAll() to check for trends or refreshHisList() to refresh the list')
print('Try hisRead() to load a bunch of trend matching criterias')
def hisAll(self):
"""
Returns all history names and id
"""
return self.allHistories.getListofIdsAndNames()
def readAll(self, filterRequest):
"""
Returns result of filter request
:rtype : pyhaystack.haystackIO.haystackRead.HReadAllResult
:type filterRequest : str
"""
# Should add some verification here
req = 'read?filter=' + filterRequest.replace("=","%3D")
result = self.read(req)
if len(result['rows']) > 1:
for each in result['rows']:
print('id: ' + each['id'])
return pyhaystack.haystackIO.haystackRead.HReadAllResult(self, result)
def hisRead(self, **kwargs):
"""
This method returns a list of history records
arguments are :
ids : a ID or a list of ID
AND_search : a list of keywords to look for in trend names
OR_search : a list of keywords to look for in trend names
rng : haystack range (today,yesterday, last24hours...
start : string representation of start time ex. '2014-01-01T00:00'
end : string representation of end time ex. '2014-01-01T00:00'
"""
self._filteredList = [] # Empty list to be returned
# Keyword Arguments
# print(kwargs)
ids = kwargs.pop('id', '').replace('r:', '')
AND_search = kwargs.pop('AND_search', '')
OR_search = kwargs.pop('OR_search', '')
rng = kwargs.pop('rng', '')
start = kwargs.pop('start', '')
end = kwargs.pop('end', '')
takeall = kwargs.pop('all', '')
# Remaining kwargs...
if kwargs: raise TypeError('Unknown argument(s) : %s' % kwargs)
# Build datetimeRange based on start and end
if start and end:
datetimeRange = start + ',' + end
else:
datetimeRange = rng
# Find histories matching ALL keywords in AND_search
for eachHistory in self.hisAll():
takeit = False
# Find histories matching ANY keywords in OR_search
if (AND_search != '') and all([keywords in eachHistory['name'] for keywords in AND_search]):
print('AND_search : Adding %s to recordList' % eachHistory['name'])
takeit = True
# Find histories matching ANY ID in id list
elif (OR_search != '') and any([keywords in eachHistory['name'] for keywords in OR_search]):
print('OR_search : Adding %s to recordList' % eachHistory['name'])
takeit = True
elif (ids != '') and any([id in eachHistory['id'] for id in ids]):
print('ID found : Adding %s to recordList' % eachHistory['name'])
takeit = True
elif takeall != '':
print('Adding %s to recordList' % eachHistory['name'])
takeit = True
if takeit:
self._filteredList.append(HisRecord(self, eachHistory['id'], datetimeRange))
if self._filteredList == []:
print('No trends found... sorry !')
return self._filteredList
def commit(self, diff, premadeDiff=False, action=""):
"""
Implements skyspark commit. It accepts a Json and returns the server's response.
:param diff:
a dict of the following type:
{
"meta":{"ver":"haystack version","commit":"add/update/remove"},
"cols":[{"name":"col1 name"},{"name":"col2 name"}]
"rows":[{"col1 name":"col1 val","col2 name":"col2 val"}]
}
The json needs to have an id. And in case of update or remove, a mod value which is the time of last change.
:return: returns the servers response as a Json.
:type diff: dict or list
Don't ask.
"""
if premadeDiff:
return self.s.post(self.queryURL + "commit", data=json.dumps(diff),
headers={'accept': 'application/json; charset=utf-8',
'content-type': 'application/json'})
else:
sDiff = {}
sDiff['meta'] = {"ver": self.haystackVersion, "commit": action}
sDiff['cols'] = []
sDiff['rows'] = []
if type(diff) == dict: # for one line commit
for key in diff.keys():
sDiff['cols'].append({"name":key})
sDiff['rows'].append(diff)
elif type(diff) == list: # for several commits
keys = []
for line in diff:
for key in line.keys():
keys.append(key)
sDiff['rows'].append(line)
for key in set(keys):
sDiff['cols'].append({"name":key})
#print sDiff
return self.s.post(self.queryURL+"commit",data=json.dumps(sDiff),
headers={'accept': 'application/json; charset=utf-8',
'content-type': 'application/json'}).text
def makeZinc(self, data):
"""
Not sure if needed. Will convert json to zinc
:param data:
:type data:
:return:
"""
pass
def hisWrite(self, df):
df.index.names = ['ts']
cols =[]
for i,name in enumerate(df.columns):
cols.append('v%d id:@%s' % (i, df.columns[i]))
df.columns=cols
data = 'ver:"2.0" commit:"his"\n'
data += df.to_csv(date_format="%Y-%m-%dT%T+02:00 Jerusalem")
return self.s.post('http://shufersal.smartgreen.co.il/api/shufersal/commit',data=data).text