def delete(self, uri, cookie, params=None, data=None):
try:
headers = {
'Accept': 'application/json',
'X-CouchDB-WWW-Authenticate': 'Cookie',
'Content-Type': 'application/x-www-form-urlencoded'
}
url = self.protocol + '://' + self.host + ':' + self.port + uri
r = requests.delete(url,
headers=headers,
cookies=cookie,
params=params,
data=data,
verify=self.verify)
if r.status_code < 400:
return r.json()
else:
raise Errors.svtError(r, url)
except requests.exceptions.RequestException as e:
raise Errors.reqErrors(url, e)
def get(self, uri, cookie):
try:
headers = {
'Accept': 'application/json',
'X-CouchDB-WWW-Authenticate': 'Cookie',
'Content-Type': 'application/x-www-form-urlencoded'
}
url = self.protocol + '://' + self.host + ':' + self.port + uri
logger.debug('GET : %s', url)
# print(url)
r = requests.get(url,
headers=headers,
verify=self.verify,
cookies=cookie)
if r.status_code == 200:
return r.json()
else:
raise Errors.svtError(r, url)
except requests.exceptions.RequestException as e:
raise Errors.reqErrors(url, e)
def isAlive(self):
try:
j = self.couch.get('/', self.cookie)
return(j['couchdb'])
except:
code = 99
msg = "Can't GET CouchDB."
msg += " Check svt_couchdb config file or Authorization"
call = ""
debug = ""
raise Errors.genError(code, msg, call, debug)
def post(self, uri, cookie=None, params=None, data=None):
try:
if cookie == "getCookie":
url = self.protocol + '://' + self.username + ':'
url += self.password + '@' + self.host + ':' + self.port + uri
headers = {'Content-type': 'application/x-www-form-urlencoded'}
r = requests.post(url,
headers=headers,
params=params,
data=data,
verify=self.verify)
else:
url = self.protocol + '://' + self.host + ':' + self.port + uri
headers = {
'X-CouchDB-WWW-Authenticate': 'Cookie',
'Content-Type': 'application/json'
}
r = requests.post(url,
cookies=cookie,
headers=headers,
params=params,
data=data,
verify=self.verify)
if r.status_code < 400 and cookie == "getCookie":
jar = r.cookies
if len(jar) != 1:
code = 99
msg = "Anormal number of cookies returned"
call = "POST: " + url
debug = (r, jar)
raise Errors.genError(code, msg, call, debug)
else:
for c in jar:
cookie = {c.name: c.value}
return (cookie)
elif r.status_code < 400:
return r.json()
else:
raise Errors.svtError(r, uri)
except requests.exceptions.RequestException as e:
raise Errors.reqErrors(url, e)
def lister(self, settings):
# Init connections parameters
cfg = Readconfig().readconfig(settings)
# Init couch connection
couch = Couch(cfg)
# Test Welcome
j = couch.isAlive()
print(j)
# Check if SVT DB exists
if not couch.hasDB():
code = 99
msg = "Missing or wrong SVT database: " + cfg.db
call = cfg.url
debug = cfg
raise Errors.genError(code, msg, call, debug)
# Call reduce
r = couch.getReduce('admin',
'isnewcollect',
startkey=None,
endkey=None,
group='2')
j = json.loads(r)
# Do some formating
res = {}
for row in j['rows']:
k = row['key']
(date, client) = k
if client in res:
res[client].append(date)
else:
res[client] = [date]
return res
def loader(self, settings, f, client):
# Init connections parameters
cfg = Readconfig().readconfig(settings)
# Init couch connection
couch = Couch(cfg)
# Test Welcome
j = couch.isAlive()
print('Is Alive ? ' + j)
# Check if SVT DB exists
if not couch.hasDB():
code = 99
msg = "Missing or wrong SVT database: " + cfg.db
call = cfg.url
debug = cfg
raise Errors.genError(code, msg, call, debug)
# Parsed arguments
print('Action: Load')
print('File: ' + f.filename)
print('Client: ' + client)
# Extract collect from zip filename
fullname = os.path.basename(f.filename)
name = os.path.splitext(fullname)[0]
collect = name.replace('output_svt_', '')
# Check if collect already in Couch using reduce grouping at 2
startkey = [collect, client]
endkey = [collect, client, {}]
print("Checking for collect pre existence")
r = couch.getReduce('admin',
'isnewcollect',
startkey=startkey,
endkey=endkey,
group='2')
new = json.loads(r)
if new['rows']:
code = 99
msg = "Client: " + client
msg += " already has a collect for date: " + collect
call = "admin/isnewcollect"
debug = [startkey, endkey, r, new]
raise Errors.genError(code, msg, call, debug)
# Decorate and create bulk json
bulk = {}
bulk['docs'] = []
warnings = []
d = Utils().decorator(f, client)
for k, s in d.items():
try:
j = json.loads(s)
bulk['docs'].append(j)
except ValueError:
warnings.append("Invalid JSON for CouchDB in file: " + k)
raise
# List the invalid files
print("\n".join(warnings))
# Bulk load
try:
data = json.dumps(bulk)
couch.postBulk(data)
return ({'svt_upload': 'OK', 'client': client, 'date': collect})
except:
print("Error with bulk loader")
raise
def deleter(self, settings, collect, client):
# Init connections parameters
cfg = Readconfig().readconfig(settings)
# Init couch connection
couch = Couch(cfg)
# Test Welcome
j = couch.isAlive()
print('Is Alive ? ' + j)
# Check if SVT DB exists
if not couch.hasDB():
code = 99
msg = "Missing or wrong SVT database: " + cfg.db
call = cfg.url
debug = cfg
raise Errors.genError(code, msg, call, debug)
# Parsed arguments
print('Action: Delete')
print('date: ' + collect)
print('Client: ' + client)
# Get those docs
startkey = [collect, client]
endkey = [collect, client, {}]
print("Finding collect docs")
r = couch.getViewIncludeDocs('admin',
'isnewcollect',
startkey=startkey,
endkey=endkey)
docs = json.loads(r)
# Check for existence
if not docs['rows']:
code = 99
msg = "Collect does not exist"
call = [client, collect]
debug = r
raise Errors.genError(code, msg, call, debug)
# Build the bulk delete JSON
deleted = []
for doc in docs['rows']:
cid = doc["doc"]["_id"]
crev = doc["doc"]["_rev"]
ccollect = doc["doc"]["svt_collect_date"]
cclient = doc["doc"]["svt_client"]
# Sanity check the key filter
if ccollect != collect or cclient != client:
code = 99
msg = "View result gives different client"
call = [client, collect, cclient, ccollect]
debug = r
raise Errors.genError(code, msg, call, debug)
deleted.append({"_id": cid, "_rev": crev, "deleted": "true"})
d = {"docs": deleted}
# Bulk delete
try:
data = json.dumps(d)
couch.postBulk(data)
return {"svt_delete": "OK"}
except:
print("Error with bulk delete")
raise
def generator(self, settings, collect, client, yamldef):
# Init connections parameters
cfg = Readconfig().readconfig(settings)
# Init couch connection
couch = Couch(cfg)
# Test Welcome
j = couch.isAlive()
print(j)
# Check if SVT DB exists
if not couch.hasDB():
code = 99
msg = "Missing or wrong SVT database: " + cfg.db
call = 'hasDB'
debug = cfg
raise Errors.genError(code, msg, call, debug)
# Parsed arguments
print('Action: Generate')
print('Collect: ' + collect)
print('Client: ' + client)
print('YAML: ' + yamldef)
# Check if collect already in Couch using reduce grouping at 2
startkey = [collect, client]
endkey = [collect, client, {}]
print("Checking for collect existence")
r = couch.getReduce('admin',
'isnewcollect',
startkey=startkey,
endkey=endkey,
group='2')
new = json.loads(r)
if not new['rows']:
code = 99
msg = "Client: " + client
msg += " does not have a collect for date: " + collect
call = "admin/isnewcollect"
debug = [startkey, endkey, r, new]
raise Errors.genError(code, msg, call, debug)
# Parse YAML
h = open(yamldef)
y = yaml.load(h)
reports = Utils().flatten(y)
# Structure for result json
res = Utils().hash()
# Call couch for reports
for (view, selector), markers in reports.items():
# Key passed to the view: will select only given collect & client
startkey = [collect, client]
endkey = [collect, client, {}]
c = couch.getView(view, selector, startkey, endkey)
r = Utils().cleanKeys(c)
# Loop over the markers
for marker in markers:
# Save the Caller
caller = [client, collect, view, selector, marker]
# Deal with the catch all special marker
if marker == 'svt_all':
hook = Svt(selector=selector).hook_all
j = json.loads(r, object_hook=hook)
else:
hook = Svt(selector=selector, marker=marker).hook_marker
j = json.loads(r, object_hook=hook)
# Special ViPR call
if 'vipr' in view:
pass
# Update hash result
res = Utils().jsonify(res, caller, j)
# Dump the hash into a json string
if not res:
res = {'svt_no_data': 'svt_no_data'}
return (res)
def jsonify(self, res, caller, rows):
ddoc = caller[2]
selector = caller[3]
marker = caller[4]
for row in rows:
# Get values from the row object
collect = row.collect
client = row.client
source = row.source
name = row.name
value = row.value
action = row.action
# Keep collect and date in the result and sanity check them
if 'info' not in res:
res['info']['svt_collect'] = collect
res['info']['svt_client'] = client
elif (res['info']['svt_collect'] != collect
and res['info']['svt_client'] != client):
code = 99
msg = "Different keys found in Generator"
call = (res, caller, row)
debug = ""
raise Errors.genError(code, msg, call, debug)
###################################################################
# multi
###################################################################
if (action == 'svt_multi' and marker == 'svt_all'
and isinstance(value, dict)):
# Getting svt_all instead of specific marker
# Need to clean output from using multifile based name
# Multiple file values are keyed by name
# print("DEBUG: GENTYPE : multi, all, dict" )
try:
del value['svt_collect_date']
del value['svt_client']
del value['svt_source']
del value['svt_source_file']
del value['_id']
del value['_rev']
except KeyError:
pass
for k in value.keys():
if k != 'svt_marked':
res['data'][source][ddoc][selector][name][k] = value[k]
elif (action == 'svt_multi' and marker == 'svt_all'):
# multi and all for non dict value
# Multiple file values are keyed by name
# print("DEBUG: GENTYPE: multi, all, non dict" )
res['data'][source][ddoc][selector][name] = value
elif (action == 'svt_multi'):
# multi and marker for non dict value
# Multiple file values are keyed by name
# Need to filter svt_no_data from other files
# Sanity check for duplicate markers across files
# print("DEBUG: GENTYPE: multi, marker, non dict" )
leaf = res['data'][source][ddoc][selector][name]
if marker not in leaf:
res['data'][source][ddoc][selector][name][marker] = value
elif value != 'svt_no_data' and leaf[marker] == 'svt_no_data':
res['data'][source][ddoc][selector][name][marker] = value
elif (leaf[marker] != 'svt_no_data' and value != 'svt_no_data'
and value != leaf[marker]):
msg = "ERROR: duplicate value for marker: " + marker
raise Exception(msg)
###################################################################
# group
###################################################################
elif (action == 'svt_group' and marker == 'svt_all'
and isinstance(value, dict) and 'svt_marked' in value):
# Marked values are to be re-keyed
# Remove the marked tag
del value['svt_marked']
for k in value.keys():
res['data'][source][ddoc][selector][name][k] = value[k]
elif (action == 'svt_group' and marker == 'svt_all'
and isinstance(value, dict)):
# Need to clean output from using multifile based values
# using name instead of selector
try:
del value[name]['svt_collect_date']
del value[name]['svt_client']
del value[name]['svt_source']
del value[name]['svt_source_file']
del value[name]['_id']
del value[name]['_rev']
except KeyError:
pass
res['data'][source][ddoc][selector][name] = value[name]
elif (action == 'svt_group' and marker == 'svt_all'):
res['data'][source][ddoc][selector][name] = value
elif (action == 'svt_group' and isinstance(value, dict)
and 'svt_marked' in value):
# Marked values are to be re-keyed
# Remove the marked tag
del value['svt_marked']
for k in value:
res['data'][source][ddoc][selector][name][k] = value[k]
elif (action == 'svt_group' and marker == name):
try:
del value['svt_collect_date']
del value['svt_client']
del value['svt_source']
del value['svt_source_file']
del value['_id']
del value['_rev']
except KeyError:
pass
res['data'][source][ddoc][selector][name] = value
###################################################################
# single
###################################################################
elif (action == 'svt_single' and marker == 'svt_all'
and isinstance(value, dict) and 'svt_marked' in value):
# Getting single file values
# Getting svt_all instead of specific marker
# Marked values are to be re-keyed
# Remove the marked tag
print("DEBUG: GENTYPE: single, all, marked, dict")
leaf = res['data'][source][ddoc][name]
del value['svt_marked']
for k in value.keys():
res['data'][source][ddoc][selector][k] = value[k]
elif (action == 'svt_single' and marker == 'svt_all'):
print("DEBUG: GENTYPE: single, all")
# Getting svt_all for single file values
# Attach them all at the selector level
# Need to clean output from using the whole json
try:
del value['svt_collect_date']
del value['svt_client']
del value['svt_source']
del value['svt_source_file']
del value['_id']
del value['_rev']
except KeyError:
pass
res['data'][source][ddoc][selector].update(value)
elif (action == 'svt_single' and isinstance(value, dict)
and 'svt_marked' in value):
print("DEBUG: GENTYPE: single, marked, dict")
# Getting single files values for marker
# Marked values are to be re-keyed
# Remove the marked tag
del value['svt_marked']
for k in value.keys():
res['data'][source][ddoc][selector][k][marker] = value[k]
elif action == 'svt_single':
print("DEBUG: GENTYPE: single, marker")
# Getting single file value for a specific marker
# Using marker won't select doc level info
res['data'][source][ddoc][selector][marker] = value
###################################################################
# standard
###################################################################
elif (action == 'svt_standard' and marker == 'svt_all'
and isinstance(value, dict) and 'svt_marked' in value):
# Getting svt_all instead of specific marker
# Marked values are to be re-keyed
# Remove the marked tag
del value['svt_marked']
for k in value.keys():
res['data'][source][ddoc][name][selector][k] = value[k]
elif (action == 'svt_standard' and marker == 'svt_all'):
# Getting svt_all for ungrouped value
# Attach them all under their name
res['data'][source][ddoc][name][selector].update(value)
elif (action == 'svt_standard' and isinstance(value, dict)
and 'svt_marked' in value):
# Getting specific marker
# Marked values are to be re-keyed
# Remove the marked tag
del value['svt_marked']
for k in value.keys():
res['data'][source][ddoc][name][selector][k][marker]\
= value[k]
elif action == 'svt_standard':
# Standard attach
res['data'][source][ddoc][name][selector][marker] = value
# else:
# msg = "Error: Got unkown action: " + action + " from map: "
# msg += ddoc + '/' + selector
# raise Exception(msg)
return (res)