def delete(self, uri, cookie, params=None, data=None): try: headers = { 'Accept': 'application/json', 'X-CouchDB-WWW-Authenticate': 'Cookie', 'Content-Type': 'application/x-www-form-urlencoded' } url = self.protocol + '://' + self.host + ':' + self.port + uri r = requests.delete(url, headers=headers, cookies=cookie, params=params, data=data, verify=self.verify) if r.status_code < 400: return r.json() else: raise Errors.svtError(r, url) except requests.exceptions.RequestException as e: raise Errors.reqErrors(url, e)
def get(self, uri, cookie): try: headers = { 'Accept': 'application/json', 'X-CouchDB-WWW-Authenticate': 'Cookie', 'Content-Type': 'application/x-www-form-urlencoded' } url = self.protocol + '://' + self.host + ':' + self.port + uri logger.debug('GET : %s', url) # print(url) r = requests.get(url, headers=headers, verify=self.verify, cookies=cookie) if r.status_code == 200: return r.json() else: raise Errors.svtError(r, url) except requests.exceptions.RequestException as e: raise Errors.reqErrors(url, e)
def isAlive(self): try: j = self.couch.get('/', self.cookie) return(j['couchdb']) except: code = 99 msg = "Can't GET CouchDB." msg += " Check svt_couchdb config file or Authorization" call = "" debug = "" raise Errors.genError(code, msg, call, debug)
def post(self, uri, cookie=None, params=None, data=None): try: if cookie == "getCookie": url = self.protocol + '://' + self.username + ':' url += self.password + '@' + self.host + ':' + self.port + uri headers = {'Content-type': 'application/x-www-form-urlencoded'} r = requests.post(url, headers=headers, params=params, data=data, verify=self.verify) else: url = self.protocol + '://' + self.host + ':' + self.port + uri headers = { 'X-CouchDB-WWW-Authenticate': 'Cookie', 'Content-Type': 'application/json' } r = requests.post(url, cookies=cookie, headers=headers, params=params, data=data, verify=self.verify) if r.status_code < 400 and cookie == "getCookie": jar = r.cookies if len(jar) != 1: code = 99 msg = "Anormal number of cookies returned" call = "POST: " + url debug = (r, jar) raise Errors.genError(code, msg, call, debug) else: for c in jar: cookie = {c.name: c.value} return (cookie) elif r.status_code < 400: return r.json() else: raise Errors.svtError(r, uri) except requests.exceptions.RequestException as e: raise Errors.reqErrors(url, e)
def lister(self, settings): # Init connections parameters cfg = Readconfig().readconfig(settings) # Init couch connection couch = Couch(cfg) # Test Welcome j = couch.isAlive() print(j) # Check if SVT DB exists if not couch.hasDB(): code = 99 msg = "Missing or wrong SVT database: " + cfg.db call = cfg.url debug = cfg raise Errors.genError(code, msg, call, debug) # Call reduce r = couch.getReduce('admin', 'isnewcollect', startkey=None, endkey=None, group='2') j = json.loads(r) # Do some formating res = {} for row in j['rows']: k = row['key'] (date, client) = k if client in res: res[client].append(date) else: res[client] = [date] return res
def loader(self, settings, f, client): # Init connections parameters cfg = Readconfig().readconfig(settings) # Init couch connection couch = Couch(cfg) # Test Welcome j = couch.isAlive() print('Is Alive ? ' + j) # Check if SVT DB exists if not couch.hasDB(): code = 99 msg = "Missing or wrong SVT database: " + cfg.db call = cfg.url debug = cfg raise Errors.genError(code, msg, call, debug) # Parsed arguments print('Action: Load') print('File: ' + f.filename) print('Client: ' + client) # Extract collect from zip filename fullname = os.path.basename(f.filename) name = os.path.splitext(fullname)[0] collect = name.replace('output_svt_', '') # Check if collect already in Couch using reduce grouping at 2 startkey = [collect, client] endkey = [collect, client, {}] print("Checking for collect pre existence") r = couch.getReduce('admin', 'isnewcollect', startkey=startkey, endkey=endkey, group='2') new = json.loads(r) if new['rows']: code = 99 msg = "Client: " + client msg += " already has a collect for date: " + collect call = "admin/isnewcollect" debug = [startkey, endkey, r, new] raise Errors.genError(code, msg, call, debug) # Decorate and create bulk json bulk = {} bulk['docs'] = [] warnings = [] d = Utils().decorator(f, client) for k, s in d.items(): try: j = json.loads(s) bulk['docs'].append(j) except ValueError: warnings.append("Invalid JSON for CouchDB in file: " + k) raise # List the invalid files print("\n".join(warnings)) # Bulk load try: data = json.dumps(bulk) couch.postBulk(data) return ({'svt_upload': 'OK', 'client': client, 'date': collect}) except: print("Error with bulk loader") raise
def deleter(self, settings, collect, client): # Init connections parameters cfg = Readconfig().readconfig(settings) # Init couch connection couch = Couch(cfg) # Test Welcome j = couch.isAlive() print('Is Alive ? ' + j) # Check if SVT DB exists if not couch.hasDB(): code = 99 msg = "Missing or wrong SVT database: " + cfg.db call = cfg.url debug = cfg raise Errors.genError(code, msg, call, debug) # Parsed arguments print('Action: Delete') print('date: ' + collect) print('Client: ' + client) # Get those docs startkey = [collect, client] endkey = [collect, client, {}] print("Finding collect docs") r = couch.getViewIncludeDocs('admin', 'isnewcollect', startkey=startkey, endkey=endkey) docs = json.loads(r) # Check for existence if not docs['rows']: code = 99 msg = "Collect does not exist" call = [client, collect] debug = r raise Errors.genError(code, msg, call, debug) # Build the bulk delete JSON deleted = [] for doc in docs['rows']: cid = doc["doc"]["_id"] crev = doc["doc"]["_rev"] ccollect = doc["doc"]["svt_collect_date"] cclient = doc["doc"]["svt_client"] # Sanity check the key filter if ccollect != collect or cclient != client: code = 99 msg = "View result gives different client" call = [client, collect, cclient, ccollect] debug = r raise Errors.genError(code, msg, call, debug) deleted.append({"_id": cid, "_rev": crev, "deleted": "true"}) d = {"docs": deleted} # Bulk delete try: data = json.dumps(d) couch.postBulk(data) return {"svt_delete": "OK"} except: print("Error with bulk delete") raise
def generator(self, settings, collect, client, yamldef): # Init connections parameters cfg = Readconfig().readconfig(settings) # Init couch connection couch = Couch(cfg) # Test Welcome j = couch.isAlive() print(j) # Check if SVT DB exists if not couch.hasDB(): code = 99 msg = "Missing or wrong SVT database: " + cfg.db call = 'hasDB' debug = cfg raise Errors.genError(code, msg, call, debug) # Parsed arguments print('Action: Generate') print('Collect: ' + collect) print('Client: ' + client) print('YAML: ' + yamldef) # Check if collect already in Couch using reduce grouping at 2 startkey = [collect, client] endkey = [collect, client, {}] print("Checking for collect existence") r = couch.getReduce('admin', 'isnewcollect', startkey=startkey, endkey=endkey, group='2') new = json.loads(r) if not new['rows']: code = 99 msg = "Client: " + client msg += " does not have a collect for date: " + collect call = "admin/isnewcollect" debug = [startkey, endkey, r, new] raise Errors.genError(code, msg, call, debug) # Parse YAML h = open(yamldef) y = yaml.load(h) reports = Utils().flatten(y) # Structure for result json res = Utils().hash() # Call couch for reports for (view, selector), markers in reports.items(): # Key passed to the view: will select only given collect & client startkey = [collect, client] endkey = [collect, client, {}] c = couch.getView(view, selector, startkey, endkey) r = Utils().cleanKeys(c) # Loop over the markers for marker in markers: # Save the Caller caller = [client, collect, view, selector, marker] # Deal with the catch all special marker if marker == 'svt_all': hook = Svt(selector=selector).hook_all j = json.loads(r, object_hook=hook) else: hook = Svt(selector=selector, marker=marker).hook_marker j = json.loads(r, object_hook=hook) # Special ViPR call if 'vipr' in view: pass # Update hash result res = Utils().jsonify(res, caller, j) # Dump the hash into a json string if not res: res = {'svt_no_data': 'svt_no_data'} return (res)
def jsonify(self, res, caller, rows): ddoc = caller[2] selector = caller[3] marker = caller[4] for row in rows: # Get values from the row object collect = row.collect client = row.client source = row.source name = row.name value = row.value action = row.action # Keep collect and date in the result and sanity check them if 'info' not in res: res['info']['svt_collect'] = collect res['info']['svt_client'] = client elif (res['info']['svt_collect'] != collect and res['info']['svt_client'] != client): code = 99 msg = "Different keys found in Generator" call = (res, caller, row) debug = "" raise Errors.genError(code, msg, call, debug) ################################################################### # multi ################################################################### if (action == 'svt_multi' and marker == 'svt_all' and isinstance(value, dict)): # Getting svt_all instead of specific marker # Need to clean output from using multifile based name # Multiple file values are keyed by name # print("DEBUG: GENTYPE : multi, all, dict" ) try: del value['svt_collect_date'] del value['svt_client'] del value['svt_source'] del value['svt_source_file'] del value['_id'] del value['_rev'] except KeyError: pass for k in value.keys(): if k != 'svt_marked': res['data'][source][ddoc][selector][name][k] = value[k] elif (action == 'svt_multi' and marker == 'svt_all'): # multi and all for non dict value # Multiple file values are keyed by name # print("DEBUG: GENTYPE: multi, all, non dict" ) res['data'][source][ddoc][selector][name] = value elif (action == 'svt_multi'): # multi and marker for non dict value # Multiple file values are keyed by name # Need to filter svt_no_data from other files # Sanity check for duplicate markers across files # print("DEBUG: GENTYPE: multi, marker, non dict" ) leaf = res['data'][source][ddoc][selector][name] if marker not in leaf: res['data'][source][ddoc][selector][name][marker] = value elif value != 'svt_no_data' and leaf[marker] == 'svt_no_data': res['data'][source][ddoc][selector][name][marker] = value elif (leaf[marker] != 'svt_no_data' and value != 'svt_no_data' and value != leaf[marker]): msg = "ERROR: duplicate value for marker: " + marker raise Exception(msg) ################################################################### # group ################################################################### elif (action == 'svt_group' and marker == 'svt_all' and isinstance(value, dict) and 'svt_marked' in value): # Marked values are to be re-keyed # Remove the marked tag del value['svt_marked'] for k in value.keys(): res['data'][source][ddoc][selector][name][k] = value[k] elif (action == 'svt_group' and marker == 'svt_all' and isinstance(value, dict)): # Need to clean output from using multifile based values # using name instead of selector try: del value[name]['svt_collect_date'] del value[name]['svt_client'] del value[name]['svt_source'] del value[name]['svt_source_file'] del value[name]['_id'] del value[name]['_rev'] except KeyError: pass res['data'][source][ddoc][selector][name] = value[name] elif (action == 'svt_group' and marker == 'svt_all'): res['data'][source][ddoc][selector][name] = value elif (action == 'svt_group' and isinstance(value, dict) and 'svt_marked' in value): # Marked values are to be re-keyed # Remove the marked tag del value['svt_marked'] for k in value: res['data'][source][ddoc][selector][name][k] = value[k] elif (action == 'svt_group' and marker == name): try: del value['svt_collect_date'] del value['svt_client'] del value['svt_source'] del value['svt_source_file'] del value['_id'] del value['_rev'] except KeyError: pass res['data'][source][ddoc][selector][name] = value ################################################################### # single ################################################################### elif (action == 'svt_single' and marker == 'svt_all' and isinstance(value, dict) and 'svt_marked' in value): # Getting single file values # Getting svt_all instead of specific marker # Marked values are to be re-keyed # Remove the marked tag print("DEBUG: GENTYPE: single, all, marked, dict") leaf = res['data'][source][ddoc][name] del value['svt_marked'] for k in value.keys(): res['data'][source][ddoc][selector][k] = value[k] elif (action == 'svt_single' and marker == 'svt_all'): print("DEBUG: GENTYPE: single, all") # Getting svt_all for single file values # Attach them all at the selector level # Need to clean output from using the whole json try: del value['svt_collect_date'] del value['svt_client'] del value['svt_source'] del value['svt_source_file'] del value['_id'] del value['_rev'] except KeyError: pass res['data'][source][ddoc][selector].update(value) elif (action == 'svt_single' and isinstance(value, dict) and 'svt_marked' in value): print("DEBUG: GENTYPE: single, marked, dict") # Getting single files values for marker # Marked values are to be re-keyed # Remove the marked tag del value['svt_marked'] for k in value.keys(): res['data'][source][ddoc][selector][k][marker] = value[k] elif action == 'svt_single': print("DEBUG: GENTYPE: single, marker") # Getting single file value for a specific marker # Using marker won't select doc level info res['data'][source][ddoc][selector][marker] = value ################################################################### # standard ################################################################### elif (action == 'svt_standard' and marker == 'svt_all' and isinstance(value, dict) and 'svt_marked' in value): # Getting svt_all instead of specific marker # Marked values are to be re-keyed # Remove the marked tag del value['svt_marked'] for k in value.keys(): res['data'][source][ddoc][name][selector][k] = value[k] elif (action == 'svt_standard' and marker == 'svt_all'): # Getting svt_all for ungrouped value # Attach them all under their name res['data'][source][ddoc][name][selector].update(value) elif (action == 'svt_standard' and isinstance(value, dict) and 'svt_marked' in value): # Getting specific marker # Marked values are to be re-keyed # Remove the marked tag del value['svt_marked'] for k in value.keys(): res['data'][source][ddoc][name][selector][k][marker]\ = value[k] elif action == 'svt_standard': # Standard attach res['data'][source][ddoc][name][selector][marker] = value # else: # msg = "Error: Got unkown action: " + action + " from map: " # msg += ddoc + '/' + selector # raise Exception(msg) return (res)