def eus_user_add(username, results): if username == None: raise Exception("User is None") isok = False try: pw = str(random.getrandbits(8 * 32)) m = hashlib.md5() m.update(pw) pw = m.hexdigest() pdata = "%s" % (pw) writebody = "" writebody = call_curl(url, method="PUT", idata=pdata, capath='/etc/pki/tls/certs/ca-bundle.crt') logger.info("%s", writebody) dom = xml.dom.minidom.parseString(writebody) found = False for x in dom.firstChild.childNodes: if x.nodeType == x.ELEMENT_NODE and (x.nodeName == 'status'): isok = (str(x.getAttribute('action')) == 'added') if x.nodeType == x.ELEMENT_NODE and (x.nodeName == 'redirect'): results['proto'] = str(x.getAttribute('proto')) results['hostname'] = str(x.getAttribute('hostname')) results['path'] = str(x.getAttribute('path')) if isok: return pw except xml.parsers.expat.ExpatError, e: logger.error("%s", returned_data) logger.error("%s", e) raise
def create_user_filter(index, user_id, server=None, prefix="myemsl_user", config=config): alias = "%s_%i" % (prefix, user_id) filter = """ { "actions": [{ "add": { "index": "%s", "alias": "%s", "filter": { "or": [{ "term": {"users": %i} }, { "term": {"aged": true} }] } } }] }""" % (index, alias, user_id) if server == None: server = config.get('elasticsearch', 'server') writebody = "" url = server if url[-1:] != '/': url += '/' url += "_aliases" try: writebody = call_curl(url, method="POST", idata=filter) except CurlException, ex: return ex.http_code
def proposals_process(chunk_items, collection): query = { "fields": ["proposals"], "query": { "constant_score": { "filter": { "ids": { "values": chunk_items } } } } } server = config.get('elasticsearch', 'server') results = json.loads( call_curl("%s/myemsl_current_simple_items/simple_items/_search" % (server), method='POST', idata=json.dumps(query))) if results['hits']['total'] != len(chunk_items): raise Exception('Failed to find some items') for hit in results['hits']['hits']: if 'fields' in hit and 'proposals' in hit['fields']: collection.update( {'_id': int(hit['_id'])}, {"$set": { 'value.p': hit['fields']['proposals'] }}, w=1)
def eus_user_add(username, results): if username == None: raise Exception("User is None") isok = False try: pw = str(random.getrandbits(8 * 32)) m = hashlib.md5() m.update(pw) pw = m.hexdigest() pdata = "%s" % (pw) writebody = "" writebody = call_curl(url, method="PUT", idata=pdata, capath="/etc/pki/tls/certs/ca-bundle.crt") logger.info("%s", writebody) dom = xml.dom.minidom.parseString(writebody) found = False for x in dom.firstChild.childNodes: if x.nodeType == x.ELEMENT_NODE and (x.nodeName == "status"): isok = str(x.getAttribute("action")) == "added" if x.nodeType == x.ELEMENT_NODE and (x.nodeName == "redirect"): results["proto"] = str(x.getAttribute("proto")) results["hostname"] = str(x.getAttribute("hostname")) results["path"] = str(x.getAttribute("path")) if isok: return pw except xml.parsers.expat.ExpatError, e: logger.error("%s", returned_data) logger.error("%s", e) raise
def create_user_filter(index, user_id, server=None, prefix="myemsl_user", config=config): alias = "%s_%i" %(prefix, user_id) filter = """ { "actions": [{ "add": { "index": "%s", "alias": "%s", "filter": { "or": [{ "term": {"users": %i} }, { "term": {"aged": true} }] } } }] }""" %(index, alias, user_id) if server == None: server = config.get('elasticsearch', 'server') writebody = "" url = server if url[-1:] != '/': url += '/' url += "_aliases" try: writebody = call_curl(url, method="POST", idata=filter) except CurlException, ex: return ex.http_code
def delete_index(index_name, server=None, config=config): if server == None: server = config.get('elasticsearch', 'server') writebody = "" url = server if url[-1:] != '/': url += '/' url += index_name try: writebody = call_curl(url, method="DELETE") except CurlException, ex: return ex.http_code
def bulk_action(index, actions, server=None, config=config): alias_cmd = '\n'.join([json.dumps(i) for i in actions]) alias_cmd += '\n' if server == None: server = config.get('elasticsearch', 'server') writebody = "" url = server if url[-1:] != '/': url += '/' url += index + "/_bulk" try: writebody = call_curl(url, method="POST", idata=alias_cmd) except CurlException, ex: return ex.http_code
def create_index(index_name, server=None, config=config, desc=None): if desc == None: desc = myemsl.elasticsearch.schema.schema_get('simple_item') if server == None: server = config.get('elasticsearch', 'server') writebody = "" url = server if url[-1:] != '/': url += '/' url += index_name try: writebody = call_curl(url, method="PUT", idata=desc) except CurlException, ex: return ex.http_code
def get_alias(index='simple_items', alias=None, server=None, config=config): if alias == None: alias = "%s_%s" %(config.get('elasticsearch', "alias"), index) if server == None: server = config.get('elasticsearch', 'server') url = server if url[-1:] != '/': url += '/' #FIXME This grabs all aliases. should find an api that maps only the single one we need. url += alias + '/_aliases' writebody = "" try: writebody = call_curl(url) except CurlException, ex: return None, ex.http_code
def get_user_remote(username, map='web_map'): """This handles authorization""" newuser = None try: url = "%s%s/%s" %(config.get('getuser', 'prefix'), config.get('getuser', map), urllib.quote(username)) writebody = call_curl(url, capath='/etc/pki/tls/certs/ca-bundle.crt') logger.info(writebody) dom = xml.dom.minidom.parseString(writebody) found = False for x in dom.firstChild.childNodes: if x.nodeType == x.ELEMENT_NODE and (x.nodeName == 'user'): newuser = str(x.getAttribute('id')) break except Exception, e: logger.error("%s", e) raise e
def get_user_remote(username, map='web_map'): """This handles authorization""" newuser = None try: url = "%s%s/%s" % (config.get('getuser', 'prefix'), config.get('getuser', map), urllib.quote(username)) writebody = call_curl(url, capath='/etc/pki/tls/certs/ca-bundle.crt') logger.info(writebody) dom = xml.dom.minidom.parseString(writebody) found = False for x in dom.firstChild.childNodes: if x.nodeType == x.ELEMENT_NODE and (x.nodeName == 'user'): newuser = str(x.getAttribute('id')) break except Exception, e: logger.error("%s", e) raise e
def eus_user_remove(username): if username == None: raise Exception("User is None") isok = False try: writebody = "" url = "%s/%s" % (config.get("metadata", "eus_auth"), urllib.quote(username)) writebody = call_curl(url, method="DELETE", capath="/etc/pki/tls/certs/ca-bundle.crt") logger.info("%s", writebody) dom = xml.dom.minidom.parseString(writebody) found = False for x in dom.firstChild.childNodes: if x.nodeType == x.ELEMENT_NODE and (x.nodeName == "status"): isok = str(x.getAttribute("action")) == "removed" break except Exception, e: logger.error("%s", e) raise
def item_get(item_id): user_tries = 2 while user_tries > 0: url = config.get('elasticsearch', 'server') if url[-1:] != '/': url += '/' #FIXME unhardcode these... url += "%s/%s/%i" %('myemsl_current_simple_items', 'simple_items', item_id) writebody = "" try: writebody = call_curl(url) except CurlException, ex: return ex.http_code, None j = json.loads(writebody) if not j.get('_source'): return 500, None return 200, j['_source']
def get_transaction(user): transxml = call_curl( "%s/%s" % (config.get('metadata', 'transaction_base_url'), user), capath=None, cainfo='/etc/myemsl/keys/server/local.crt', sslcert='/etc/myemsl/keys/server/local.pem', sslcerttype='PEM') dom = xml.dom.minidom.parseString(transxml) transaction = -1 found = False for x in dom.firstChild.childNodes: if x.nodeType == x.ELEMENT_NODE and (x.nodeName == 'transaction'): transaction = int(x.getAttribute('id')) found = True break if not found: raise Exception("Could not get transaction number from server") return transaction
def get_transaction(user): transxml = call_curl( "%s/%s" %(config.get('metadata', 'transaction_base_url'), user), capath=None, cainfo='/etc/myemsl/keys/server/local.crt', sslcert='/etc/myemsl/keys/server/local.pem', sslcerttype='PEM' ) dom = xml.dom.minidom.parseString(transxml) transaction = -1 found = False for x in dom.firstChild.childNodes: if x.nodeType == x.ELEMENT_NODE and (x.nodeName == 'transaction'): transaction = int(x.getAttribute('id')) found = True break if not found: raise Exception("Could not get transaction number from server") return transaction
def item_get(item_id): user_tries = 2 while user_tries > 0: url = config.get('elasticsearch', 'server') if url[-1:] != '/': url += '/' #FIXME unhardcode these... url += "%s/%s/%i" % ('myemsl_current_simple_items', 'simple_items', item_id) writebody = "" try: writebody = call_curl(url) except CurlException, ex: return ex.http_code, None j = json.loads(writebody) if not j.get('_source'): return 500, None return 200, j['_source']
def eus_user_remove(username): if username == None: raise Exception("User is None") isok = False try: writebody = "" url = "%s/%s" % (config.get('metadata', 'eus_auth'), urllib.quote(username)) writebody = call_curl(url, method="DELETE", capath='/etc/pki/tls/certs/ca-bundle.crt') logger.info("%s", writebody) dom = xml.dom.minidom.parseString(writebody) found = False for x in dom.firstChild.childNodes: if x.nodeType == x.ELEMENT_NODE and (x.nodeName == 'status'): isok = (str(x.getAttribute('action')) == 'removed') break except Exception, e: logger.error("%s", e) raise
def proposals_process(chunk_items, collection): query = { "fields": [ "proposals" ], "query": { "constant_score": { "filter": { "ids": { "values": chunk_items } } } } } server = config.get('elasticsearch', 'server') results = json.loads(call_curl("%s/myemsl_current_simple_items/simple_items/_search" %(server), method='POST', idata=json.dumps(query))) if results['hits']['total'] != len(chunk_items): raise Exception('Failed to find some items') for hit in results['hits']['hits']: if 'fields' in hit and 'proposals' in hit['fields']: collection.update({'_id': int(hit['_id'])}, {"$set": {'value.p': hit['fields']['proposals']}}, w=1)
def create_alias(index, alias, server=None, config=config): ##FIXME delete alias if it already exists alias_cmd = """ { "actions": [{ "add": { "index": "%s", "alias": "%s" } }] }""" %(index, alias) if server == None: server = config.get('elasticsearch', 'server') writebody = "" url = server if url[-1:] != '/': url += '/' url += "_aliases" try: writebody = call_curl(url, method="POST", idata=alias_cmd) except CurlException, ex: return ex.http_code
def elasticsearchquery(user, index, type, req, retries=1, auth_add=False, search_type=None, scan=None): req_data = req.read() if type != 'released_publications' and user == '': req.write("Forbidden") return 401 user_tries = 2 while user_tries > 0: if type != "simple_items" and type != "released_publications": req.write("Forbidden") return 401 if type == "released_publications": index = "myemsl_current_released_publications" auth_add = False server = config.get('elasticsearch', 'server') url = server if url[-1:] != '/': url += '/' if scan: url += '/_search/scroll?scroll=10m' else: url += "%s/%s/_search" % (index, type) if search_type: url += '?search_type=scan&scroll=10m&size=50' writebody = "" skipit = 1 try: writebody = call_curl(url, method="POST", idata=req_data) except CurlException, ex: skipit = 0 if skipit: if auth_add: auth_items = [] j = json.loads(writebody) for i in j['hits']['hits']: auth_items.append(i['_id']) if len(auth_items) > 0: token = myemsl.token.simple_items_token_gen( auth_items, person_id=int(user)) logger.debug("Requested auth. %s" % (auth_items)) j['myemsl_auth_token'] = token req.write(json.dumps(j)) return 200 else: req.write(writebody) return 200 retval = "" alias_tries = 3 while alias_tries > 0: retval, code = myemsl.elasticsearch.get_alias(index=type) if code == 404: time.sleep(1) alias_tries -= 1 elif code == 200: break else: return code if alias_tries < 1: return 500 if type != 'released_publications': user_id = int(user) code = myemsl.elasticsearch.create_user_filter(retval, user_id) if code < 200 or code > 299: return 404 user_tries -= 1
def elasticsearchquery(user, index, type, req, retries=1, auth_add=False, search_type=None, scan=None): req_data = req.read() if type != 'released_publications' and user == '': req.write("Forbidden") return 401 user_tries = 2 while user_tries > 0: if type != "simple_items" and type != "released_publications": req.write("Forbidden") return 401 if type == "released_publications": index = "myemsl_current_released_publications" auth_add = False server = config.get('elasticsearch', 'server') url = server if url[-1:] != '/': url += '/' if scan: url += '/_search/scroll?scroll=10m' else: url += "%s/%s/_search" %(index, type) if search_type: url += '?search_type=scan&scroll=10m&size=50' writebody = "" skipit = 1 try: writebody = call_curl(url, method="POST", idata=req_data) except CurlException, ex: skipit = 0 if skipit: if auth_add: auth_items = [] j = json.loads(writebody) for i in j['hits']['hits']: auth_items.append(i['_id']) if len(auth_items) > 0: token = myemsl.token.simple_items_token_gen(auth_items, person_id=int(user)) logger.debug("Requested auth. %s" %(auth_items)) j['myemsl_auth_token'] = token req.write(json.dumps(j)) return 200 else: req.write(writebody) return 200 retval = "" alias_tries = 3 while alias_tries > 0: retval, code = myemsl.elasticsearch.get_alias(index=type) if code == 404: time.sleep(1) alias_tries -= 1 elif code == 200: break else: return code if alias_tries < 1: return 500 if type != 'released_publications': user_id = int(user) code = myemsl.elasticsearch.create_user_filter(retval, user_id) if code < 200 or code > 299: return 404 user_tries -= 1