def eus_user_add(username, results):
if username == None:
raise Exception("User is None")
isok = False
try:
pw = str(random.getrandbits(8 * 32))
m = hashlib.md5()
m.update(pw)
pw = m.hexdigest()
pdata = "%s" % (pw)
writebody = ""
writebody = call_curl(url,
method="PUT",
idata=pdata,
capath='/etc/pki/tls/certs/ca-bundle.crt')
logger.info("%s", writebody)
dom = xml.dom.minidom.parseString(writebody)
found = False
for x in dom.firstChild.childNodes:
if x.nodeType == x.ELEMENT_NODE and (x.nodeName == 'status'):
isok = (str(x.getAttribute('action')) == 'added')
if x.nodeType == x.ELEMENT_NODE and (x.nodeName == 'redirect'):
results['proto'] = str(x.getAttribute('proto'))
results['hostname'] = str(x.getAttribute('hostname'))
results['path'] = str(x.getAttribute('path'))
if isok:
return pw
except xml.parsers.expat.ExpatError, e:
logger.error("%s", returned_data)
logger.error("%s", e)
raise
def create_user_filter(index,
user_id,
server=None,
prefix="myemsl_user",
config=config):
alias = "%s_%i" % (prefix, user_id)
filter = """
{
"actions": [{
"add": {
"index": "%s",
"alias": "%s",
"filter": {
"or": [{
"term": {"users": %i}
}, {
"term": {"aged": true}
}]
}
}
}]
}""" % (index, alias, user_id)
if server == None:
server = config.get('elasticsearch', 'server')
writebody = ""
url = server
if url[-1:] != '/':
url += '/'
url += "_aliases"
try:
writebody = call_curl(url, method="POST", idata=filter)
except CurlException, ex:
return ex.http_code
def proposals_process(chunk_items, collection):
query = {
"fields": ["proposals"],
"query": {
"constant_score": {
"filter": {
"ids": {
"values": chunk_items
}
}
}
}
}
server = config.get('elasticsearch', 'server')
results = json.loads(
call_curl("%s/myemsl_current_simple_items/simple_items/_search" %
(server),
method='POST',
idata=json.dumps(query)))
if results['hits']['total'] != len(chunk_items):
raise Exception('Failed to find some items')
for hit in results['hits']['hits']:
if 'fields' in hit and 'proposals' in hit['fields']:
collection.update(
{'_id': int(hit['_id'])},
{"$set": {
'value.p': hit['fields']['proposals']
}},
w=1)
def eus_user_add(username, results):
if username == None:
raise Exception("User is None")
isok = False
try:
pw = str(random.getrandbits(8 * 32))
m = hashlib.md5()
m.update(pw)
pw = m.hexdigest()
pdata = "%s" % (pw)
writebody = ""
writebody = call_curl(url, method="PUT", idata=pdata, capath="/etc/pki/tls/certs/ca-bundle.crt")
logger.info("%s", writebody)
dom = xml.dom.minidom.parseString(writebody)
found = False
for x in dom.firstChild.childNodes:
if x.nodeType == x.ELEMENT_NODE and (x.nodeName == "status"):
isok = str(x.getAttribute("action")) == "added"
if x.nodeType == x.ELEMENT_NODE and (x.nodeName == "redirect"):
results["proto"] = str(x.getAttribute("proto"))
results["hostname"] = str(x.getAttribute("hostname"))
results["path"] = str(x.getAttribute("path"))
if isok:
return pw
except xml.parsers.expat.ExpatError, e:
logger.error("%s", returned_data)
logger.error("%s", e)
raise
def create_user_filter(index, user_id, server=None, prefix="myemsl_user", config=config):
alias = "%s_%i" %(prefix, user_id)
filter = """
{
"actions": [{
"add": {
"index": "%s",
"alias": "%s",
"filter": {
"or": [{
"term": {"users": %i}
}, {
"term": {"aged": true}
}]
}
}
}]
}""" %(index, alias, user_id)
if server == None:
server = config.get('elasticsearch', 'server')
writebody = ""
url = server
if url[-1:] != '/':
url += '/'
url += "_aliases"
try:
writebody = call_curl(url, method="POST", idata=filter)
except CurlException, ex:
return ex.http_code
def delete_index(index_name, server=None, config=config):
if server == None:
server = config.get('elasticsearch', 'server')
writebody = ""
url = server
if url[-1:] != '/':
url += '/'
url += index_name
try:
writebody = call_curl(url, method="DELETE")
except CurlException, ex:
return ex.http_code
def delete_index(index_name, server=None, config=config):
if server == None:
server = config.get('elasticsearch', 'server')
writebody = ""
url = server
if url[-1:] != '/':
url += '/'
url += index_name
try:
writebody = call_curl(url, method="DELETE")
except CurlException, ex:
return ex.http_code
def bulk_action(index, actions, server=None, config=config):
alias_cmd = '\n'.join([json.dumps(i) for i in actions])
alias_cmd += '\n'
if server == None:
server = config.get('elasticsearch', 'server')
writebody = ""
url = server
if url[-1:] != '/':
url += '/'
url += index + "/_bulk"
try:
writebody = call_curl(url, method="POST", idata=alias_cmd)
except CurlException, ex:
return ex.http_code
def create_index(index_name, server=None, config=config, desc=None):
if desc == None:
desc = myemsl.elasticsearch.schema.schema_get('simple_item')
if server == None:
server = config.get('elasticsearch', 'server')
writebody = ""
url = server
if url[-1:] != '/':
url += '/'
url += index_name
try:
writebody = call_curl(url, method="PUT", idata=desc)
except CurlException, ex:
return ex.http_code
def get_alias(index='simple_items', alias=None, server=None, config=config):
if alias == None:
alias = "%s_%s" %(config.get('elasticsearch', "alias"), index)
if server == None:
server = config.get('elasticsearch', 'server')
url = server
if url[-1:] != '/':
url += '/'
#FIXME This grabs all aliases. should find an api that maps only the single one we need.
url += alias + '/_aliases'
writebody = ""
try:
writebody = call_curl(url)
except CurlException, ex:
return None, ex.http_code
def get_user_remote(username, map='web_map'):
"""This handles authorization"""
newuser = None
try:
url = "%s%s/%s" %(config.get('getuser', 'prefix'), config.get('getuser', map), urllib.quote(username))
writebody = call_curl(url, capath='/etc/pki/tls/certs/ca-bundle.crt')
logger.info(writebody)
dom = xml.dom.minidom.parseString(writebody)
found = False
for x in dom.firstChild.childNodes:
if x.nodeType == x.ELEMENT_NODE and (x.nodeName == 'user'):
newuser = str(x.getAttribute('id'))
break
except Exception, e:
logger.error("%s", e)
raise e
def get_user_remote(username, map='web_map'):
"""This handles authorization"""
newuser = None
try:
url = "%s%s/%s" % (config.get('getuser', 'prefix'),
config.get('getuser', map), urllib.quote(username))
writebody = call_curl(url, capath='/etc/pki/tls/certs/ca-bundle.crt')
logger.info(writebody)
dom = xml.dom.minidom.parseString(writebody)
found = False
for x in dom.firstChild.childNodes:
if x.nodeType == x.ELEMENT_NODE and (x.nodeName == 'user'):
newuser = str(x.getAttribute('id'))
break
except Exception, e:
logger.error("%s", e)
raise e
def eus_user_remove(username):
if username == None:
raise Exception("User is None")
isok = False
try:
writebody = ""
url = "%s/%s" % (config.get("metadata", "eus_auth"), urllib.quote(username))
writebody = call_curl(url, method="DELETE", capath="/etc/pki/tls/certs/ca-bundle.crt")
logger.info("%s", writebody)
dom = xml.dom.minidom.parseString(writebody)
found = False
for x in dom.firstChild.childNodes:
if x.nodeType == x.ELEMENT_NODE and (x.nodeName == "status"):
isok = str(x.getAttribute("action")) == "removed"
break
except Exception, e:
logger.error("%s", e)
raise
def item_get(item_id):
user_tries = 2
while user_tries > 0:
url = config.get('elasticsearch', 'server')
if url[-1:] != '/':
url += '/'
#FIXME unhardcode these...
url += "%s/%s/%i" %('myemsl_current_simple_items', 'simple_items', item_id)
writebody = ""
try:
writebody = call_curl(url)
except CurlException, ex:
return ex.http_code, None
j = json.loads(writebody)
if not j.get('_source'):
return 500, None
return 200, j['_source']
def get_transaction(user):
transxml = call_curl(
"%s/%s" % (config.get('metadata', 'transaction_base_url'), user),
capath=None,
cainfo='/etc/myemsl/keys/server/local.crt',
sslcert='/etc/myemsl/keys/server/local.pem',
sslcerttype='PEM')
dom = xml.dom.minidom.parseString(transxml)
transaction = -1
found = False
for x in dom.firstChild.childNodes:
if x.nodeType == x.ELEMENT_NODE and (x.nodeName == 'transaction'):
transaction = int(x.getAttribute('id'))
found = True
break
if not found:
raise Exception("Could not get transaction number from server")
return transaction
def get_transaction(user):
transxml = call_curl(
"%s/%s" %(config.get('metadata', 'transaction_base_url'), user),
capath=None,
cainfo='/etc/myemsl/keys/server/local.crt',
sslcert='/etc/myemsl/keys/server/local.pem',
sslcerttype='PEM'
)
dom = xml.dom.minidom.parseString(transxml)
transaction = -1
found = False
for x in dom.firstChild.childNodes:
if x.nodeType == x.ELEMENT_NODE and (x.nodeName == 'transaction'):
transaction = int(x.getAttribute('id'))
found = True
break
if not found:
raise Exception("Could not get transaction number from server")
return transaction
def item_get(item_id):
user_tries = 2
while user_tries > 0:
url = config.get('elasticsearch', 'server')
if url[-1:] != '/':
url += '/'
#FIXME unhardcode these...
url += "%s/%s/%i" % ('myemsl_current_simple_items', 'simple_items',
item_id)
writebody = ""
try:
writebody = call_curl(url)
except CurlException, ex:
return ex.http_code, None
j = json.loads(writebody)
if not j.get('_source'):
return 500, None
return 200, j['_source']
def eus_user_remove(username):
if username == None:
raise Exception("User is None")
isok = False
try:
writebody = ""
url = "%s/%s" % (config.get('metadata',
'eus_auth'), urllib.quote(username))
writebody = call_curl(url,
method="DELETE",
capath='/etc/pki/tls/certs/ca-bundle.crt')
logger.info("%s", writebody)
dom = xml.dom.minidom.parseString(writebody)
found = False
for x in dom.firstChild.childNodes:
if x.nodeType == x.ELEMENT_NODE and (x.nodeName == 'status'):
isok = (str(x.getAttribute('action')) == 'removed')
break
except Exception, e:
logger.error("%s", e)
raise
def proposals_process(chunk_items, collection):
query = {
"fields": [
"proposals"
],
"query": {
"constant_score": {
"filter": {
"ids": {
"values": chunk_items
}
}
}
}
}
server = config.get('elasticsearch', 'server')
results = json.loads(call_curl("%s/myemsl_current_simple_items/simple_items/_search" %(server), method='POST', idata=json.dumps(query)))
if results['hits']['total'] != len(chunk_items):
raise Exception('Failed to find some items')
for hit in results['hits']['hits']:
if 'fields' in hit and 'proposals' in hit['fields']:
collection.update({'_id': int(hit['_id'])}, {"$set": {'value.p': hit['fields']['proposals']}}, w=1)
def create_alias(index, alias, server=None, config=config):
##FIXME delete alias if it already exists
alias_cmd = """
{
"actions": [{
"add": {
"index": "%s",
"alias": "%s"
}
}]
}""" %(index, alias)
if server == None:
server = config.get('elasticsearch', 'server')
writebody = ""
url = server
if url[-1:] != '/':
url += '/'
url += "_aliases"
try:
writebody = call_curl(url, method="POST", idata=alias_cmd)
except CurlException, ex:
return ex.http_code
def elasticsearchquery(user,
index,
type,
req,
retries=1,
auth_add=False,
search_type=None,
scan=None):
req_data = req.read()
if type != 'released_publications' and user == '':
req.write("Forbidden")
return 401
user_tries = 2
while user_tries > 0:
if type != "simple_items" and type != "released_publications":
req.write("Forbidden")
return 401
if type == "released_publications":
index = "myemsl_current_released_publications"
auth_add = False
server = config.get('elasticsearch', 'server')
url = server
if url[-1:] != '/':
url += '/'
if scan:
url += '/_search/scroll?scroll=10m'
else:
url += "%s/%s/_search" % (index, type)
if search_type:
url += '?search_type=scan&scroll=10m&size=50'
writebody = ""
skipit = 1
try:
writebody = call_curl(url, method="POST", idata=req_data)
except CurlException, ex:
skipit = 0
if skipit:
if auth_add:
auth_items = []
j = json.loads(writebody)
for i in j['hits']['hits']:
auth_items.append(i['_id'])
if len(auth_items) > 0:
token = myemsl.token.simple_items_token_gen(
auth_items, person_id=int(user))
logger.debug("Requested auth. %s" % (auth_items))
j['myemsl_auth_token'] = token
req.write(json.dumps(j))
return 200
else:
req.write(writebody)
return 200
retval = ""
alias_tries = 3
while alias_tries > 0:
retval, code = myemsl.elasticsearch.get_alias(index=type)
if code == 404:
time.sleep(1)
alias_tries -= 1
elif code == 200:
break
else:
return code
if alias_tries < 1:
return 500
if type != 'released_publications':
user_id = int(user)
code = myemsl.elasticsearch.create_user_filter(retval, user_id)
if code < 200 or code > 299:
return 404
user_tries -= 1
def elasticsearchquery(user, index, type, req, retries=1, auth_add=False, search_type=None, scan=None):
req_data = req.read()
if type != 'released_publications' and user == '':
req.write("Forbidden")
return 401
user_tries = 2
while user_tries > 0:
if type != "simple_items" and type != "released_publications":
req.write("Forbidden")
return 401
if type == "released_publications":
index = "myemsl_current_released_publications"
auth_add = False
server = config.get('elasticsearch', 'server')
url = server
if url[-1:] != '/':
url += '/'
if scan:
url += '/_search/scroll?scroll=10m'
else:
url += "%s/%s/_search" %(index, type)
if search_type:
url += '?search_type=scan&scroll=10m&size=50'
writebody = ""
skipit = 1
try:
writebody = call_curl(url, method="POST", idata=req_data)
except CurlException, ex:
skipit = 0
if skipit:
if auth_add:
auth_items = []
j = json.loads(writebody)
for i in j['hits']['hits']:
auth_items.append(i['_id'])
if len(auth_items) > 0:
token = myemsl.token.simple_items_token_gen(auth_items, person_id=int(user))
logger.debug("Requested auth. %s" %(auth_items))
j['myemsl_auth_token'] = token
req.write(json.dumps(j))
return 200
else:
req.write(writebody)
return 200
retval = ""
alias_tries = 3
while alias_tries > 0:
retval, code = myemsl.elasticsearch.get_alias(index=type)
if code == 404:
time.sleep(1)
alias_tries -= 1
elif code == 200:
break
else:
return code
if alias_tries < 1:
return 500
if type != 'released_publications':
user_id = int(user)
code = myemsl.elasticsearch.create_user_filter(retval, user_id)
if code < 200 or code > 299:
return 404
user_tries -= 1
