def show_keyword_map(collection_name):
print "Show Map Page"
errors = []
results = {"four": 4}
print collection_name
# collection_name = request.args.get('collection_name', 'ritesh', type=str)
# print "Collection clicked in the list iddds : ", collection_name
db = libmongo.get_db()
keywords = db.ks.find_one({
"unique_name": collection_name
}).get("keyword_list")
variables = db.vs.find_one({
"unique_name": collection_name
}).get("variable_list")
maps = db.ms.find_one({"unique_name": collection_name})
maps["mapped_keys"] = list()
for kk in maps["cfk"].values():
maps["mapped_keys"] += kk
# print keywords, variables, maps
sorted_maps = dict()
sorted_maps["unique_name"] = collection_name
sorted_maps["kv"] = get_sorted_kv_map(maps)
sorted_maps["vk"] = get_sorted_vk_map(maps)
# print sorted_maps
return render_template('show_keyword_map.html', errors=errors, results=results, variables=variables, \
keywords=keywords, maps=maps, collections=collections, \
sorted_maps=sorted_maps)
def update_keyword_map(collection_name):
errors = []
results = {"four": 4}
print collection_name
TO_UPDATE = {}
if request.method == 'POST':
print "here is the update thing "
# print list(request.form.keys())
# print request.form
db = libmongo.get_db()
col_map = db.ms.find_one({"unique_name": collection_name})
kvs = col_map.get("kv")
# print col_map
for keyword in request.form.keys():
variables = request.form.getlist(keyword)
variables = list(set(variables))
key = keyword.split("-", 2)[-1]
old_mapped_kv = kvs.get(key).get("mapped")
old_ranked_kv = kvs.get(key).get("ranked")
old_mapped_kv_set = set(old_mapped_kv.keys())
old_ranked_kv_set = set(old_ranked_kv.keys())
new_mapped_kv_set = set()
mapped_d = dict()
ranked_d = dict()
for variable in variables:
v, i = variable.split("-->", 1)
mapped_d[v] = int(i) #populate mapped new
new_mapped_kv_set.add(v)
added_to_mapped = new_mapped_kv_set.difference(old_mapped_kv_set)
removed_from_mapped = old_mapped_kv_set.difference(
new_mapped_kv_set)
for am_key in added_to_mapped:
# del old_ranked_kv[am_key] #remove from ranked; add to mapped
old_mapped_kv[am_key] = old_ranked_kv.pop(am_key)
for ar_key in removed_from_mapped:
old_ranked_kv[ar_key] = old_mapped_kv.pop(
ar_key) #populate ranked map / add to ranked
kvs[key] = {"mapped": old_mapped_kv, "ranked": old_ranked_kv}
# TO_UPDATE[key] = {"mapped" : old_mapped_kv, "ranked": old_ranked_kv}
# print "things to update is :", kvs
print "Collection clicked in the list iddds : ", collection_name
query = {"unique_name": collection_name}
update = {"$set": {"kv": kvs}}
result = db.ms.update_one(query, update)
if result.matched_count == 1:
print "Successful update"
# keywords = db.ks.find_one({"unique_name": collection_name}).get("keyword_list")
# variables = db.vs.find_one({"unique_name": collection_name}).get("variable_list")
# maps = db.ms.find_one({"unique_name": collection_name})
# print keywords, variables, maps
return redirect(
url_for("show_keyword_map", collection_name=collection_name))
def update_variable_map(collection_name):
errors = []
results = {"four": 4}
print collection_name
TO_UPDATE = {}
if request.method == 'POST':
print "here is the update thing for variable"
# print list(request.form.keys())
db = libmongo.get_db()
col_map = db.ms.find_one({"unique_name": collection_name})
vks = col_map.get("vk")
for variable in request.form.keys():
keywords = request.form.getlist(variable)
keywords = list(set(keywords))
key = variable.split("-", 2)[-1]
old_mapped_vk = vks.get(key).get("mapped")
old_ranked_vk = vks.get(key).get("ranked")
old_mapped_vk_set = set(old_mapped_vk.keys())
old_ranked_vk_set = set(old_ranked_vk.keys())
new_mapped_vk_set = set()
mapped_d = dict()
ranked_d = dict()
for keyword in keywords:
v, i = keyword.split("-->", 1)
mapped_d[v] = int(i) #populate mapped new
new_mapped_vk_set.add(v)
added_to_mapped = new_mapped_vk_set.difference(old_mapped_vk_set)
removed_from_mapped = old_mapped_vk_set.difference(
new_mapped_vk_set)
for am_key in added_to_mapped:
# del old_ranked_vk[am_key] #remove from ranked
old_mapped_vk[am_key] = old_ranked_vk.pop(am_key)
for ar_key in removed_from_mapped:
old_ranked_vk[ar_key] = old_mapped_vk.pop(
ar_key) #populate ranked map / add to ranked
vks[key] = {"mapped": old_mapped_vk, "ranked": old_ranked_vk}
# print "things to update is :", TO_UPDATE
print "Collection clicked in the list iddds : ", collection_name
query = {"unique_name": collection_name}
update = {"$set": {"vk": vks}}
result = db.ms.update_one(query, update)
if result.matched_count == 1:
print "Successful variable map update"
return redirect(
url_for("show_keyword_map", collection_name=collection_name))
def index():
global collections
errors = []
results = {}
# variables = ["these", "are", "dynamic", "variables", "for", "testing"]
# keywords = ["these", "are", "dynamic", "keywords", "for", "testing"]
if request.method == "POST":
# get url that the user has entered
try:
url = request.form['url']
# r = requests.get(url)
r = url
# print r
except:
errors.append(
"Unable to get URL. Please make sure it's valid and try again."
)
return render_template('index.html',
errors=errors,
results=results,
variables=variables,
keywords=keywords)
if r:
results[r] = len(r)
for result in results:
print result[0], result[1]
print results
print results
# print variables
try:
db = libmongo.get_db()
collections = db.ms.find()
#create searchable data
grid_data = list()
for collection in db.ms.find():
grid_data.append(
dict(
db.ks.find_one({"unique_name": collection['unique_name']},
{
"_id": 0,
"unique_name": 1,
"daac": 1,
"dataset_id": 1
})))
# grid_data = list(db.ks.find({}, {"_id":0, "unique_name": 1, "daac": 1, "dataset_id": 1}))
# grid_data = [data for data in grid_data_cursor]
except Exception, e:
print "Db exception error", e
def search_service():
#search from ms(map) collection
try:
if request.method == "GET":
db = libmongo.get_db()
unique_name = request.args.get("unique_name")
dataset_id = request.args.get("dataset_id")
variable = request.args.get("variable")
keyword = request.args.get("keyword")
unique_name = unique_name if unique_name is not None else dataset_id
if not (unique_name or dataset_id or variable or keyword):
js = to_json({"search": "No search options provided", \
"example": "..url/service/search?unique_name=name_without_slash&dataset_id=something&variable=var1&keyword=key2"})
resp = Response(js, status=200, mimetype='application/json')
return resp
else:
if unique_name:
result = db.ms.find_one({"unique_name": unique_name})
# result = get_map_result(result)
if keyword:
js = to_json({keyword: result["kv"][keyword]})
resp = Response(js,
status=200,
mimetype='application/json')
return resp
elif variable:
js = to_json({variable: result["vk"][variable]})
resp = Response(js,
status=200,
mimetype='application/json')
return resp
else:
js = to_json(get_map_result(result))
resp = Response(js,
status=200,
mimetype='application/json')
return resp
else:
js = to_json(
{"search": "Missing dataset_id or unique_name"})
resp = Response(js,
status=200,
mimetype='application/json')
return resp
except Exception, e:
print(e)
return not_found()
def rest_service_keyword(db_table_name, id):
"""Return specific UFO sighting
ex) GET /service/keyword/123456
"""
try:
if request.method == 'GET':
db = libmongo.get_db()
table_name = python_to_mongo_map.get(db_table_name)
result = db[table_name].find_one({'_id': ObjectId(id)})
js = to_json(
get_map_result(result)) if table_name == "ms" else to_json(
get_result(result))
resp = Response(js, status=200, mimetype='application/json')
return resp
except Exception, e:
print(e)
return not_found()
def rest_service(db_table_name):
"""Return a list of all UFO sightings
ex) GET /service/<db_table_name>
GET /service/db_table_name?dataset_id=something
"""
try:
if request.method == 'GET':
db = libmongo.get_db()
table_name = python_to_mongo_map.get(db_table_name)
dataset_id = request.args.get("dataset_id")
print dataset_id
if not dataset_id: #if dataset_id is not given
results = db[table_name].find()
json_results = []
if not table_name == "ms":
for result in results:
json_results.append(get_result(result))
else:
for result in results:
json_results.append(get_map_result(result))
# resp = jsonify({"keywords": json_results})
# resp.status_code = 200
# return resp
js = to_json(json_results)
resp = Response(js, status=200, mimetype='application/json')
return resp
else:
result = db[table_name].find_one({"dataset_id": dataset_id})
js = to_json(
get_map_result(result)) if table_name == "ms" else to_json(
get_result(result))
resp = Response(js, status=200, mimetype='application/json')
return resp
except Exception, e:
print(e)
return not_found()
def update_cfk_map(collection_name):
errors = []
results = {"four": 4}
print collection_name
if request.method == 'POST':
# print list(request.form.keys())
db = libmongo.get_db()
cfk = dict()
for variable in request.form.keys():
keywords = request.form.getlist(variable)
keywords = list(set(keywords))
key = variable.split("-", 2)[-1]
# print key, keywords
cfk[key] = list(keywords)
query = {"unique_name": collection_name}
update = {"$set": {"cfk": cfk}}
result = db.ms.update_one(query, update)
if result.matched_count == 1:
print "Successful cfk map update"
return redirect(
url_for("show_keyword_map", collection_name=collection_name))
def show_map(collection_name):
errors = []
results = {"four": 4}
print collection_name
# collection_name = request.args.get('collection_name', 'ritesh', type=str)
# print "Collection clicked in the list is : ", collection_name
db = libmongo.get_db()
keywords = db.ks.find_one({
"unique_name": collection_name
}).get("keyword_list")
variables = db.vs.find_one({
"unique_name": collection_name
}).get("variable_list")
maps = db.ms.find_one({"unique_name": collection_name})
# print keywords, variables, maps
return render_template('maps.html',
errors=errors,
results=results,
variables=variables,
keywords=keywords,
maps=maps,
collections=collections)
def edit_cfu_map(collection_name):
errors = []
results = {"four": 4}
print collection_name
# collection_name = request.args.get('collection_name', 'ritesh', type=str)
# print "Collection clicked in the list iddds : ", collection_name
db = libmongo.get_db()
keywords = db.ks.find_one({
"unique_name": collection_name
}).get("keyword_list")
variables = db.vs.find_one({
"unique_name": collection_name
}).get("variable_list")
maps = db.ms.find_one({"unique_name": collection_name})
# print keywords, variables, maps
sorted_maps = dict()
sorted_maps["unique_name"] = collection_name
sorted_maps["kv"] = get_sorted_kv_map(maps)
sorted_maps["vk"] = get_sorted_vk_map(maps)
return render_template('edit_cfu_map.html', errors=errors, results=results, variables=variables, \
keywords=keywords, maps=maps, collections=collections, \
sorted_maps=sorted_maps)