def create_objects(collection_name, objects, limit=None, offset=None, *args, **kwargs):
try:
response = createResponse200()
mongo = Mongo()
mongo.connect()
database = mongo.use_db(database_name)
if type(objects) is not list:
objects = [objects]
# Checking if any of the object already exist by _id
objects_id = [ object.get('_id', None) for object in objects ]
query = {'_id' : {'$in' : objects_id }}
existing = mongo.find(collection_name, query)
existing_ids = [ e.get('_id') for e in existing ]
if existing:
raise Exception("Some objects already exist in '%s' (same _id). Trying using UPDATE method. No objects created." % collection_name)
# no limits or offset so far
created = mongo.insert(collection_name, objects)
response['result']['objects'] = created
response['result']['count'] = len(created)
response['result']['limit'] = limit
response['result']['offset'] = offset
except Exception, e:
response = createResponse401(str(e))
def update_objects(collection_name,
objects=None,
limit=0,
offset=0,
*args,
**kwargs):
try:
response = createResponse200()
mongo = Mongo()
mongo.connect()
database = mongo.use_db(database_name)
updated = []
if type(objects) is not list:
objects = [objects]
for object in objects:
object_id = {'_id': object.get('_id')}
up = mongo.update(collection_name, object_id, object)
updated.append(up)
response['result']['objects'] = updated
response['result']['count'] = len(updated)
response['result']['limit'] = limit
response['result']['offset'] = offset
except Exception, e:
response = createResponse401(str(e))
def update_objects(collection_name, objects=None, limit=0, offset=0, *args, **kwargs):
try:
response = createResponse200()
mongo = Mongo()
mongo.connect()
database = mongo.use_db(database_name)
updated = []
if type(objects) is not list:
objects = [objects]
for object in objects:
object_id = { '_id' : object.get('_id') }
up = mongo.update(collection_name, object_id, object)
updated.append(up)
response['result']['objects'] = updated
response['result']['count'] = len(updated)
response['result']['limit'] = limit
response['result']['offset'] = offset
except Exception, e:
response = createResponse401(str(e))
def delete_objects(collection_name, objects=None, limit=0, offset=0, *args, **kwargs):
try:
response = createResponse200()
mongo = Mongo()
mongo.connect()
database = mongo.use_db(database_name)
if type(objects) is not list:
objects = [objects]
deleted = []
for object in objects:
if isinstance(object, basestring):
object = { '_id' : get_mongo_id(object) }
d = mongo.delete(collection_name, object, safe=True)
if d.get('n') == 1:
deleted.append(object.get('_id'))
if type(object) is None:
deleted = "All the items have been deleted"
response['result']['deleted_objects'] = deleted
response['result']['count'] = len(deleted)
response['result']['limit'] = limit
response['result']['offset'] = offset
except Exception, e:
response = createResponse401(str(e))
def submit_vote_to_db(request):
response = {'success': False}
import json
if request.is_ajax():
if request.method == 'POST':
print request.body
response['success'] = True
jbody = json.loads(request.body)
print jbody
# sanitze the vote object before using it in the mongo update() function
from mongosanitizer.sanitizer import sanitize
sanitize(jbody)
# hash/encrypt the voter id before using it in the mongo update() functions
import hashlib
email = str(jbody['voter_id'])
hashed_email_object = hashlib.md5(email.encode())
jbody['voter_id'] = hashed_email_object.hexdigest()
# add vote to mongdb
from mongo import Mongo
mongo = Mongo()
collection_q = mongo.connect(host, port, db, collection_questions_responses)
query = {}
# relaease this to append the vote to its correct concept
query['ConceptId'] = jbody['concept_id'] #1
push = {}
push['$push'] = {}
push['$push']['votes'] = jbody
collection_q.update(
query,
push
)
# add voted tasks to users collections
collection_u = mongo.connect(host, port, db, collection_users)
query_ = {}
query_['UserID'] = jbody['voter_id']
push_set = {}
push_set['$push'] = {}
push_set['$push']['voted_concepts_list'] = jbody['concept_id']
push_set['$push']['voted_cursor_list'] = jbody['voter_survey_cursor']
push_set['$push']['voting_dates'] = jbody['vote_date']
push_set['$set'] = {}
push_set['$set']['last_vote_date'] = jbody['vote_date']
collection_u.update(
query_,
push_set,
upsert=True
)
from django.http import JsonResponse
print JsonResponse(response)
return HttpResponse(json.dumps(response), content_type="application/json")
def get_data_from_mongodb(request):
from mongo import Mongo
import copy, json
print 'connecting to mongodb . . . '
mongo = Mongo()
# get all data 188 concepts and their votes
# mongo.getfromMongo(collection = '', query = '')
# mongo.appendDataListToMongo(host=mongo_host, port=mongo_port, dbName=mongo_db,
# collectionName=datasets_collection, Item_JsonObject=dataset_object,
# id_field=id_field)
collection = mongo.connect(host, port, db, collection_questions_responses)
cursor = collection.find(no_cursor_timeout=True)
response = {}
# response['prev_tasks'] = []
response['concepts'] = []
concept = {}
for item in cursor:
# response['prev_tasks'].extend(copy.deepcopy(concept['votes']['voted_tasks']))
concept['id'] = copy.deepcopy(item['ConceptId'])
concept['imports'] = copy.deepcopy(item['attributes'])
response['concepts'].append(copy.deepcopy(concept))
concept.clear()
from django.http import JsonResponse
print JsonResponse(response)
return HttpResponse(json.dumps(response), content_type="application/json")
def read_objects(collection_name, query={}, limit=0, offset=0, sort=settings.DEFAULT_SORT, *args, **kwargs):
try:
response = createResponse200()
mongo = Mongo()
mongo.connect()
database = mongo.use_db(database_name)
count = mongo.count(collection_name, query, limit=0, offset=0, sort=sort)
found = mongo.find(collection_name, query, limit=limit, offset=offset, sort=sort)
response['result']['total'] = count
response['result']['has_more'] = True if limit + offset < count else False
response['result']['objects'] = found
response['result']['count'] = len(found)
response['result']['limit'] = limit
response['result']['offset'] = offset
except Exception, e:
response = createResponse401(str(e))
def create_objects(collection_name,
objects,
limit=None,
offset=None,
*args,
**kwargs):
try:
response = createResponse200()
mongo = Mongo()
mongo.connect()
database = mongo.use_db(database_name)
if type(objects) is not list:
objects = [objects]
# Checking if any of the object already exist by _id
objects_id = [object.get('_id', None) for object in objects]
query = {'_id': {'$in': objects_id}}
existing = mongo.find(collection_name, query)
existing_ids = [e.get('_id') for e in existing]
if existing:
raise Exception(
"Some objects already exist in '%s' (same _id). Trying using UPDATE method. No objects created."
% collection_name)
# no limits or offset so far
created = mongo.insert(collection_name, objects)
response['result']['objects'] = created
response['result']['count'] = len(created)
response['result']['limit'] = limit
response['result']['offset'] = offset
except Exception, e:
response = createResponse401(str(e))
def delete_objects(collection_name,
objects=None,
limit=0,
offset=0,
*args,
**kwargs):
try:
response = createResponse200()
mongo = Mongo()
mongo.connect()
database = mongo.use_db(database_name)
if type(objects) is not list:
objects = [objects]
deleted = []
for object in objects:
if isinstance(object, basestring):
object = {'_id': get_mongo_id(object)}
d = mongo.delete(collection_name, object, safe=True)
if d.get('n') == 1:
deleted.append(object.get('_id'))
if type(object) is None:
deleted = "All the items have been deleted"
response['result']['deleted_objects'] = deleted
response['result']['count'] = len(deleted)
response['result']['limit'] = limit
response['result']['offset'] = offset
except Exception, e:
response = createResponse401(str(e))
def get_voted_tasks_from_mongodb(request, userid):
from mongo import Mongo
import copy, json
print 'connecting to mongodb . . . '
jresponse = {}
# response['prev_tasks'] = []
jresponse['voted_tasks_concepts'] = []
jresponse['voted_tasks_cursors'] = []
mongo = Mongo()
collection = mongo.connect(host, port, db, collection_users)
query = {}
# sanitze the userid before using it in the mongo find() function
from mongosanitizer.sanitizer import sanitize
sanitize(userid)
# hash/encrypt user_id email before querying db to find match
import hashlib
email = userid
hashed_email_object = hashlib.md5(email.encode())
# prepare query
query['UserID'] = hashed_email_object.hexdigest()
cursor = collection.find(query, no_cursor_timeout=True)
voted_tasks_concepts = []
voted_tasks_cursors = []
for item in cursor:
# response['prev_tasks'].extend(copy.deepcopy(concept['votes']['voted_tasks']))
print item['voted_concepts_list']
print item['voted_cursor_list']
jresponse['voted_tasks_concepts'].extend(copy.deepcopy(item['voted_concepts_list']))
jresponse['voted_tasks_cursors'].extend(copy.deepcopy(item['voted_cursor_list']))
# from django.http import JsonResponse
# print response(JsonResponse(jresponse))
# return response(JsonResponse(jresponse))
response = HttpResponse(json.dumps(jresponse), content_type="application/json")
# cookies set
if not request.COOKIES.get('userid'):
# response = HttpResponse()
response.set_cookie('userid', userid, 3600 * 24 * 365) # one year cookie
# return response
elif request.COOKIES.get('userid') != userid:
response.set_cookie('userid', userid, 3600 * 24 * 365) # one year cookie
return response
def read_objects(collection_name,
query={},
limit=0,
offset=0,
sort=settings.DEFAULT_SORT,
*args,
**kwargs):
try:
response = createResponse200()
mongo = Mongo()
mongo.connect()
database = mongo.use_db(database_name)
count = mongo.count(collection_name,
query,
limit=0,
offset=0,
sort=sort)
found = mongo.find(collection_name,
query,
limit=limit,
offset=offset,
sort=sort)
response['result']['total'] = count
response['result'][
'has_more'] = True if limit + offset < count else False
response['result']['objects'] = found
response['result']['count'] = len(found)
response['result']['limit'] = limit
response['result']['offset'] = offset
except Exception, e:
response = createResponse401(str(e))
def get_tasks_from_mongodb(request):
from mongo import Mongo
import copy, json
print 'connecting to mongodb . . . '
mongo = Mongo()
# get all data 188 concepts and their votes
# mongo.getfromMongo(collection = '', query = '')
# mongo.appendDataListToMongo(host=mongo_host, port=mongo_port, dbName=mongo_db,
# collectionName=datasets_collection, Item_JsonObject=dataset_object,
# id_field=id_field)
collection = mongo.connect(host, port, db, collection_questions_responses)
# get non embty votes only
query = {}
query["votes"] = {
u"$exists": True,
u"$not": {
u"$size": 0.0
}
}
cursor = collection.find(query, no_cursor_timeout=True)
# add your own taska from your ref.csv before loading new tasks from survey results
prev_tasks = []
# removed to prevent bias
# preprepared_tasks = get_tasks_dic()
# prev_tasks.extend(preprepared_tasks)
for item in cursor:
# response['prev_tasks'].extend(copy.deepcopy(concept['votes']['voted_tasks']))
print item['ConceptId']
# loop in votes
for vote in item['votes']:
if vote:
# print vote
prev_tasks.extend(copy.deepcopy(vote['vote_tasks'])) # ['attributes']))
# from django.http import JsonResponse
# print JsonResponse(prev_tasks)
# return JsonResponse(prev_tasks, safe=False)
return HttpResponse(json.dumps(prev_tasks), content_type="application/json")
from mongo import Mongo from neo import Neo global mongo mongo = Mongo() mongo.connect(host="localhost",port="27017",username="******",password="******",database_name="recommendation-engine") global neo neo = Neo() neo.connect(host="localhost",port="7474",username="******",password="******")
def post(self):
for i in range(1):
self.set_header("Access-Control-Allow-Origin", "*")
LOG.info('API IN[%s]' % (self.__class__.__name__))
LOG.info('PARAMETER IN[%s]' % self.request.arguments)
ret = {'code':'','message':''}
essential_keys = set(['json','html','topic','level','type','group','chapter','ref'])
if Base.check_parameter(set(self.request.arguments.keys()),essential_keys):
ret['code'] = 1
ret['message'] = '无效参数'
LOG.error('ERR[in parameter invalid]')
break
question_json = ''.join(self.request.arguments['json'])
question_html = ''.join(self.request.arguments['html'])
question_topic = ''.join(self.request.arguments['topic'])
question_level = ''.join(self.request.arguments['level'])
question_type = ''.join(self.request.arguments['type'])
question_group = ''.join(self.request.arguments['group'])
question_chapter = ''.join(self.request.arguments['chapter'])
ref = ''.join(self.request.arguments['ref'])
if Business.is_level(question_level) is False:
ret['code'] = 1
ret['message'] = '无效参数'
LOG.error('ERR[level is invalid]')
break
try:
#question_json = urllib.unquote(question_json)
#question_json = question_json.replace("'","\"")
encode_json = json.loads(question_json,encoding = 'utf-8')
#question_html = urllib.unquote(question_html)
#question_html = question_html.replace("'","\"")
encode_html = json.loads(question_html,encoding = 'utf-8')
LOG.info('Json Loads Successful')
answer_num = 0
if Base.empty(question_topic) and Base.empty(question_chapter):
ret['code'] = 1
ret['message'] = '无效参数'
LOG.error('ERR[topic and chapter empty]')
break
if Base.empty(question_group):
ret['code'] = 1
ret['message'] = '无效参数'
LOG.error('ERR[group empty]')
break
if Base.empty(question_topic) is False:
topic_list = question_topic.split(',')
for question_theme in topic_list:
if Business.is_topic(question_theme) is False:
ret['code'] = 1
ret['message'] = '无效参数'
LOG.error('ERR[topic %s invalid]' % question_theme)
break
type_name = Business.is_type(question_type)
if type_name is False:
ret['code'] = 1
ret['message'] = '无效参数'
LOG.error('ERR[type is invalid]')
break
option_num = 0
LOG.info('Json Parse Start')
if type_name == '选择题'.decode('utf-8'):
if 'answer' in encode_json.keys():
answer_num = len(encode_json['answer'])
option_num = len(encode_json['options'])
if type_name == '填空题'.decode('utf-8'):
if 'answer' in encode_json.keys():
answer_num = max([int(group['index']) for group in encode_json['answer']])
LOG.info('Json Parse End')
if not Base.empty(question_chapter):
if Business.chapter_id_exist(question_chapter) is False:
ret['code'] = 1
ret['message'] = '无效参数'
LOG.error('ERR[seriess %s invalid]' % question_theme)
break
except (ValueError,KeyError,TypeError):
ret['code'] = 1
ret['message'] = '无效参数'
LOG.error('ERR[json format invalid]')
break
except CKException:
ret['code'] = 3
ret['message'] = '服务器错误'
LOG.error('ERR[mysql exception]')
break
key = question_topic + question_level + question_type + question_group
secret_key = hashlib.sha1(key).hexdigest()
qiniu = QiniuWrap()
json_key = 'tmp_' + secret_key + '.json'
if not qiniu.upload_data("temp",json_key,question_json):
ret['code'] = 4
ret['message'] = '服务器错误'
LOG.error('ERR[json upload qiniu exception]')
break
html_key = 'tmp_' + secret_key + '.html'
if not qiniu.upload_data("temp",html_key,question_html):
ret['code'] = 4
ret['message'] = '服务器错误'
LOG.error('ERR[html upload qiniu exception]')
break
configer = Configer()
remote_host = configer.get_configer('REMOTE','host')
remote_port = configer.get_configer('REMOTE','port')
remote_uri = configer.get_configer('REMOTE','uri')
remote_timeout = configer.get_configer('REMOTE','timeout')
remote_url = "http://%s:%s/%s" % (remote_host,remote_port,remote_uri)
token = self.get_cookie("teacher_id")
LOG.info('TOKEN[%s]' % token)
if token is None:
ret['code'] = 6
ret['message'] = 'token失效'
LOG.error('ERROR[token empty]')
break
post_data = {'token' : token}
client = httpclient.AsyncHTTPClient()
response = yield gen.Task(client.fetch,remote_url,request_timeout = int(remote_timeout),method = 'POST',body = urllib.urlencode(post_data
))
#response = Http.post(remote_url,post_data)
if 200 == response.code:
encode_body = json.loads(response.body,encoding = 'utf-8')
if 0 == encode_body['code'] or 2 == encode_body['code']:
ret['code'] = 7
ret['message'] = 'token失效'
LOG.error('ERR[token not exist]')
break
if 1 == encode_body['code']:
subject_id = encode_body['subject_id']
grade_id = encode_body['grade_id']
system_id = encode_body['system_id']
org_type = encode_body['org_type']
if 0 != int(question_group):
if Business.group_id_exist(question_group,system_id) is False:
ret['code'] = 8
ret['message'] = '无效参数'
LOG.error('ERROR[group not exist]')
break
db = Mysql()
question_sql = "insert into entity_question (difficulty,question_docx,html,upload_time,update_time,question_type,subject_id,new_format,upload_id,upload_src,question_group,grade_id,state,is_single,question_type_id,answer_num,count_ref,paper_year,parent_question_id,count_options) values (%(level)d,'%(json)s','%(html)s',now(),now(),'%(type)s',%(subject_id)d,1,%(upload_id)d,%(upload_src)d,%(question_group)d,%(grade_id)d,'ENABLED',1,%(question_type_id)d,%(answer_num)d,0,0,0,%(count_options)d);"
link_topic_sql = "insert into link_question_topic (question_id,topic_id) values (%(q_id)d,%(t_id)d);"
link_chapter_sql = "insert into link_question_chapter (question_id,chapter_id) values (%(q_id)d,%(c_id)d);"
try:
db.connect_master()
db.start_event()
question_res = db.exec_event(question_sql,level = int(question_level),json = json_key,html = html_key,type = type_name,subject_id = int(subject_id),upload_id = int(system_id),upload_src = int(org_type),question_group = int(question_group),grade_id = int(grade_id),question_type_id = int(question_type),answer_num = answer_num,count_options = option_num)
question_sql = db.get_last_sql()
question_id = db.get_last_id()
LOG.info('RES[%s] - INS[%d]' % (question_res,question_id))
if Base.empty(question_topic) is False:
topic_list = question_topic.split(',')
for question_theme in topic_list:
topic_res = db.exec_event(link_topic_sql,q_id = int(question_id),t_id = int(question_theme))
topic_sql = db.get_last_sql()
topic_id = db.get_last_id()
LOG.info('RES[%s] - INS[%d]' % (topic_res,topic_id))
if not Base.empty(question_chapter):
chapter_res = db.exec_event(link_chapter_sql,q_id = int(question_id),c_id = int(question_chapter))
chapter_sql = db.get_last_sql()
chapter_id = db.get_last_id()
LOG.info('RES[%s] - INS[%d]' % (chapter_res,chapter_id))
except DBException as e:
db.rollback()
db.end_event()
ret['code'] = 3
ret['message'] = '服务器错误'
LOG.error('ERR[insert mysql error]')
break
else:
ret['code'] = 3
ret['message'] = '服务器错误'
LOG.error('ERROR[remote error]')
break
mongo = Mongo()
try:
mongo.connect('resource')
mongo.select_collection('mongo_question_json')
json_id = mongo.insert_one({"content":encode_json,"question_id":question_id})
LOG.info('MONGO[insert json] - DATA[%s] - INS[%s] - Question Id[%d]' % (json.dumps(encode_json),json_id,question_id))
mongo.select_collection('mongo_question_html')
html_id = mongo.insert_one({"content":encode_html,"question_id":question_id})
LOG.info('MONGO[insert html] - DATA[%s] - INS[%s] - Question Id[%d]' % (json.dumps(encode_html),html_id,question_id))
except DBException as e:
db.rollback()
db.end_event()
ret['code'] = 3
ret['message'] = '服务器错误'
LOG.error('ERR[mongo exception]')
break
db.end_event()
if int(ref):
doit = Doit()
doit.local_img(str(question_id))
LOG.info('Local Img [%s]' % str(question_id))
ret['code'] = 0
ret['message'] = 'success'
ret['id'] = question_id
LOG.info('PARAMETER OUT[%s]' % ret)
LOG.info('API OUT[%s]' % (self.__class__.__name__))
self.write(json.dumps(ret))
self.finish()
def local_img(self,string):
from gl import LOG
update_flag = False
LOG.info('start local img,question id [%s]' % string)
question_id = int(string)
mongo = Mongo()
mongo.connect('resource')
mongo.select_collection('mongo_question_json')
json = mongo.find_one({'question_id':question_id},{'content':1})
mongo.select_collection('mongo_question_html')
html = str(mongo.find_one({'question_id':question_id},{'content':1}))
#img_expr = parse("content[*].*[*]")
#img_list = [match.value for match in img_expr.find(json) if isinstance(match.value,dict) and\
# 'type' in match.value.keys() and match.value['type'] == 'image']
#pprint.pprint(json)
content = ''
if json:
content = json['content']
for key,wrap in content.items():
for idx,item in enumerate(content[key]):
if isinstance(item,str):
continue
if isinstance(item,dict):
if 'group' in item.keys():
group = item['group']
for index,item1 in enumerate(group):
if isinstance(item1,dict) and 'type' in item1.keys() and item1['type'] == 'image':
ori_url = item1['value']
qiniu_url = self._upload_qiniu(ori_url)
if qiniu_url:
content[key][idx]['group'][index]['value'] = qiniu_url
update_flag = True
html = html.replace(ori_url,qiniu_url)
if 'type' in item.keys() and item['type'] == 'image':
ori_url = item['value']
qiniu_url = self._upload_qiniu(ori_url)
if qiniu_url:
content[key][idx]['value'] = qiniu_url
update_flag = True
html = html.replace(ori_url,qiniu_url)
if isinstance(item,list):
for index,item1 in enumerate(item):
if 'type' in item1.keys() and item1['type'] == 'image':
ori_url = item1['value']
qiniu_url = self._upload_qiniu(ori_url)
if qiniu_url:
content[key][idx][index]['value'] = qiniu_url
update_flag = True
html = html.replace(ori_url,qiniu_url)
if update_flag:
mongo.select_collection('mongo_question_json')
json_effected = mongo.update_many({'question_id':question_id},{'$set':{'content':content}})
mongo.select_collection('mongo_question_html')
html_effected = mongo.update_many({'question_id':question_id},{'$set':{'content':html}})
LOG.info('mongo update successful json[%d] -- html[%d]' % (json_effected,html_effected))
def local_img(self, string):
from gl import LOG
update_flag = False
LOG.info('start local img,question id [%s]' % string)
question_id = int(string)
mongo = Mongo()
mongo.connect('resource')
mongo.select_collection('mongo_question_json')
json = mongo.find_one({'question_id': question_id}, {'content': 1})
mongo.select_collection('mongo_question_html')
html = str(mongo.find_one({'question_id': question_id},
{'content': 1}))
#img_expr = parse("content[*].*[*]")
#img_list = [match.value for match in img_expr.find(json) if isinstance(match.value,dict) and\
# 'type' in match.value.keys() and match.value['type'] == 'image']
#pprint.pprint(json)
content = ''
if json:
content = json['content']
for key, wrap in content.items():
for idx, item in enumerate(content[key]):
if isinstance(item, str):
continue
if isinstance(item, dict):
if 'group' in item.keys():
group = item['group']
for index, item1 in enumerate(group):
if isinstance(
item1, dict) and 'type' in item1.keys(
) and item1['type'] == 'image':
ori_url = item1['value']
qiniu_url = self._upload_qiniu(ori_url)
if qiniu_url:
content[key][idx]['group'][index][
'value'] = qiniu_url
update_flag = True
html = html.replace(ori_url, qiniu_url)
if 'type' in item.keys() and item['type'] == 'image':
ori_url = item['value']
qiniu_url = self._upload_qiniu(ori_url)
if qiniu_url:
content[key][idx]['value'] = qiniu_url
update_flag = True
html = html.replace(ori_url, qiniu_url)
if isinstance(item, list):
for index, item1 in enumerate(item):
if 'type' in item1.keys(
) and item1['type'] == 'image':
ori_url = item1['value']
qiniu_url = self._upload_qiniu(ori_url)
if qiniu_url:
content[key][idx][index][
'value'] = qiniu_url
update_flag = True
html = html.replace(ori_url, qiniu_url)
if update_flag:
mongo.select_collection('mongo_question_json')
json_effected = mongo.update_many({'question_id': question_id},
{'$set': {
'content': content
}})
mongo.select_collection('mongo_question_html')
html_effected = mongo.update_many({'question_id': question_id},
{'$set': {
'content': html
}})
LOG.info('mongo update successful json[%d] -- html[%d]' %
(json_effected, html_effected))
def post(self):
for i in range(1):
self.set_header("Access-Control-Allow-Origin", "*")
LOG.info('API IN[%s]' % (self.__class__.__name__))
LOG.info('PARAMETER IN[%s]' % self.request.arguments)
ret = {'code': '', 'message': ''}
essential_keys = set([
'json', 'html', 'topic', 'level', 'type', 'group', 'chapter',
'ref'
])
if Base.check_parameter(set(self.request.arguments.keys()),
essential_keys):
ret['code'] = 1
ret['message'] = '无效参数'
LOG.error('ERR[in parameter invalid]')
break
question_json = ''.join(self.request.arguments['json'])
question_html = ''.join(self.request.arguments['html'])
question_topic = ''.join(self.request.arguments['topic'])
question_level = ''.join(self.request.arguments['level'])
question_type = ''.join(self.request.arguments['type'])
question_group = ''.join(self.request.arguments['group'])
question_chapter = ''.join(self.request.arguments['chapter'])
ref = ''.join(self.request.arguments['ref'])
if Business.is_level(question_level) is False:
ret['code'] = 1
ret['message'] = '无效参数'
LOG.error('ERR[level is invalid]')
break
try:
#question_json = urllib.unquote(question_json)
#question_json = question_json.replace("'","\"")
encode_json = json.loads(question_json, encoding='utf-8')
#question_html = urllib.unquote(question_html)
#question_html = question_html.replace("'","\"")
encode_html = json.loads(question_html, encoding='utf-8')
LOG.info('Json Loads Successful')
answer_num = 0
if Base.empty(question_topic) and Base.empty(question_chapter):
ret['code'] = 1
ret['message'] = '无效参数'
LOG.error('ERR[topic and chapter empty]')
break
if Base.empty(question_group):
ret['code'] = 1
ret['message'] = '无效参数'
LOG.error('ERR[group empty]')
break
if Base.empty(question_topic) is False:
topic_list = question_topic.split(',')
for question_theme in topic_list:
if Business.is_topic(question_theme) is False:
ret['code'] = 1
ret['message'] = '无效参数'
LOG.error('ERR[topic %s invalid]' % question_theme)
break
type_name = Business.is_type(question_type)
if type_name is False:
ret['code'] = 1
ret['message'] = '无效参数'
LOG.error('ERR[type is invalid]')
break
option_num = 0
LOG.info('Json Parse Start')
if type_name == '选择题'.decode('utf-8'):
if 'answer' in encode_json.keys():
answer_num = len(encode_json['answer'])
option_num = len(encode_json['options'])
if type_name == '填空题'.decode('utf-8'):
if 'answer' in encode_json.keys():
answer_num = max([
int(group['index'])
for group in encode_json['answer']
])
LOG.info('Json Parse End')
if not Base.empty(question_chapter):
if Business.chapter_id_exist(question_chapter) is False:
ret['code'] = 1
ret['message'] = '无效参数'
LOG.error('ERR[seriess %s invalid]' % question_theme)
break
except (ValueError, KeyError, TypeError):
ret['code'] = 1
ret['message'] = '无效参数'
LOG.error('ERR[json format invalid]')
break
except CKException:
ret['code'] = 3
ret['message'] = '服务器错误'
LOG.error('ERR[mysql exception]')
break
key = question_topic + question_level + question_type + question_group
secret_key = hashlib.sha1(key).hexdigest()
qiniu = QiniuWrap()
json_key = 'tmp_' + secret_key + '.json'
if not qiniu.upload_data("temp", json_key, question_json):
ret['code'] = 4
ret['message'] = '服务器错误'
LOG.error('ERR[json upload qiniu exception]')
break
html_key = 'tmp_' + secret_key + '.html'
if not qiniu.upload_data("temp", html_key, question_html):
ret['code'] = 4
ret['message'] = '服务器错误'
LOG.error('ERR[html upload qiniu exception]')
break
configer = Configer()
remote_host = configer.get_configer('REMOTE', 'host')
remote_port = configer.get_configer('REMOTE', 'port')
remote_uri = configer.get_configer('REMOTE', 'uri')
remote_timeout = configer.get_configer('REMOTE', 'timeout')
remote_url = "http://%s:%s/%s" % (remote_host, remote_port,
remote_uri)
token = self.get_cookie("teacher_id")
LOG.info('TOKEN[%s]' % token)
if token is None:
ret['code'] = 6
ret['message'] = 'token失效'
LOG.error('ERROR[token empty]')
break
post_data = {'token': token}
client = httpclient.AsyncHTTPClient()
response = yield gen.Task(client.fetch,
remote_url,
request_timeout=int(remote_timeout),
method='POST',
body=urllib.urlencode(post_data))
#response = Http.post(remote_url,post_data)
if 200 == response.code:
encode_body = json.loads(response.body, encoding='utf-8')
if 0 == encode_body['code'] or 2 == encode_body['code']:
ret['code'] = 7
ret['message'] = 'token失效'
LOG.error('ERR[token not exist]')
break
if 1 == encode_body['code']:
subject_id = encode_body['subject_id']
grade_id = encode_body['grade_id']
system_id = encode_body['system_id']
org_type = encode_body['org_type']
if 0 != int(question_group):
if Business.group_id_exist(question_group,
system_id) is False:
ret['code'] = 8
ret['message'] = '无效参数'
LOG.error('ERROR[group not exist]')
break
db = Mysql()
question_sql = "insert into entity_question (difficulty,question_docx,html,upload_time,update_time,question_type,subject_id,new_format,upload_id,upload_src,question_group,grade_id,state,is_single,question_type_id,answer_num,count_ref,paper_year,parent_question_id,count_options) values (%(level)d,'%(json)s','%(html)s',now(),now(),'%(type)s',%(subject_id)d,1,%(upload_id)d,%(upload_src)d,%(question_group)d,%(grade_id)d,'ENABLED',1,%(question_type_id)d,%(answer_num)d,0,0,0,%(count_options)d);"
link_topic_sql = "insert into link_question_topic (question_id,topic_id) values (%(q_id)d,%(t_id)d);"
link_chapter_sql = "insert into link_question_chapter (question_id,chapter_id) values (%(q_id)d,%(c_id)d);"
try:
db.connect_master()
db.start_event()
question_res = db.exec_event(
question_sql,
level=int(question_level),
json=json_key,
html=html_key,
type=type_name,
subject_id=int(subject_id),
upload_id=int(system_id),
upload_src=int(org_type),
question_group=int(question_group),
grade_id=int(grade_id),
question_type_id=int(question_type),
answer_num=answer_num,
count_options=option_num)
question_sql = db.get_last_sql()
question_id = db.get_last_id()
LOG.info('RES[%s] - INS[%d]' %
(question_res, question_id))
if Base.empty(question_topic) is False:
topic_list = question_topic.split(',')
for question_theme in topic_list:
topic_res = db.exec_event(
link_topic_sql,
q_id=int(question_id),
t_id=int(question_theme))
topic_sql = db.get_last_sql()
topic_id = db.get_last_id()
LOG.info('RES[%s] - INS[%d]' %
(topic_res, topic_id))
if not Base.empty(question_chapter):
chapter_res = db.exec_event(
link_chapter_sql,
q_id=int(question_id),
c_id=int(question_chapter))
chapter_sql = db.get_last_sql()
chapter_id = db.get_last_id()
LOG.info('RES[%s] - INS[%d]' %
(chapter_res, chapter_id))
except DBException as e:
db.rollback()
db.end_event()
ret['code'] = 3
ret['message'] = '服务器错误'
LOG.error('ERR[insert mysql error]')
break
else:
ret['code'] = 3
ret['message'] = '服务器错误'
LOG.error('ERROR[remote error]')
break
mongo = Mongo()
try:
mongo.connect('resource')
mongo.select_collection('mongo_question_json')
json_id = mongo.insert_one({
"content": encode_json,
"question_id": question_id
})
LOG.info(
'MONGO[insert json] - DATA[%s] - INS[%s] - Question Id[%d]'
% (json.dumps(encode_json), json_id, question_id))
mongo.select_collection('mongo_question_html')
html_id = mongo.insert_one({
"content": encode_html,
"question_id": question_id
})
LOG.info(
'MONGO[insert html] - DATA[%s] - INS[%s] - Question Id[%d]'
% (json.dumps(encode_html), html_id, question_id))
except DBException as e:
db.rollback()
db.end_event()
ret['code'] = 3
ret['message'] = '服务器错误'
LOG.error('ERR[mongo exception]')
break
db.end_event()
if int(ref):
doit = Doit()
doit.local_img(str(question_id))
LOG.info('Local Img [%s]' % str(question_id))
ret['code'] = 0
ret['message'] = 'success'
ret['id'] = question_id
LOG.info('PARAMETER OUT[%s]' % ret)
LOG.info('API OUT[%s]' % (self.__class__.__name__))
self.write(json.dumps(ret))
self.finish()
def post(self):
self.set_header("Access-Control-Allow-Origin", "*")
enter_func(self)
if not set([
'id', 'json', 'html', 'topic', 'level', 'type', 'group',
'chapter'
]).issubset(self.request.arguments.keys()):
return leave_func(self, 1)
theme = self.request.arguments['topic'][0]
type_id = self.request.arguments['type'][0]
level_id = self.request.arguments['level'][0]
group_id = self.request.arguments['group'][0]
chapter_id = self.request.arguments['chapter'][0]
question_id = self.request.arguments['id'][0]
question_json = self.request.arguments['json'][0]
question_html = self.request.arguments['html'][0]
try:
if not (type_id.isdigit() and int(type_id) and level_id.isdigit()
and int(level_id) and group_id.isdigit()
and question_id.isdigit() and int(question_id) and
theme + chapter_id and question_json and question_html):
return leave_func(self, 1)
if chapter_id and not chapter_id.isdigit():
return leave_func(self, 1)
if Business.is_level(level_id) is False:
LOG.error('invalid level_id[%s]' % level_id)
return leave_func(self, 1)
try:
#question_json = urllib.unquote(question_json)
encode_json = {}
encode_json['content'] = json.loads(question_json,
encoding='utf-8')
#question_html = urllib.unquote(question_html)
encode_html = {}
encode_html['content'] = json.loads(question_html,
encoding='utf-8')
except:
traceback.print_exc()
LOG.error(sys.exc_info())
return leave_func(self, 100)
sql_list = []
if chapter_id:
sql_list.append(
'INSERT INTO link_question_chapter(question_id, chapter_id) VALUE(%s, %s) ON DUPLICATE KEY UPDATE chapter_id = %s'
% (question_id, chapter_id, chapter_id)) # 生成更新章节关联信息的SQL
if theme: # 主题
sql_list.append(
'DELETE FROM link_question_topic WHERE question_id = %s' %
question_id) # 生成删除原有主题关联的SQL
for theme_id in theme.split(','): # 将传入的主题号按逗号切割
if Business.is_topic(theme_id) is False: # 判断主题号是否存在
LOG.error('invalid theme_id[%s]' % theme_id)
return leave_func(self, 1)
sql_list.append(
'INSERT INTO link_question_topic (question_id, topic_id) VALUES (%s, %s)'
% (question_id, theme_id)) # 生成将新主题关联插库的SQL
question_type = Business.is_type(type_id)
if question_type is False: # 判断题目类型是否存在
LOG.error('invalid type_id[%s]' % type_id)
return leave_func(self, 1)
answer_num = 0
if 'answer' in encode_json['content'].keys():
if type_id == '1':
answer_num = len(encode_json['content']['answer'])
elif type_id == '2':
answer_num = len([
int(answer_group['index'])
for answer_group in encode_json['content']['answer']
])
sql_list.append(
'UPDATE entity_question SET difficulty = %s, update_time = now(), question_type = "%s", question_group = %s, answer_num = %s WHERE id = %s'
% (level_id, question_type, group_id, answer_num,
question_id)) # 生成更新题目属性的SQL
mysql_handle = Mysql().get_handle()
mysql_cursor = mysql_handle.cursor(MySQLdb.cursors.DictCursor)
mysql_cursor.execute(
'SELECT question_docx, html FROM entity_question WHERE id = %s'
% question_id) # 通过题目ID查询存储的json/html文件名
result = mysql_cursor.fetchall()
if not result:
LOG.error('invalid question_id[%s]' % question_id)
return leave_func(self, 1)
qiniu = QiniuWrap()
mongo = Mongo()
mongo.connect('resource')
if result[0]['question_docx'] and '.json' in result[0][
'question_docx']:
json_name = result[0]['question_docx']
# 将七牛上的json文件删除后重新上传
qiniu.bucket.delete("temp", json_name)
qiniu.upload_data("temp", json_name, question_json)
# 将MongoDB中的json文件删除后重新上传
mongo.select_collection('mongo_question_json')
LOG.debug(
'resource.mongo_question_json.remove( { question_id: %s } )'
% question_id)
mongo.remove({"question_id": int(question_id)})
encode_json['question_id'] = int(question_id)
LOG.debug('resource.mongo_question_json.insert_one( %s )' %
encode_json)
mongo.insert_one(encode_json)
if result[0]['html'] and '.html' in result[0]['html']:
html_name = result[0]['html']
# 将七牛上的html文件删除后重新上传
qiniu.bucket.delete("temp", html_name)
qiniu.upload_data("temp", html_name, question_html)
# 将MongoDB中的html文件删除后重新上传
mongo.select_collection('mongo_question_html')
LOG.debug(
'resource.mongo_question_html.remove( { question_id: %s } )'
% question_id)
mongo.remove({"question_id": int(question_id)})
encode_html['question_id'] = int(question_id)
LOG.debug('resource.mongo_question_html.insert_one( %s )' %
encode_html)
mongo.insert_one(encode_html)
for sql in sql_list:
LOG.info(sql)
mysql_cursor.execute(sql)
mysql_handle.commit()
mysql_cursor.close()
mysql_handle.close()
leave_func(self, 0)
self.write(error_process(0))
except Exception, e:
LOG.error(e)
return leave_func(self, 100)
from mongo import Mongo
from mqtt import MQTT
from signal import pause
mongo = Mongo()
mqtt = MQTT(mongo)
mongo.connect()
mqtt.run()
try:
pause()
except KeyboardInterrupt:
pass
mqtt.stop()
mongo.disconnect()
from mongo import Mongo
from matcher import Matcher
METHOD = "ML_direct_fb" # edit this depending on your need
mon = Mongo()
mon.connect()
###############
if METHOD == "getuser_fb":
(f, fd) = mon.getFacebookUser('MayorMurielBowser', returnDoc=True)
elif METHOD == "getuser_tw":
(t, td) = mon.getTwitterUser('PerlmanOfficial', returnDoc=True)
else:
matcher = Matcher(mon) # note that this takes a bit of time to load
if METHOD == "direct_fb":
match = matcher.findMatchForFacebookUser('Itzhakperlmanofficial',
useML=False)
elif METHOD == "direct_tw":
match = matcher.findMatchForTwitterUser('PerlmanOfficial', useML=False)
elif METHOD == "indirect_fb":
match = matcher.findIndirectMatchForFacebookUser(
'Itzhakperlmanofficial', useML=False)
elif METHOD == "indirect_tw":
match = matcher.findIndirectMatchForTwitterUser('PerlmanOfficial',
useML=False)
def post(self):
self.set_header("Access-Control-Allow-Origin", "*")
enter_func(self)
if not set(['id', 'json', 'html', 'topic', 'level', 'type', 'group', 'chapter']).issubset(self.request.arguments.keys()):
return leave_func(self, 1)
theme = self.request.arguments['topic'][0]
type_id = self.request.arguments['type'][0]
level_id = self.request.arguments['level'][0]
group_id = self.request.arguments['group'][0]
chapter_id = self.request.arguments['chapter'][0]
question_id = self.request.arguments['id'][0]
question_json = self.request.arguments['json'][0]
question_html = self.request.arguments['html'][0]
try:
if not (type_id.isdigit() and int(type_id) and level_id.isdigit() and int(level_id) and group_id.isdigit() and question_id.isdigit() and int(question_id) and theme + chapter_id and question_json and question_html):
return leave_func(self, 1)
if chapter_id and not chapter_id.isdigit():
return leave_func(self, 1)
if Business.is_level(level_id) is False:
LOG.error('invalid level_id[%s]' % level_id)
return leave_func(self, 1)
try:
#question_json = urllib.unquote(question_json)
encode_json = {}
encode_json['content'] = json.loads(question_json, encoding = 'utf-8')
#question_html = urllib.unquote(question_html)
encode_html = {}
encode_html['content'] = json.loads(question_html, encoding = 'utf-8')
except:
traceback.print_exc()
LOG.error(sys.exc_info())
return leave_func(self, 100)
sql_list = []
if chapter_id:
sql_list.append('INSERT INTO link_question_chapter(question_id, chapter_id) VALUE(%s, %s) ON DUPLICATE KEY UPDATE chapter_id = %s' % (question_id, chapter_id, chapter_id)) # 生成更新章节关联信息的SQL
if theme: # 主题
sql_list.append('DELETE FROM link_question_topic WHERE question_id = %s' % question_id) # 生成删除原有主题关联的SQL
for theme_id in theme.split(','): # 将传入的主题号按逗号切割
if Business.is_topic(theme_id) is False: # 判断主题号是否存在
LOG.error('invalid theme_id[%s]' % theme_id)
return leave_func(self, 1)
sql_list.append('INSERT INTO link_question_topic (question_id, topic_id) VALUES (%s, %s)' % (question_id, theme_id)) # 生成将新主题关联插库的SQL
question_type = Business.is_type(type_id)
if question_type is False: # 判断题目类型是否存在
LOG.error('invalid type_id[%s]' % type_id)
return leave_func(self, 1)
answer_num = 0
if 'answer' in encode_json['content'].keys():
if type_id == '1':
answer_num = len(encode_json['content']['answer'])
elif type_id == '2':
answer_num = len([int(answer_group['index']) for answer_group in encode_json['content']['answer']])
sql_list.append('UPDATE entity_question SET difficulty = %s, update_time = now(), question_type = "%s", question_group = %s, answer_num = %s WHERE id = %s' % (level_id, question_type, group_id, answer_num, question_id)) # 生成更新题目属性的SQL
mysql_handle = Mysql().get_handle()
mysql_cursor = mysql_handle.cursor(MySQLdb.cursors.DictCursor)
mysql_cursor.execute('SELECT question_docx, html FROM entity_question WHERE id = %s' % question_id) # 通过题目ID查询存储的json/html文件名
result = mysql_cursor.fetchall()
if not result:
LOG.error('invalid question_id[%s]' % question_id)
return leave_func(self, 1)
qiniu = QiniuWrap()
mongo = Mongo()
mongo.connect('resource')
if result[0]['question_docx'] and '.json' in result[0]['question_docx']:
json_name = result[0]['question_docx']
# 将七牛上的json文件删除后重新上传
qiniu.bucket.delete("temp", json_name)
qiniu.upload_data("temp", json_name, question_json)
# 将MongoDB中的json文件删除后重新上传
mongo.select_collection('mongo_question_json')
LOG.debug('resource.mongo_question_json.remove( { question_id: %s } )' % question_id)
mongo.remove( { "question_id" : int(question_id) } )
encode_json['question_id'] = int(question_id)
LOG.debug('resource.mongo_question_json.insert_one( %s )' % encode_json)
mongo.insert_one(encode_json)
if result[0]['html'] and '.html' in result[0]['html']:
html_name = result[0]['html']
# 将七牛上的html文件删除后重新上传
qiniu.bucket.delete("temp", html_name)
qiniu.upload_data("temp", html_name, question_html)
# 将MongoDB中的html文件删除后重新上传
mongo.select_collection('mongo_question_html')
LOG.debug('resource.mongo_question_html.remove( { question_id: %s } )' % question_id)
mongo.remove( { "question_id" : int(question_id) } )
encode_html['question_id'] = int(question_id)
LOG.debug('resource.mongo_question_html.insert_one( %s )' % encode_html)
mongo.insert_one(encode_html)
for sql in sql_list:
LOG.info(sql)
mysql_cursor.execute(sql)
mysql_handle.commit()
mysql_cursor.close()
mysql_handle.close()
leave_func(self, 0)
self.write(error_process(0))
except Exception, e:
LOG.error(e)
return leave_func(self, 100)