def load_questions(): cnf = config.MYSQL db = Mysql(*cnf) sql = "SELECT id, question, answer, store_id from question" r = db.execute(sql) result = list(r.fetchall()) db.close() return result
def get(self): enter_func(self) if not set(['id']).issubset(self.request.arguments.keys()): return leave_func(self, 1) id = int(self.request.arguments['id'][0]) mysql_handle = Mysql().get_handle() mysql_cursor = mysql_handle.cursor(MySQLdb.cursors.DictCursor) sql = 'UPDATE entity_question SET state = "DISABLED" WHERE id = %d' % id LOG.info('mysql> %s' % sql) mysql_cursor.execute(sql) mysql_handle.commit() mysql_cursor.close() mysql_handle.close() leave_func(self, 0) return self.write(error_process(0))
class MySQLDB(object): def __init__(self, host, db, user, password): self.connection = None self.host, self.db, self.user, self.password = host, db, user, password self.connect() def connect(self): if not self.connection: self.connection = Mysql(host=self.host, db=self.db, user=self.user, password=self.password) self.connection.connect() return self.connection def insertSQL(self, table, columnValueDict): sql = 'INSERT INTO %s(%s) VALUES(%s)' % (table, ",".join( columnValueDict.keys()), ",".join( self.to_str(columnValueDict.values()))) return self.connection.execute(sql) def insert(self, table, columns, data): sql = 'INSERT INTO %s(%s) VALUES(%s)' % (table, ",".join(columns), ",".join(self.to_str(data))) return self.connection.execute(sql) def insertSQLIgnore(self, table, columnValueDict): sql = 'INSERT IGNORE INTO %s(%s) VALUES(%s)' % (table, ",".join( columnValueDict.keys()), ",".join( self.to_str(columnValueDict.values()))) return self.connection.execute(sql) def execute(self, sql): return self.connection.execute(sql) def commit(self): self.connection.commit() def close(self): self.connection.commit() self.connection.close() self.connection = None def to_str(self, arr): a = arr[:] for i in range(len(arr)): a[i] = "'%s'" % arr[i] return a
class MySQLDB(object): def __init__(self, host, db, user, password): self.connection = None self.host, self.db, self.user, self.password = host, db, user, password self.connect() def connect(self): if not self.connection: self.connection = Mysql(host=self.host, db=self.db, user=self.user, password=self.password) self.connection.connect() return self.connection def insertSQL(self, table, columnValueDict): sql = 'INSERT INTO %s(%s) VALUES(%s)' % (table, ",".join(columnValueDict.keys()), ",".join(self.to_str(columnValueDict.values()))) return self.connection.execute(sql) def insert(self, table, columns, data): sql = 'INSERT INTO %s(%s) VALUES(%s)' % (table, ",".join(columns), ",".join(self.to_str(data))) return self.connection.execute(sql) def insertSQLIgnore(self, table, columnValueDict): sql = 'INSERT IGNORE INTO %s(%s) VALUES(%s)' % (table, ",".join(columnValueDict.keys()), ",".join(self.to_str(columnValueDict.values()))) return self.connection.execute(sql) def execute(self, sql): return self.connection.execute(sql) def commit(self): self.connection.commit() def close(self): self.connection.commit() self.connection.close() self.connection = None def to_str(self, arr): a = arr[:] for i in range(len(arr)): a[i] = "'%s'" % arr[i] return a
def parseListHtml(page, titleindex): next_page = {'page': page, 'title': titleindex} common.save_now_page(next_page) mysql = Mysql() s = '' if page > 1: s = '_' + repr(page) print(url.format(titles[titleindex], s)) try: response = requests.get(url.format(titles[titleindex], s), headers=headers, timeout=10) response.encoding = 'gb2312' if response.status_code == 200: soup = BeautifulSoup(response.text, 'html.parser') re_coms = soup.find_all('ul', attrs={'class': 'recom_list'}) articles = [] for re_com in re_coms: article = Article(re_com.a.string, re_com.find('span', attrs={'class': 'gd1'}).a.attrs['href']) article.author = 'OK学车' article.contentHead = parseContentHead(re_com.find('li', attrs={'class': 'recom_nr'}).text) article.type = types[titles[titleindex]] articles.append(article) parseArticle(articles) # 保存到数据库 mysql.insert_array(articles) mysql.close() # common.save_file(titles[titleIndex], '第{0}页'.format(page), repr(common.convert_to_dicts(articles))) sleep_time = random.randint(5, 10) print('休息', sleep_time, 's后再获取') time.sleep(sleep_time) parseListHtml(page + 1, titleindex) else: mysql.close() if titleindex + 1 < len(titles): parseListHtml(1, titleindex + 1) except Exception as e: print(traceback.format_exc()) print('网页获取失败:', e) mysql.close() sleep_time = random.randint(1, 5) print(repr(sleep_time), 's后重新获取') time.sleep(sleep_time) parseListHtml(page + 1, titleindex)
return datalist for i in getFileData("""%s/%s_StatusCode.txt"""%(Dir,db_yesterday)): sql = """insert into StatisticsStatus(StatisticsDate,StatusCode,StatusNum) values('%s',%s,%s)"""%(db_yesterday,i[1],i[0]) mysql.insert(sql) for i in getFileData("""%s/%s.MMGlobal.txt"""%(Dir,db_yesterday)): sql = """insert into StatisticsHour(StaticsticsDate,Hour,Channel,ReqNum,ResTime) values('%s',%s,'%s',%s,%s)"""%(db_yesterday,i[1],i[0],i[2],i[3]) mysql.insert(sql) for i in getFileData("""%s/%s_time_and_revalue.txt"""%(Dir,db_yesterday)): sql = """ insert into PageStatistics( StaticsticsDate, Channel, RequestNum, TimeAvg, LessOneSecNum, LessOneSecPercent, BetweenOneThreeSecNum, BetweenOneThreeSecPercent, GreaterThreeNum, GreaterThreeNumPercent) values ('%s','%s',%s,'%s',%s,'%s',%s,'%s',%s,'%s') """%(db_yesterday,i[0],i[1],i[2],i[3],i[4],i[5],i[6],i[7],i[8]) mysql.insert(sql) mysql.close()
def post(self): self.set_header("Access-Control-Allow-Origin", "*") enter_func(self) if not set([ 'id', 'json', 'html', 'topic', 'level', 'type', 'group', 'chapter' ]).issubset(self.request.arguments.keys()): return leave_func(self, 1) theme = self.request.arguments['topic'][0] type_id = self.request.arguments['type'][0] level_id = self.request.arguments['level'][0] group_id = self.request.arguments['group'][0] chapter_id = self.request.arguments['chapter'][0] question_id = self.request.arguments['id'][0] question_json = self.request.arguments['json'][0] question_html = self.request.arguments['html'][0] try: if not (type_id.isdigit() and int(type_id) and level_id.isdigit() and int(level_id) and group_id.isdigit() and question_id.isdigit() and int(question_id) and theme + chapter_id and question_json and question_html): return leave_func(self, 1) if chapter_id and not chapter_id.isdigit(): return leave_func(self, 1) if Business.is_level(level_id) is False: LOG.error('invalid level_id[%s]' % level_id) return leave_func(self, 1) try: #question_json = urllib.unquote(question_json) encode_json = {} encode_json['content'] = json.loads(question_json, encoding='utf-8') #question_html = urllib.unquote(question_html) encode_html = {} encode_html['content'] = json.loads(question_html, encoding='utf-8') except: traceback.print_exc() LOG.error(sys.exc_info()) return leave_func(self, 100) sql_list = [] if chapter_id: sql_list.append( 'INSERT INTO link_question_chapter(question_id, chapter_id) VALUE(%s, %s) ON DUPLICATE KEY UPDATE chapter_id = %s' % (question_id, chapter_id, chapter_id)) # 生成更新章节关联信息的SQL if theme: # 主题 sql_list.append( 'DELETE FROM link_question_topic WHERE question_id = %s' % question_id) # 生成删除原有主题关联的SQL for theme_id in theme.split(','): # 将传入的主题号按逗号切割 if Business.is_topic(theme_id) is False: # 判断主题号是否存在 LOG.error('invalid theme_id[%s]' % theme_id) return leave_func(self, 1) sql_list.append( 'INSERT INTO link_question_topic (question_id, topic_id) VALUES (%s, %s)' % (question_id, theme_id)) # 生成将新主题关联插库的SQL question_type = Business.is_type(type_id) if question_type is False: # 判断题目类型是否存在 LOG.error('invalid type_id[%s]' % type_id) return leave_func(self, 1) answer_num = 0 if 'answer' in encode_json['content'].keys(): if type_id == '1': answer_num = len(encode_json['content']['answer']) elif type_id == '2': answer_num = len([ int(answer_group['index']) for answer_group in encode_json['content']['answer'] ]) sql_list.append( 'UPDATE entity_question SET difficulty = %s, update_time = now(), question_type = "%s", question_group = %s, answer_num = %s WHERE id = %s' % (level_id, question_type, group_id, answer_num, question_id)) # 生成更新题目属性的SQL mysql_handle = Mysql().get_handle() mysql_cursor = mysql_handle.cursor(MySQLdb.cursors.DictCursor) mysql_cursor.execute( 'SELECT question_docx, html FROM entity_question WHERE id = %s' % question_id) # 通过题目ID查询存储的json/html文件名 result = mysql_cursor.fetchall() if not result: LOG.error('invalid question_id[%s]' % question_id) return leave_func(self, 1) qiniu = QiniuWrap() mongo = Mongo() mongo.connect('resource') if result[0]['question_docx'] and '.json' in result[0][ 'question_docx']: json_name = result[0]['question_docx'] # 将七牛上的json文件删除后重新上传 qiniu.bucket.delete("temp", json_name) qiniu.upload_data("temp", json_name, question_json) # 将MongoDB中的json文件删除后重新上传 mongo.select_collection('mongo_question_json') LOG.debug( 'resource.mongo_question_json.remove( { question_id: %s } )' % question_id) mongo.remove({"question_id": int(question_id)}) encode_json['question_id'] = int(question_id) LOG.debug('resource.mongo_question_json.insert_one( %s )' % encode_json) mongo.insert_one(encode_json) if result[0]['html'] and '.html' in result[0]['html']: html_name = result[0]['html'] # 将七牛上的html文件删除后重新上传 qiniu.bucket.delete("temp", html_name) qiniu.upload_data("temp", html_name, question_html) # 将MongoDB中的html文件删除后重新上传 mongo.select_collection('mongo_question_html') LOG.debug( 'resource.mongo_question_html.remove( { question_id: %s } )' % question_id) mongo.remove({"question_id": int(question_id)}) encode_html['question_id'] = int(question_id) LOG.debug('resource.mongo_question_html.insert_one( %s )' % encode_html) mongo.insert_one(encode_html) for sql in sql_list: LOG.info(sql) mysql_cursor.execute(sql) mysql_handle.commit() mysql_cursor.close() mysql_handle.close() leave_func(self, 0) self.write(error_process(0)) except Exception, e: LOG.error(e) return leave_func(self, 100)
def get(self): self.set_header("Access-Control-Allow-Origin", "*") enter_func(self) if not set(['id']).issubset(self.request.arguments.keys()): return leave_func(self, 1) question_id = self.request.arguments['id'][0] if not question_id.isdigit(): return leave_func(self, 1) try: mysql = Mysql().get_handle() cursor = mysql.cursor(MySQLdb.cursors.DictCursor) sql = 'SELECT question_type, subject_id, difficulty, question_group FROM entity_question WHERE id = %s' % question_id LOG.info('mysql> %s' % sql) cursor.execute(sql) result = cursor.fetchall() if not result: LOG.error('question_id[%s] nonexistent!' % question_id) return leave_func(self, 2) level_id = result[0]['difficulty'] group_id = result[0]['question_group'] question_type = result[0]['question_type'] subject_id = result[0]['subject_id'] # 获取题目类型 sql = 'SELECT type_id id, name FROM entity_question_type WHERE name = "%s"' % question_type LOG.info('mysql> %s' % sql) cursor.execute(sql) ret = cursor.fetchall() if not ret: LOG.error('invalid question_type[%s] of question_id[%s]!' % (question_type, question_id)) return leave_func(self, 100) question_type = ret[0] # 获取主题 sql = 'SELECT id, SUBSTRING_INDEX(name, "\n", 1) name FROM entity_topic WHERE id IN (SELECT topic_id FROM link_question_topic WHERE question_id = %s)' % question_id LOG.info('mysql> %s' % repr(sql)[1:-1]) cursor.execute(sql) theme_list = list(cursor.fetchall()) # # 获取专题 # sql = 'select id, substring_index(name, "\n", 1) name from entity_seriess where id in (select series_id from link_question_series where question_id = %s)' % question_id # LOG.info('mysql> %s' % repr(sql)[1:-1]) # cursor.execute(sql) # special_list = list(cursor.fetchall()) mongo = Mongo().get_handle() json_body = mongo.resource.mongo_question_json.find_one( {'question_id': int(question_id)}) if not json_body: LOG.error( 'json body of question_id[%s] is nonexistent in MongoDB.' % question_id) return leave_func(self, 2) if 'content' in json_body: json_body = json_body['content'] else: json_body = {} html_body = mongo.resource.mongo_question_html.find_one( {'question_id': int(question_id)}) if not html_body: LOG.error( 'html body of question_id[%s] is nonexistent in MongoDB.' % question_id) return leave_func(self, 2) if 'content' in html_body: html_body = html_body['content'] else: html_body = {} chapter_info = [] sql = 'SELECT chapter_id FROM link_question_chapter WHERE question_id = %s' % question_id LOG.info('mysql> %s' % sql) cursor.execute(sql) ret = cursor.fetchall() if ret: chapter_id = ret[0]['chapter_id'] sql = 'SELECT id, level, parent_id, REPLACE(prefix_name, "\r\n", "") prefix_name, REPLACE(name, "\r\n", "") name FROM entity_teaching_chapter WHERE id = %s' % chapter_id LOG.info('mysql> %s' % repr(sql)[1:-1]) cursor.execute(sql) ret = cursor.fetchall() if not ret: LOG.error('chapter_id[%s] nonexistent!' % chapter_id) else: chapter_id = ret[0]['id'] parent_id = ret[0]['parent_id'] prefix_name = ret[0]['prefix_name'] name = ret[0]['name'] level = ret[0]['level'] chapter_info.insert(0, { 'id': chapter_id, 'prefix': prefix_name, 'name': name }) for i in range(int(level) - 1): sql = 'SELECT id, level, parent_id, REPLACE(prefix_name, "\r\n", "") prefix_name, REPLACE(name, "\r\n", "") name FROM entity_teaching_chapter WHERE id = %s' % parent_id LOG.info('mysql> %s' % repr(sql)[1:-1]) cursor.execute(sql) ret = cursor.fetchall() if not ret: break chapter_id = ret[0]['id'] parent_id = ret[0]['parent_id'] prefix_name = ret[0]['prefix_name'] name = ret[0]['name'] level = ret[0]['level'] chapter_info.insert(0, { 'id': chapter_id, 'prefix': prefix_name, 'name': name }) result = error_process(0) result['json'] = json_body result['html'] = html_body result['type'] = question_type result['topic'] = theme_list result['level'] = level_id result['group'] = group_id result['chapter_info'] = chapter_info mongo.close() cursor.close() mysql.close() leave_func(self, 0) return self.write(json.dumps(result, ensure_ascii=False)) except Exception, e: LOG.error(e) return leave_func(self, 100)
def post(self): self.set_header("Access-Control-Allow-Origin", "*") enter_func(self) if not set(['id', 'json', 'html', 'topic', 'level', 'type', 'group', 'chapter']).issubset(self.request.arguments.keys()): return leave_func(self, 1) theme = self.request.arguments['topic'][0] type_id = self.request.arguments['type'][0] level_id = self.request.arguments['level'][0] group_id = self.request.arguments['group'][0] chapter_id = self.request.arguments['chapter'][0] question_id = self.request.arguments['id'][0] question_json = self.request.arguments['json'][0] question_html = self.request.arguments['html'][0] try: if not (type_id.isdigit() and int(type_id) and level_id.isdigit() and int(level_id) and group_id.isdigit() and question_id.isdigit() and int(question_id) and theme + chapter_id and question_json and question_html): return leave_func(self, 1) if chapter_id and not chapter_id.isdigit(): return leave_func(self, 1) if Business.is_level(level_id) is False: LOG.error('invalid level_id[%s]' % level_id) return leave_func(self, 1) try: #question_json = urllib.unquote(question_json) encode_json = {} encode_json['content'] = json.loads(question_json, encoding = 'utf-8') #question_html = urllib.unquote(question_html) encode_html = {} encode_html['content'] = json.loads(question_html, encoding = 'utf-8') except: traceback.print_exc() LOG.error(sys.exc_info()) return leave_func(self, 100) sql_list = [] if chapter_id: sql_list.append('INSERT INTO link_question_chapter(question_id, chapter_id) VALUE(%s, %s) ON DUPLICATE KEY UPDATE chapter_id = %s' % (question_id, chapter_id, chapter_id)) # 生成更新章节关联信息的SQL if theme: # 主题 sql_list.append('DELETE FROM link_question_topic WHERE question_id = %s' % question_id) # 生成删除原有主题关联的SQL for theme_id in theme.split(','): # 将传入的主题号按逗号切割 if Business.is_topic(theme_id) is False: # 判断主题号是否存在 LOG.error('invalid theme_id[%s]' % theme_id) return leave_func(self, 1) sql_list.append('INSERT INTO link_question_topic (question_id, topic_id) VALUES (%s, %s)' % (question_id, theme_id)) # 生成将新主题关联插库的SQL question_type = Business.is_type(type_id) if question_type is False: # 判断题目类型是否存在 LOG.error('invalid type_id[%s]' % type_id) return leave_func(self, 1) answer_num = 0 if 'answer' in encode_json['content'].keys(): if type_id == '1': answer_num = len(encode_json['content']['answer']) elif type_id == '2': answer_num = len([int(answer_group['index']) for answer_group in encode_json['content']['answer']]) sql_list.append('UPDATE entity_question SET difficulty = %s, update_time = now(), question_type = "%s", question_group = %s, answer_num = %s WHERE id = %s' % (level_id, question_type, group_id, answer_num, question_id)) # 生成更新题目属性的SQL mysql_handle = Mysql().get_handle() mysql_cursor = mysql_handle.cursor(MySQLdb.cursors.DictCursor) mysql_cursor.execute('SELECT question_docx, html FROM entity_question WHERE id = %s' % question_id) # 通过题目ID查询存储的json/html文件名 result = mysql_cursor.fetchall() if not result: LOG.error('invalid question_id[%s]' % question_id) return leave_func(self, 1) qiniu = QiniuWrap() mongo = Mongo() mongo.connect('resource') if result[0]['question_docx'] and '.json' in result[0]['question_docx']: json_name = result[0]['question_docx'] # 将七牛上的json文件删除后重新上传 qiniu.bucket.delete("temp", json_name) qiniu.upload_data("temp", json_name, question_json) # 将MongoDB中的json文件删除后重新上传 mongo.select_collection('mongo_question_json') LOG.debug('resource.mongo_question_json.remove( { question_id: %s } )' % question_id) mongo.remove( { "question_id" : int(question_id) } ) encode_json['question_id'] = int(question_id) LOG.debug('resource.mongo_question_json.insert_one( %s )' % encode_json) mongo.insert_one(encode_json) if result[0]['html'] and '.html' in result[0]['html']: html_name = result[0]['html'] # 将七牛上的html文件删除后重新上传 qiniu.bucket.delete("temp", html_name) qiniu.upload_data("temp", html_name, question_html) # 将MongoDB中的html文件删除后重新上传 mongo.select_collection('mongo_question_html') LOG.debug('resource.mongo_question_html.remove( { question_id: %s } )' % question_id) mongo.remove( { "question_id" : int(question_id) } ) encode_html['question_id'] = int(question_id) LOG.debug('resource.mongo_question_html.insert_one( %s )' % encode_html) mongo.insert_one(encode_html) for sql in sql_list: LOG.info(sql) mysql_cursor.execute(sql) mysql_handle.commit() mysql_cursor.close() mysql_handle.close() leave_func(self, 0) self.write(error_process(0)) except Exception, e: LOG.error(e) return leave_func(self, 100)
def get(self): self.set_header("Access-Control-Allow-Origin", "*") enter_func(self) if not set(['id']).issubset(self.request.arguments.keys()): return leave_func(self, 1) question_id = self.request.arguments['id'][0] if not question_id.isdigit(): return leave_func(self, 1) try: mysql = Mysql().get_handle() cursor = mysql.cursor(MySQLdb.cursors.DictCursor) sql = 'SELECT question_type, subject_id, difficulty, question_group FROM entity_question WHERE id = %s' % question_id LOG.info('mysql> %s' % sql) cursor.execute(sql) result = cursor.fetchall() if not result: LOG.error('question_id[%s] nonexistent!' % question_id) return leave_func(self, 2) level_id = result[0]['difficulty'] group_id = result[0]['question_group'] question_type = result[0]['question_type'] subject_id = result[0]['subject_id'] # 获取题目类型 sql = 'SELECT type_id id, name FROM entity_question_type WHERE name = "%s"' % question_type LOG.info('mysql> %s' % sql) cursor.execute(sql) ret = cursor.fetchall() if not ret: LOG.error('invalid question_type[%s] of question_id[%s]!' % (question_type, question_id)) return leave_func(self, 100) question_type = ret[0] # 获取主题 sql = 'SELECT id, SUBSTRING_INDEX(name, "\n", 1) name FROM entity_topic WHERE id IN (SELECT topic_id FROM link_question_topic WHERE question_id = %s)' % question_id LOG.info('mysql> %s' % repr(sql)[1:-1]) cursor.execute(sql) theme_list = list(cursor.fetchall()) # # 获取专题 # sql = 'select id, substring_index(name, "\n", 1) name from entity_seriess where id in (select series_id from link_question_series where question_id = %s)' % question_id # LOG.info('mysql> %s' % repr(sql)[1:-1]) # cursor.execute(sql) # special_list = list(cursor.fetchall()) mongo = Mongo().get_handle() json_body = mongo.resource.mongo_question_json.find_one( { 'question_id': int(question_id) } ) if not json_body: LOG.error('json body of question_id[%s] is nonexistent in MongoDB.' % question_id) return leave_func(self, 2) if 'content' in json_body: json_body = json_body['content'] else: json_body = {} html_body = mongo.resource.mongo_question_html.find_one( { 'question_id': int(question_id) } ) if not html_body: LOG.error('html body of question_id[%s] is nonexistent in MongoDB.' % question_id) return leave_func(self, 2) if 'content' in html_body: html_body = html_body['content'] else: html_body = {} chapter_info = [] sql = 'SELECT chapter_id FROM link_question_chapter WHERE question_id = %s' % question_id LOG.info('mysql> %s' % sql) cursor.execute(sql) ret = cursor.fetchall() if ret: chapter_id = ret[0]['chapter_id'] sql = 'SELECT id, level, parent_id, REPLACE(prefix_name, "\r\n", "") prefix_name, REPLACE(name, "\r\n", "") name FROM entity_teaching_chapter WHERE id = %s' % chapter_id LOG.info('mysql> %s' % repr(sql)[1:-1]) cursor.execute(sql) ret = cursor.fetchall() if not ret: LOG.error('chapter_id[%s] nonexistent!' % chapter_id) else: chapter_id = ret[0]['id'] parent_id = ret[0]['parent_id'] prefix_name = ret[0]['prefix_name'] name = ret[0]['name'] level = ret[0]['level'] chapter_info.insert(0, { 'id': chapter_id, 'prefix': prefix_name, 'name': name }) for i in range(int(level) - 1): sql = 'SELECT id, level, parent_id, REPLACE(prefix_name, "\r\n", "") prefix_name, REPLACE(name, "\r\n", "") name FROM entity_teaching_chapter WHERE id = %s' % parent_id LOG.info('mysql> %s' % repr(sql)[1:-1]) cursor.execute(sql) ret = cursor.fetchall() if not ret: break chapter_id = ret[0]['id'] parent_id = ret[0]['parent_id'] prefix_name = ret[0]['prefix_name'] name = ret[0]['name'] level = ret[0]['level'] chapter_info.insert(0, { 'id': chapter_id, 'prefix': prefix_name, 'name': name }) result = error_process(0) result['json'] = json_body result['html'] = html_body result['type'] = question_type result['topic'] = theme_list result['level'] = level_id result['group'] = group_id result['chapter_info'] = chapter_info mongo.close() cursor.close() mysql.close() leave_func(self, 0) return self.write(json.dumps(result, ensure_ascii=False)) except Exception, e: LOG.error(e) return leave_func(self, 100)