def main(gs_search_id,gs_basic_id,url): Log().found_log(gs_search_id,gs_basic_id) name = 'person' flag = Judge_status().judge(gs_basic_id, name, Person, url) # if __name__ == '__main__': # main(gs_py_id,gs_basic_id,url)
def get_all_info(gs_basic_id, gs_search_id, info_list): Log().found_log(gs_basic_id, gs_search_id) info = class_dict["basic"]().get_info(info_list["basic"]) flag = class_dict["basic"]().update_to_db(info, gs_basic_id) print "basic:%s" % flag pripid = '28890' name = info["name"] for key, value in info_list.iteritems(): # 这两块信息较为特殊,拿出来单独处理 if key == "basic" or key == "report" or key == "report1": continue # 如果没有获取到对应的信息就跳过 if key not in info_list.keys(): continue Judge(pripid, name, config.dict_url[key]).update_info(key, class_dict[key], value, gs_basic_id) # 单独处理年报 tr_list = info_list["report"].xpath(".//tr") fill_data = {} for i, singledata in enumerate(tr_list): td_list = singledata.xpath(".//td") if len(td_list) == 0: continue fill_data[i] = deal_html_code.change_chinese_date( deal_html_code.remove_symbol(td_list[2].xpath("string(.)"))) if len(fill_data) == 0: print "report:-1" else: print "report:%s" % len(fill_data) SHX_report.main(info_list["report1"], fill_data, gs_basic_id)
def main(gs_py_id, gs_basic_id, urllist): Log().found_log(gs_py_id, gs_basic_id) object = Permit() name = 'permit' mark, number, insert_total, update_total = 0, 0, 0, 0 for i, single in enumerate(urllist): time.sleep(0.5) # print single if "变更" in single: pass else: info, remark = object.name(single) if remark == 1: flag, total, insert, update = object.update_to_db( info, gs_basic_id) if flag < 100000001: number += 1 insert_total += insert update_total += update if insert_total + update_total == 0: mark = 100000006 else: mark = insert_total + update_total string = '%s:' % name + str(mark) + '||' + str(number) + '||' + str( insert_total) + '||' + str(update_total) print string Judge_status().update_py(gs_py_id, update_permit_py, mark)
def main(gs_py_id, gs_basic_id, urllist): Log().found_log(gs_py_id, gs_basic_id) name = 'brand' for i, single in enumerate(urllist): time.sleep(0.5) flag = Judge_status().judge(gs_basic_id, name, Brand, url) Judge_status().update_py(gs_py_id, update_brand_py, flag)
def main(gs_py_id, gs_basic_id, url): try: Log().found_log(gs_py_id, gs_basic_id) info, flag = Share_History().name(url) if flag == 1: Share_History().upadte_to_db(info, gs_basic_id) except Exception, e: logging.info("sharehistory error:%s" % e)
def main(gs_search_id, gs_basic_id, url): Log().found_log(gs_search_id, gs_basic_id) name = 'shareholder' flag = Judge_status().judge(gs_basic_id, name, Shareholder, url) # if __name__ == '__main__': # main(gs_py_id,gs_basic_id,url)
def main(): Log().found_log(gs_py_id, gs_basic_id) try: HOST, USER, PASSWD, DB, PORT = config.HOST, config.USER, config.PASSWD, config.DB, config.PORT connect, cursor = Connect_to_DB().ConnectDB(HOST, USER, PASSWD, DB, PORT) flag = update_report_main(gs_basic_id, gs_py_id, year, cursor, connect) string = '%s:' % year + str(flag) print string except Exception, e: logging.error('report error:%s' % e)
def main(gs_py_id,gs_basic_id,data): Log().found_log(gs_py_id,gs_basic_id) HOST, USER, PASSWD, DB, PORT = config.HOST, config.USER, config.PASSWD, config.DB, config.PORT connect, cursor = Connect_to_DB().ConnectDB(HOST, USER, PASSWD, DB, PORT) object = Basic() info = object.name(data) remark = object.update_to_db(cursor,connect,gs_basic_id,info) updated_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())) cursor.execute(update_py,(gs_py_id,remark,updated_time,gs_py_id)) print 'basic:%s' % str(remark) cursor.close() connect.close()
def main(gs_py_id,gs_basic_id,url): Log().found_log(gs_py_id,gs_basic_id) insert,error,total = 0,0,0 try: HOST, USER, PASSWD, DB, PORT = config.HOST, config.USER, config.PASSWD, config.DB, config.PORT connect, cursor = Connect_to_DB().ConnectDB(HOST, USER, PASSWD, DB, PORT) object = Report() flag, href_list, cookies = object.get_year_href(url) cookies = requests.utils.dict_from_cookiejar(cookies) if flag == 1: total, insert, error = object.get_all_report_info(href_list,cursor,connect,gs_basic_id,gs_py_id,cookies) if total == 0: flag = -1 except Exception,e: # print e flag = 100000005 logging.info("report error:%s"%e)
def main(): Log().found_log1() try: HOST, USER, PASSWD, DB, PORT = config.HOST, config.USER, config.PASSWD, config.DB, config.PORT connect, cursor = Connect_to_DB().ConnectDB(HOST, USER, PASSWD, DB, PORT) count = cursor.execute(select_info) if count == 0: logging.info("there is no task need to do!") else: for gs_new_id, name, province in cursor.fetchall(): if province == 'SHH' or province == "HEB" or province == "SCH" or province == "YUN": if name == '' and code == '' and code == '': logging.info("this is a useless info") else: second_url, flag, information = get_index( name, province) if flag == 1: code = information[0][1] ccode = information[0][2] if code != None and ccode != None: count = cursor.execute(select_string, (code, ccode, code)) update_info(count, cursor, connect, second_url, province, information, gs_new_id) elif code != None: count = cursor.execute(select_string, (code, code)) update_info(count, cursor, connect, second_url, province, information, gs_new_id) elif ccode != None: count = cursor.execute(ccode, ccode) update_info(count, cursor, connect, second_url, province, information, gs_new_id) else: logging.info( "get cookies failed or there is no search info ,the status is %s" % flag) cursor.execute(update_status1, (gs_new_id, gs_new_id)) connect.commit() else: logging.info("the province is out of range") except Exception, e: logging.info("get list error:%s" % e)
def main(gs_py_id,gs_basic_id,data,province): Log().found_log(gs_py_id, gs_basic_id) try: HOST, USER, PASSWD, DB, PORT = config.HOST, config.USER, config.PASSWD, config.DB, config.PORT connect, cursor = Connect_to_DB().ConnectDB(HOST, USER, PASSWD, DB, PORT) info = Freeze().name(data,province) flag, total, insert_flag, update_flag = Freeze().update_to_db(cursor,connect,gs_basic_id,info) flag = Judge_status().judge(flag,total) string = 'freeze:' + str(flag) + '||' + str(total) + '||' + str(insert_flag) + '||' + str(update_flag) print string if flag == -1: pass else: updated_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())) cursor.execute(update_freeze_py, (gs_py_id, flag, updated_time, gs_py_id)) connect.commit() except Exception,e: logging.error("freeze error :%s"%e)
def get_info(): Log().found_log(user_id, unique_id) success, error, insert = 0, 0, 0 total = 0 try: HOST, USER, PASSWD, DB, PORT = config.HOST, config.USER, config.PASSWD, config.DB, config.PORT connect, cursor = Connect_to_DB().ConnectDB(HOST, USER, PASSWD, DB, PORT) info, flag = get_list(keyword) if flag == 1: datalist, total, success, error = get_detail(info) for key in datalist.keys(): information = datalist[key] tempinsert = update_to_search(information, keyword, user_id, unique_id, cursor, connect) if tempinsert == 1: insert += 1 except Exception, e: flag = 100000005 logging.error("unknow error:%s" % e)
def main(): Log().found_log(gs_py_id,gs_basic_id) try: HOST, USER, PASSWD, DB, PORT = config.HOST, config.USER, config.PASSWD, config.DB, config.PORT connect, cursor = Connect_to_DB().ConnectDB(HOST, USER, PASSWD, DB, PORT) printinfo = { "url":0, 'flag':0 } second_url,flag,entName = get_index(code) if flag ==100000003: second_url,flag,entName = get_index(ccode) if flag == 100000003: select_string = select_name % gs_basic_id cursor.execute(select_string) name = cursor.fetchall()[0][0] second_url, flag, entName = get_index(name) if name != entName: second_url = 0 flag = 100000002 except Exception,e: logging.info("get list error:%s"%e)
def main(gs_py_id, gs_basic_id, url): Log().found_log(gs_py_id, gs_basic_id) name = 'change' flag = Judge_status().judge(gs_basic_id, name, Change, url) Judge_status().update_py(gs_py_id, update_change_py, flag)
def main(gs_py_id, gs_basic_id, data): Log().found_log(gs_py_id, gs_basic_id) print_info = Judge_status().update_py(gs_py_id, gs_basic_id, Punish, "punish2", data, update_punish_py) return print_info
def main(gs_py_id, gs_basic_id, url): Log().found_log(gs_py_id, gs_basic_id) name = 'shareholder' flag = Judge_status().judge(gs_basic_id, name, Shareholder, url) Judge_status().update_py(gs_py_id, update_share_py, flag)
def main(gs_py_id, gs_basic_id, url): Log().found_log(gs_py_id, gs_basic_id) name = 'person' flag = Judge_status().judge(gs_basic_id, name, Person, url)
def main(gs_py_id, gs_basic_id, data): Log().found_log(gs_py_id, gs_basic_id) Judge_status().update_py(gs_py_id, gs_basic_id, Punish, "punish", data, update_punish_py)
def main(gs_py_id,gs_basic_id,data): Log().found_log(gs_py_id, gs_basic_id) Judge_status().update_py(gs_py_id,gs_basic_id,Share,"shareholder",data,update_share_py)
def main(gs_py_id, gs_basic_id, url): Log().found_log(gs_py_id, gs_basic_id) name = 'branch' flag = Judge_status().judge(gs_basic_id, name, Branch, url) Judge_status().update_py(gs_py_id, update_branch_py, flag)
def main(gs_py_id, gs_basic_id, url): Log().found_log(gs_py_id, gs_basic_id) name = 'clear' flag = Judge_status().judge(gs_basic_id, name, Clear, url) Judge_status().update_py(gs_py_id, update_clear_py, flag)
def main(gs_py_id, gs_basic_id, url): Log().found_log(gs_py_id, gs_basic_id) name = 'share_history' flag = Judge_status().judge(gs_basic_id, name, Share_History, url)
def main(gs_py_id,gs_basic_id,data): Log().found_log(gs_py_id, gs_basic_id) Judge_status().update_py(gs_py_id,gs_basic_id,Stock,"stock",data,update_stock_py)
def main(gs_py_id, gs_basic_id, url): Log().found_log(gs_py_id, gs_basic_id) name = 'except' flag = Judge_status().judge(gs_basic_id, name, Except, url)
def main(gs_py_id, gs_basic_id, url): Log().found_log(gs_py_id, gs_basic_id) name = 'check' flag = Judge_status().judge(gs_basic_id, name, Check, url)
def main(gs_py_id, gs_basic_id, data): Log().found_log(gs_py_id, gs_basic_id) print_info = Judge_status().update_py(gs_py_id, gs_basic_id, Clear, "clear", data, update_clear_py) return print_info
def main(gs_py_id, gs_basic_id, url): Log().found_log(gs_py_id, gs_basic_id) name = 'punish' flag = Judge_status().judge(gs_basic_id, name, Punish, url)
def main(gs_search_id, gs_basic_id, url): Log().found_log(gs_search_id, gs_basic_id) name = 'change' flag = Judge_status().judge(gs_basic_id, name, Change, url)
def main(gs_py_id, gs_basic_id, url): Log().found_log(gs_py_id, gs_basic_id) name = 'Freeze' flag = Judge_status().judge(gs_basic_id, name, Freeze, url) Judge_status().update_py(gs_py_id, update_freeze_py, flag)
def main(gs_py_id,gs_basic_id,data): Log().found_log(gs_py_id, gs_basic_id) Judge_status().update_py(gs_py_id,gs_basic_id,Brand,"brand",data,update_brand_py)