def __init__(self, event, watch_config, **kwargs): self._mask = getattr(event, 'mask', None) self.file_name = getattr(event, 'name', None) # path without filename/foldername self.base_path = getattr(event, 'path', None) # path with filename/foldername self.source_absolute = getattr(event, 'pathname', None) self.isdir = getattr(event, 'dir', None) self.type = self.get_type() self.__dict__.update(kwargs) self.target_base_dir = watch_config['target'] # target dir self.source_base_dir = watch_config['source'] # source dir self.syncers = watch_config['syncers'] self.config = watch_config self.source_relative = os.path.join( os.path.relpath(self.source_absolute, self.source_base_dir) ) self.source_base_dir_relative = os.path.join( os.path.relpath(self.base_path, self.source_base_dir) ) self.target_absolute = os.path.join( self.target_base_dir, self.source_relative ) self.target_base_dir_absolute = os.path.normpath(os.path.join( self.target_base_dir, self.source_base_dir_relative )) self.moved_from_path = getattr(event, 'src_pathname', None) log.debug('EVENT %s (%s): %s ' % ( self.type, self.syncers, self.source_absolute))
def query_data(self, sql): con = self.__db_pool.connection() cursor = con.cursor(cursor=pymysql.cursors.DictCursor) try: log.debug('执行sql:%s' % sql) cursor.execute(sql) results = cursor.fetchall() for result in results: for fields in result: if isinstance(result[fields], datetime.datetime): result[fields] = str( result[fields].strftime('%Y-%m-%d %H:%M:%S')) elif isinstance(result[fields], datetime.date): result[fields] = str( result[fields].strftime('%Y-%m-%d')) elif isinstance(result[fields], decimal.Decimal): result[fields] = float(result[fields]) # if len(results) >= 1: # log.debug('sql查询结果:\n %s' % results) else: log.debug('sql查询结果为空') return results except Exception as e: log.error('执行sql异常:\n%s' % e) finally: cursor.close() con.close()
def get(self): log.debug("AlarmHandler get in") try: id = int(self.get_argument("id")) except: log.debug("param id is not int") return data = db.get_alarm(id) result = {} if data: result["result"] = "ok" result["data"] = {"id" : data[0], "create_time" : data[1], "zwaveid" : data[2], "deviceid": data[3], "deal_progress":data[4], "deal_user":data[5], } else: result["result"] = "error" result["message"] = "can not find this user" self.send_data(result)
def add_root_url(parser_params={}): log.debug(''' 添加根url parser_params : %s ''' % str(parser_params)) url = 'http://www.1kkk.com' html = tools.get_html_by_urllib(url) regex = '<li class="">.*?href="(.*?)" target="_parent"><span>.*?</span></a></li>' infos = tools.get_info(html, regex) china_cartoon = ['//manhua-china//'] infos = infos + china_cartoon for info in infos: info = info[:-1] url = 'http://www.1kkk.com' + info url_fenye = url + '-p' urls = url + '-p1' html = tools.get_html_by_urllib(urls) page_count = '\.\.\.<a href=".*?">(.*?)</a><a href=".*?">下一页</a>' page_count = tools.get_info(html, page_count) if not page_count: while url: html = tools.get_html_by_urllib(url) url = '<div id="search_fy">.*<a href="(.*?)" style=\'padding: 5px 20px; margin: 0 8px;\'> 下一页 </a>' url = tools.get_info(html, url) url = ''.join(url) url = 'http://www.1kkk.com' + url base_parser.add_url('WP_urls', SITE_ID, url) else: page_count = int(''.join(page_count)) for page in range(1, page_count + 1): url = url_fenye + '%d' % page base_parser.add_url('WP_urls', SITE_ID, url)
def post(self): log.debug("ManageHandler post in") data = self.get_data() if data: log.debug(data) if data.has_key("ManageTelephone") and data.has_key("ManagePassword") \ and data.has_key("ManageName") : if data["ManageName"] == "": log.error("ManageName is NULL") else: db.insert_manage(data["ManageName"], data["ManageTelephone"], data["ManagePassword"]) result ={} result["result"] = "ok" self.send_data(result) str = "add manage name %s telephone %s"%(data["ManageName"].encode('utf-8'),data["ManageTelephone"].encode('utf-8') ) save_record(self.login_user, "manage", 0, "add", str) return else: log.error("Manaage data key is not right") else: log.error("data is none") result = {} result["result"] = "error" result["message"] = "Manaage info is error" self.send_data(result)
def login(self): """登录QQ空间""" log.run().debug("执行Like.login()") # 打印日志 author_info = False try: log.info().info("正在读取用户信息") # 打印日志 with open("config/user.json", "r", encoding="utf-8") as usr: infos = json.load(usr) account = infos['account'] password = infos['password'] author_info = True log.info().info("用户信息读取成功") # 打印日志 except Exception as e: log.exception().exception(e) # 打印日志 log.error().error("用户信息读取失败") # 打印日志 if author_info: # 登录部分 log.info().info("开始登录QQ空间") # 打印日志 driver = webdriver.PhantomJS() driver.maximize_window() url = "https://qzone.qq.com/" driver.get(url) driver.implicitly_wait(3) try: driver.switch_to.frame("login_frame") try: driver.find_element_by_id('switcher_plogin').click() except: log.run().info("默认显示账号密码登录,不需要切换") driver.find_element_by_id('u').clear() driver.find_element_by_id('u').send_keys(account) driver.find_element_by_id('p').click() driver.find_element_by_id('p').send_keys(password) driver.find_element_by_id('login_button').click() time.sleep(3) driver.implicitly_wait(20) log.debug().debug("即将开始验证QQ登录") return self.login_on(driver) # 判断是否登录 except Exception as login_01: log.exception().exception(login_01) # 打印日志 log.error().info("QQ空间登录模块获取失败") # 打印日志 return 'error' else: return 'error'
def add_root_url(parser_params={}): log.debug(''' 添加根url parser_params : %s ''' % str(parser_params)) search_keyword1 = parser_params['search_keyword1'] search_keyword2 = parser_params['search_keyword2'] search_keyword3 = parser_params['search_keyword3'] remark = parser_params search_keywords = [] for str_key1 in search_keyword1: for str_key2 in search_keyword2: search_keywords.append((str_key1 + str_key2).strip()) else: if not search_keyword1: search_keywords = search_keyword2 if not search_keyword2: search_keywords = search_keyword1 for i in search_keywords: # print(i) if not i.strip(): continue for num in range(0, 760, 10): link = "https://www.baidu.com/s?wd=%s%s&pn=%d" % (i, ' 视频', num) # print(link) link = tools.quote(link, safe='#/:?=&%') if not base_parser.add_url('VA_urls', SITE_ID, link, remark=remark): base_parser.update_url('VA_urls', link, Constance.TODO)
def add_root_url(parser_params = {}): log.debug(''' 添加根url parser_params : %s '''% str(parser_params)) _db = base_parser.MongoDB() _db.set_unique_key('PROGRAM_EPISODE_info', 'episode_url') _db.update('PROGRAM_urls', {'depth': 0, 'site_id': SITE_ID}, {'status': 0}, multi=True) for page_num in range(1, 14): urls = [ 'http://list.youku.com/category/show/c_85_g_热门网综_s_1_d_1_p_%d.html' % page_num, 'http://list.youku.com/category/show/c_97_g_优酷出品_s_1_d_1_p_%d.html' % page_num, 'http://list.youku.com/category/show/c_96_g_优酷出品_s_1_d_1_p_%d.html' % page_num, ] for url in urls: print(url) print('********************************************************') html = tools.get_html_by_urllib(url) if tools.get_info(html, ['小酷没有筛选到相关视频']): continue links = tools.get_tag(html, 'div', {'class': 'p-thumb'}) for link in links: try: link = link.a['href'] link = tools.get_full_url('http:', link) link_html = tools.get_html_by_urllib(link) link = tools.get_tag(link_html, 'a', {'class': 'desc-link'}, find_all=False) link = link['href'] link = tools.get_full_url('http:', link) base_parser.add_url('PROGRAM_urls', SITE_ID, link, depth=0) except Exception as e: log.error(e) print(link_html)
def add_root_url(parser_params={}): log.debug(''' 添加根url parser_params : %s ''' % str(parser_params)) def get_page_count(url): html, r = tools.get_html_by_requests(url) regex = '<div class="mod_pages" r-notemplate="true">.*>(\d*)</a>.*?</span>' page_count = tools.get_info(html, regex, fetch_one=True) print(page_count) return int(page_count) if page_count else 0 def inner_add_root_url(root_url, page_url, remark): page_count = get_page_count(root_url) for page in range(0, page_count + 1): url = page_url % (page * 30) base_parser.add_url('PROGRAM_urls', SITE_ID, url, remark=remark) #首页-综艺-腾讯出品 root_url = 'http://v.qq.com/x/list/variety?offset=0&isource=2' page_url = 'http://v.qq.com/x/list/variety?offset=%d&isource=2' # inner_add_root_url(root_url, page_url, remark = {'classify' : '综艺'}) # 首页-电视剧-腾讯出品(自制剧) root_url = 'http://v.qq.com/x/list/tv?sort=4&offset=0&iarea=-1&iyear=-1&itype=843' page_url = 'http://v.qq.com/x/list/tv?sort=4&offset=%d&iarea=-1&iyear=-1&itype=843' inner_add_root_url(root_url, page_url, remark={'classify': '电视剧'})
def get(self): log.debug("alarmAllHandler get in") try: alarm_id = int(self.get_argument("alarm")) except: index = 1 data = db.get_audio_list(alarm_id) result = {} result["result"] = "ok" result["data"] =[] if data: for one in data: if os.path.exists("./static/audio/%s"%one[6]): has_audio=1 else: has_audio =0 result["data"].append({ "deal_user":one[2], "telephone":one[3], "deal_time":one[4], "deal_remark":one[5], "audio":one[6], "has_audio":has_audio }) self.send_data(result) else: self.send_data(result)
def parser_program_url(url_info): log.debug('处理 \n' + tools.dumps_json(url_info)) root_url = url_info['url'] depth = url_info['depth'] site_id = url_info['site_id'] remark = url_info['remark'] classify = remark['classify'] # 解析 html, request = tools.get_html_by_requests(root_url) if not html: base_parser.update_url('PROGRAM_urls', root_url, Constance.EXCEPTION) return program_blocks = tools.get_tag(html, 'li', {'class': "list_item"}) for program_block in program_blocks: program_block = str(program_block) # 地址 regex = 'r-props="{id: \'(.*?)\'' program_id = tools.get_info(program_block, regex, fetch_one=True) program_url = 'http://v.qq.com/detail/5/%s.html' % program_id base_parser.add_url("PROGRAM_urls", site_id, program_url, depth=1, remark={ 'program_id': program_id, 'classify': classify }) base_parser.update_url("PROGRAM_urls", root_url, Constance.DONE)
def add_root_url(search_keyword1=[], search_keyword2=[], search_keyword3=[]): log.debug( ''' 添加根url search_keyword1 = %s search_keyword2 = %s search_keyword3 = %s ''' % (str(search_keyword1), str(search_keyword2), str(search_keyword3))) remark = { 'search_keyword1': search_keyword1, 'search_keyword2': search_keyword2, 'search_keyword3': search_keyword3 } search_keywords = search_keyword1 + search_keyword2 for search_keyword in search_keywords: # 取页数 url = 'https://movie.douban.com/subject_search?start=0&search_text=%s&cat=1002' % search_keyword html = tools.get_html_by_urllib(url) regex = '<div class="paginator">.*<a href.*?>(.*?)</a><span class="next"' page_count = tools.get_info(html, regex) page_count = int(page_count[0]) if page_count else 0 print(page_count) for page in range(0, page_count): url = 'https://movie.douban.com/subject_search?start=%d&search_text=%s&cat=1002' % ( page * 15, search_keyword) if not base_parser.add_url('VA_urls', SITE_ID, url, remark=remark): base_parser.update_url('VA_urls', url, Constance.TODO)
def add_site_info(): log.debug('添加网站信息') site_id = SITE_ID name = NAME table = 'PROGRAM_site_info' url = "http://best.le.com/" base_parser.add_website_info(table, site_id, url, name)
def deal_request(self, name): web.header('Content-Type', 'text/html;charset=UTF-8') data_json = json.loads(json.dumps(web.input())) data = data_json.get('data') # data为str req_url = data_json.get('req_url') # log.debug(''' # method : %s # data :%s # '''%(name, data)) log.debug(''' method : %s url :%s ''' % (name, req_url)) reponse = '' if name == 'get_article_list': reponse = self.get_article_list(data, req_url) elif name == 'get_article_content': reponse = self.get_article_content(data, req_url) elif name == 'get_read_watched_count': reponse = self.get_read_watched_count(data, req_url) elif name == 'get_comment': reponse = self.get_comment(data, req_url) # log.debug(''' # ---------reponse--------- # %s'''%reponse) return reponse # 此处返回''空字符串 不会触发node-js http 的回调
def add_site_info(): log.debug('添加网站信息') site_id = SITE_ID name = NAME table = 'PROGRAM_site_info' url = "http://news.v1.cn/V1make.shtml" base_parser.add_website_info(table, site_id, url, name)
def add_site_info(): log.debug('添加网站信息') site_id = SITE_ID name = NAME table = 'site_info' url = 'https://www.itouchtv.cn/' base_parser.add_website_info(table, site_id, url, name)
def parser(url_info): url_info['_id'] = str(url_info['_id']) log.debug('处理 \n' + tools.dumps_json(url_info)) root_url = url_info['url'] depth = url_info['depth'] site_id = url_info['site_id'] remark = url_info['remark'] offset = remark.get('offset') html = tools.get_html_by_webdirver(root_url) headers = tools.get_tag(html, 'div', {'class': 'result'}, find_all=True) if not headers: base_parser.update_url('BAIDU_NEWS_urls', root_url, Constance.DONE) for header in headers: # 查看更多相关新闻 regex = ' <span class="c-info"><a.*?href="(.*?)".*?查看更多相关新闻' more_news_url = tools.get_info(str(header), regex, fetch_one = True) if more_news_url: more_news_url = tools.get_full_url('http://news.baidu.com', more_news_url) more_news_url = more_news_url.replace('amp;', '') base_parser.add_url('BAIDU_NEWS_urls', SITE_ID, more_news_url, depth = 1, remark = {'offset':0}) url = header.h3.a['href'] article_extractor = ArticleExtractor(url) content = title = release_time = author = website_domain ='' content = article_extractor.get_content() if content: title = article_extractor.get_title() release_time = article_extractor.get_release_time() author = article_extractor.get_author() website_domain = tools.get_domain(url) uuid = tools.get_uuid(title, website_domain) website_name = '' website_position = None log.debug(''' uuid %s title %s author %s release_time %s domain %s url %s content %s '''%(uuid, title, author, release_time, website_domain, url, '...')) # 入库 if tools.is_have_chinese(content): is_continue = self_base_parser.add_news_acticle(uuid, title, author, release_time, website_name , website_domain, website_position, url, content) if not is_continue: break else: # 循环正常结束 该页均正常入库, 继续爬取下页 offset += 50 url = tools.replace_str(root_url, 'pn=\d*', 'pn=%d'%offset) base_parser.add_url('BAIDU_NEWS_urls', SITE_ID, url, depth = 0, remark = {'offset': offset}) base_parser.update_url('BAIDU_NEWS_urls', root_url, Constance.DONE)
def add_site_info(): log.debug('添加网站信息') table = 'BAIDU_NEWS_site_info' url = 'http://news.baidu.com' base_parser.add_website_info(table, site_id=SITE_ID, url=url, name=NAME)
def add_site_info(): log.debug('添加网站信息') table = 'VIDEO_NEWS_site_info' url = 'https://v.qq.com' base_parser.add_website_info(table, site_id=SITE_ID, url=url, name=NAME)
def calculate_time(*args, **kw): began_time = time.time() callfunc = func(*args, **kw) end_time = time.time() log.debug(func.__name__ + " run time = " + str(end_time - began_time)) return callfunc
def add_root_url(search_keyword1=[], search_keyword2=[], search_keyword3=[]): log.debug( ''' 添加根url search_keyword1 = %s search_keyword2 = %s search_keyword3 = %s ''' % (str(search_keyword1), str(search_keyword2), str(search_keyword3))) remark = { 'search_keyword1': search_keyword1, 'search_keyword2': search_keyword2, 'search_keyword3': search_keyword3 } search_keywords = search_keyword1 + search_keyword2 for search_keyword in search_keywords: if not search_keyword: continue # 最多显示10页 for page in range(1, 11): url = 'http://weixin.sogou.com/weixin?type=2&query=' + search_keyword + '&page=%d&ie=utf8' % page if not base_parser.add_url('VA_urls', SITE_ID, url, remark=remark): base_parser.update_url('VA_urls', url, Constance.TODO)
def main(): while True: if task_status.is_doing: #done log.debug('is doing sleep ...%ss' % SLEEP_TIME) time.sleep(SLEEP_TIME) continue task_status.is_doing = True keywords = Keywords().get_keywords() def begin_callback(): log.info('\n********** spider_main begin **********') def end_callback(): log.info('\n********** spider_main end **********') task_status.is_doing = False # 配置spider spider = Spider(tab_list, tab_unique_key_list, tab_ensure_index_list, parser_count=1, site_parsers=parser_siteid_list, begin_callback=begin_callback, end_callback=end_callback, parser_params=keywords) # 添加parser for parser in parser_list: spider.add_parser(parser) spider.start()
def connect(self): """ Connect to the broker, define the callbacks, and subscribe This will also set the Last Will and Testament (LWT) The LWT will be published in the event of an unclean or unexpected disconnection. """ self.is_connected = False # Add the callbacks self.mqttc.on_connect = self.on_mqtt_connect self.mqttc.on_disconnect = self.on_mqtt_disconnect # Set the Last Will and Testament (LWT) *before* connecting self.mqttc.will_set(self.lwt, payload="0", qos=0, retain=True) # Attempt to connect log.debug("Connecting to {}:{}...".format(self.host, self.port)) try: self.mqttc.connect(self.host, self.port, 60) except Exception as e: log.error("Error connecting to {}:{}: {}".format( self.host, self.port, str(e))) self.is_connected = True # Let the connection run forever self.mqttc.loop_start() return self.is_connected
def start(self, args): """Start up all of the application components""" enable_log() # Make sure the results have some data in it before the web requests # come in self.results = App._init_results(self.config) if args.poll_seconds: self.config.override_monitor_setting( 'poll_seconds', args.poll_seconds ) if self.config.log_file: set_log_file(self.config.log_file) log.debug("Opening log file {}".format(self.config.log_file)) self._start_monitoring() if self.config.http_port: log.info("Starting web frontend on port {}".format( self.config.http_port )) self._start_frontend()
def is_have_new_article(self, account_id='', account=''): ''' @summary: 检查公众号今日是否发文 --------- @param account_id: @param account: --------- @result: ''' account_block = self.__get_account_blocks(account_id, account) if account_block == constance.VERIFICATION_CODE: return constance.VERIFICATION_CODE regex = "timeConvert\('(\d*?)'\)" release_time = tools.get_info(account_block, regex, fetch_one=True) if release_time: release_time = int(release_time) release_time = tools.timestamp_to_date(release_time) log.debug("最近发文时间 %s" % release_time) if release_time >= tools.get_current_date('%Y-%m-%d'): return constance.UPDATE else: return constance.NOT_UPDATE else: return constance.ERROR
def get(self): log.debug("ManaageAllHandler get in") try: index = int(self.get_argument("index")) except: index = 1 if index > 0: pass else: index = 1 data = db.get_manage_list(index) result = {} if data: result["result"] = "ok" result["maxindex"] = data["maxindex"] result["curruntindex"] = data["curruntindex"] result["data"] =[] for one in data["data"]: result["data"].append({"manageid" : one[0], "managename" : one[1], "managetelephone" : one[2], "managelogintime": one[5], "managelogouttime": one[6] }) self.send_data(result) else: result["result"] = "error" result["message"] = "get Manage list failed" self.send_data(result)
def add_root_url(parser_params={}): log.debug(''' 添加根url parser_params : %s ''' % str(parser_params)) for program in parser_params: #[[91, '山东卫视', '调查', '新闻'], [...]] program_id = program[0] chan_name = program[1] program_name = program[2] program_type = program[3] image_url = program[4] if program_type != '其他': url = 'http://so.iqiyi.com/so/q_%s %s?source=input&sr=1170053009947' % ( program_name, program_type) else: url = 'http://so.iqiyi.com/so/q_%s?source=input&sr=1170053009947' % ( program_name) base_parser.add_url('mms_urls', SITE_ID, url, remark={ 'program_id': program_id, 'program_name': program_name, 'chan_name': chan_name, 'program_type': program_type, 'image_url': image_url })
def __input_data(self): # log.debug('read_pos %d, write_pos %d buffer size %d'%(self._read_pos, self._write_pos, self.get_max_read_size())) # log.debug('buffer can write size = %d'%self.get_max_write_size()) if self.get_max_write_size() == 0: log.debug("collector 已满 size = %d" % self.get_max_read_size()) return url_count = self._url_count if self._url_count <= self.get_max_write_size( ) else self.get_max_write_size() urls_list = [] if self._depth: urls_list = self._db.find(self._tab_urls, { "status": Constance.TODO, "depth": { "$lte": self._depth } }, limit=url_count) else: urls_list = self._db.find(self._tab_urls, {"status": Constance.TODO}, limit=url_count) #更新已取到的url状态为doing for url in urls_list: self._db.update(self._tab_urls, url, {'status': Constance.DOING}) # 存url self.put_urls(urls_list) if self.is_all_have_done(): self.stop()
def parser_comment(content_id, wall_id, page=1): log.debug('正在爬取第 %s 页文章评论 content_id = %s' % (page, content_id)) flow_comment_url = 'http://sns-comment.iqiyi.com/v2/comment/get_comments.action?contentid={content_id}&page={page}&authcookie=null&page_size=40&wallId={wall_id}&agenttype=117&t={timestamp_m}'.format( content_id=content_id, page=page, wall_id=wall_id, timestamp_m=int(tools.get_current_timestamp() * 1000)) comment_json = tools.get_json_by_requests(flow_comment_url) data = comment_json.get('data', {}) # 可作为翻页的依据 total_count = data.get('totalCount', 0) count = data.get('count', 0) replies = data.get('replies', []) for reply in replies: reply_source = reply.get("replySource", {}) if not deal_comment(reply_source): break if not deal_comment(reply): break else: if replies: parser_comment(content_id, wall_id, page + 1)
def add_account_info(self, account_info): log.debug(''' -----公众号信息----- %s''' % tools.dumps_json(account_info)) WechatService._es.add('wechat_account', account_info, account_info.get('__biz'))
def deal_comment(reply): if not reply: return comment_id = reply.get('id') pre_id = reply.get('replyId') content = reply.get('content') article_id = reply.get('mainContentId') release_time = reply.get('addTime') release_time = tools.timestamp_to_date(release_time) head_url = reply.get('userInfo', {}).get('icon') consumer = reply.get('userInfo', {}).get('uname') gender = int(reply.get('userInfo', {}).get('gender')) up_count = reply.get('likes') # TODO emotion = random.randint(0, 2) hot_id = comment_id log.debug(''' 评论id: %s 父id %s 文章id %s 发布人: %s 头像地址 %s 性别 %s 内容: %s 点赞量 %s 发布时间 %s ''' % (comment_id, pre_id, article_id, consumer, head_url, gender, content, up_count, release_time)) return self_base_parser.add_comment(comment_id, pre_id, article_id, consumer, head_url, gender, content, up_count, release_time, emotion, hot_id)
def add_root_url(parser_params={}): log.debug(''' 添加根url parser_params : %s ''' % str(parser_params)) def inner_add_url(url, remark): html = tools.get_html_by_urllib(url) regex = '<li><span></span><a href="(.*?)">.*?</a></li>' infos = tools.get_info(html, regex) for info in infos: info = ''.join(info) type_url = 'http://shouji.baidu.com' + info type_html = tools.get_html_by_urllib(type_url) page_count = '<div class="pager">.*">(.*?)</a>.*?<li class="next">' page_count = tools.get_info(type_html, page_count) page_count = ''.join(page_count) if not page_count: page_count = '1' page_count = int(page_count) for page in range(1, page_count + 1): url = type_url + 'list_%d.html' % page if not base_parser.add_url( 'GameApp_urls', SITE_ID, url, remark=remark): base_parser.update_url('GameApp_urls', url, Constance.TODO) inner_add_url('http://shouji.baidu.com/game/401/', Constance.休闲益智) inner_add_url('http://shouji.baidu.com/game/board_102_200/', Constance.网络游戏) inner_add_url('http://shouji.baidu.com/game/403/', Constance.飞行射击) inner_add_url('http://shouji.baidu.com/game/406/', Constance.赛车竞速) inner_add_url('http://shouji.baidu.com/game/405/', Constance.体育竞技) inner_add_url('http://shouji.baidu.com/game/407/', Constance.体育竞技) inner_add_url('http://shouji.baidu.com/game/408/', Constance.经营策略) inner_add_url('http://shouji.baidu.com/game/402/', Constance.角色扮演)
def creat_db_pool(self, host): connection_config = decrypt(getenv('INTERFACE_CIPHER')) host = decrypt(host) if connection_config.get(host, None) is None: log.error("IP域名错误") exit(0) user = connection_config.get(host).get('user') password = connection_config.get(host).get('password') port = connection_config.get(host).get('port') log.debug('创建数据库连接池:%s' % host) self.__db_pool = PooledDB(creator=pymysql, mincached=3, maxcached=5, maxshared=0, maxconnections=20, blocking=True, maxusage=None, setsession=None, host=host, port=port, user=user, db=None, passwd=password) log.debug('创建数据库连接池完成!')
def add_site_info(): log.debug('添加网站信息') site_id = SITE_ID name = NAME table = 'site_info' url = 'https://www.wandoujia.com/app/org.fungo.fungolive' base_parser.add_website_info(table, site_id, url, name)
def get(self): log.debug("EventsHandler get in") try: alarmid = int(self.get_argument("alarmid")) except: log.debug("param alarmid is not int") return data = db.get_events(alarmid) result = {} if data: result["result"] = "ok" result["data"] = [] for one in data: result["data"].append({"id": one[0], "type":one[1], "eventtime": one[4], "context": one[5] }) else: result["result"] = "error" result["message"] = "can not find this user" self.send_data(result)
def get_domain(url): domain = '' try: domain = get_tld(url) except Exception as e: log.debug(e) return domain
def add_root_url(parser_params={}): log.debug(''' 添加根url parser_params : %s ''' % str(parser_params)) base_parser.add_url('VAApp_urls', SITE_ID, URL, remark=NEWS_LOCAL)
def get(self, url): try: url = self.url + url log.debug( url) urllib2.urlopen(url) except: print "get request error"
def releasealarm(self, zwaveid, user): values = {'zwavedeviceid': zwaveid, 'token': self.token, "employeename" : user} print values data = self.post("/thirdpart/zufang/unalarmdevicewarning", values) if data: if data["resultCode"] == 0: log.debug("release alarm ok") else: log.debug("release alarm error")
def run(self): log.debug(self.__class__.__name__ + " running") while True: item = self.queue.get() if item is None: # trick to break out of while break self.consume_item(item) # TODO what if a file gets added again while syncing in progress? self.queue.task_done()
def rm(self, path, *args, **kwargs): if path == '/': log.critical('prevented delete / (root)') return try: self.client.file_delete(path) except dropbox.rest.ErrorResponse as e: log.debug('Delete failed: %s (%s)' % (e.reason, path)) if not e.reason == 'Not Found': raise e
def get_syncer_instances(filter=lambda: True): # Import syncers from 'syncers' package and start them. # Does something like: from syncers.dropbox import Dropbox syncer_instances = {} # find classes inside syncers package that have the superclass SyncBase available_syncers = dict(find_modules_with_super_class(syncers, SyncBase)) log.debug("available_syncers: %s" % list(available_syncers.keys())) for syncer in builtins.filter(filter, available_syncers.keys()): syncer_instances[syncer] = getattr(import_module(available_syncers[syncer]), syncer)() return syncer_instances
def post(self, url, values): try: url = self.url + url data = urllib.urlencode(values) req = urllib2.Request(url, data) response = urllib2.urlopen(req) data= response.read() log.debug(data) data = json.loads(data) return data except: print "post request error" return None
def login(self): token_file = os.path.expanduser(self.configuration['token_file']) token_dir = os.path.dirname(token_file) if not os.path.exists(token_dir): os.makedirs(token_dir) try: with open(token_file) as token: self.access_token = token.read() except IOError: self.access_token = None if not (self.access_token): self.authorize() self.client = dropbox.client.DropboxClient(self.access_token) log.debug('dropbox authorized: ' + self.client.account_info()['email'])
def put(self): log.debug("alarmHandler put in") data = self.get_data() if data: log.debug(data) if data.has_key("deal_progress")and data.has_key("alarmId") : db.update_alarm_progress(data["alarmId"],data["deal_progress"]) result = {} result["result"] = "ok" self.send_data(result) return else: result = {} result["result"] = "error" result["message"] = "data is error" self.send_data(result)
def put(self): log.debug("ManageChangePasswordHandler put in") data = self.get_data() if data: log.debug(data) if data.has_key("ManagePassword") and data.has_key("ManageId"): db.update_manage_passwd(data["ManageId"],data["ManagePassword"]) result = {} result["result"] = "ok" self.send_data(result) save_record(self.login_user, "manage",data["ManageId"], "change_passwd", "update manage password") return return else: result = {} result["result"] = "error" result["message"] = "data is error" self.send_data(result)
def post(self): log.debug("UploadHandler post in") if self.request.files == {}: result = {} result["error"] = "null file upload" self.send_data(result) return file_metas = self.request.files['file']# 提取表单中‘name’为‘file’的文件元数据 for meta in file_metas: filename = meta['filename'] upload_path = "./static/audio" filepath = os.path.join(upload_path, filename) log.debug("write %s"%filepath) with open(filepath, 'wb') as up: # 有些文件需要已二进制的形式存储,实际中可以更改 up.write(meta['body']) result = {} result["result"] = "ok" self.send_data(result)
def post(self): log.debug("ReleaseAlarmHandler post in") data = self.get_data() if data: log.debug(data) if data.has_key("alarmId"): zwaveid = db.get_zwaveid_from_alarm(data["alarmId"]) result = {} result["result"] = "ok" self.send_data(result) client.releasealarm(zwaveid, self.login_user) return else: result = {} result["result"] = "error" result["message"] = "data is error" self.send_data(result)
def post(self): log.debug("AudioHandler post in") data = self.get_data() if data: log.debug(data) if data.has_key("AlarmID") and data.has_key("AlarmRemark") \ and data.has_key("AlarmAudio")and data.has_key("AlarmTelephone"): db.insert_alarm_deal(data["AlarmID"], self.login_user, data["AlarmTelephone"],data["AlarmRemark"], "%s.wav"%data["AlarmAudio"]) db.update_alarm_progress(data["AlarmID"],1) result = {} result["result"] = "ok" self.send_data(result) return else: result = {} result["result"] = "error" result["message"] = "data is error" self.send_data(result)
def get(self): log.debug("alarmAllHandler get in") try: index = int(self.get_argument("index")) except: index = 1 if index > 0: pass else: index = 1 data = db.get_alarm_list(index) result = {} if data: result["result"] = "ok" result["maxindex"] = data["maxindex"] result["curruntindex"] = data["curruntindex"] result["data"] =[] for one in data["data"]: context = "" eventlist = db.get_events(one[0]) for event in eventlist: context = context + event[5] + " " if one[5] == 0: user = "******" else: user = db.get_username_by_id(one[5]) result["data"].append({"id" : one[0], "create_time" : one[1], "zwaveid": one[2], "deviceid":one[3], "deal_progress":one[4], "deal_user":one[5], "deal_user_name": user, "deal_context": context }) self.send_data(result) else: result["result"] = "error" result["message"] = "get alarm list failed" self.send_data(result)
def put(self): log.debug("ManageHandler put in") data = self.get_data() if data: log.debug(data) if data.has_key("ManageTelephone") and data.has_key("ManageId") \ and data.has_key("ManageName"): db.update_manage(data["ManageId"],data["ManageName"], data["ManageTelephone"]) result = {} result["result"] = "ok" self.send_data(result) str = "update manage name %s telephone %s" % ( data["ManageName"].encode('utf-8'), data["ManageTelephone"].encode('utf-8')) save_record(self.login_user, "manage", data["ManageId"], "update", str) return else: result = {} result["result"] = "error" result["message"] = "data is error" self.send_data(result)
def get(self): log.debug("ManageHandler get in") try: id = int(self.get_argument("id")) except: log.debug("param id is not int") return data = db.get_manage(id) result = {} if data: result["result"] = "ok" result["data"] = {"manageid" : data[0], "managename" : data[1], "managetelephone" : data[2] } else: result["result"] = "error" result["message"] = "can not find this user" self.send_data(result)
def alarm_sync(): db =DB() client.get_token() while True: try: time.sleep(10) lastid = db.get_sync_id() events = client.get_alarm(lastid) if events == None: client.get_token() continue else: if events == []: #log.debug("not has any event") continue for one in events: log.debug(one) db.save_event(one["id"], one["type"],one["deviceid"],one["zwavedeviceid"],one["eventtime"],one["objparam"]) except: import traceback traceback.print_exc() continue
def get(self, html): url = "%s/%s"%(BASEDIR,html) log.debug(url) #try : self.render(url)
def stop(self): self.queue.put(None) # trick to break out of while log.debug(self.__class__.__name__ + " stopped")
def __init__(self, queue=None): self.queue = queue or OrderedSetQueue() super().__init__() log.debug(self.__class__.__name__ + " init")
def get(self): log.debug("UploadHandler get in")
def get(self): log.debug("RedirectHandler") self.redirect("/static/index.html") pass