class FetcherBase: def __init__(self, item_id, track_type,log=None): self.log = LogAdapter(log) self.item_id = item_id self.track_type = track_type self.name = "FetcherBase" self.proxy_dict = "" self.track_dict={} self.http = HttpClient() self.http.req_timeout = 30 self.conn = None self.db = None self.debug_level = 0 self.initialised = False def SetProxy(self,proxy_dict): self.proxy_dict = proxy_dict if proxy_dict is not None and len(proxy_dict) > 0: self.http.AddProxy(self.proxy_dict) def SetDBConn(self,conn): self.conn = conn self.db = track_db.TrackDB(db_conn=self.conn, log=self.log) def Fetch(self): try: self._fetch() except Exception as e: self._error_handle(e) def _chk_new_items(self,item_list): try: counter = len(item_list) if counter == 0: self.log.debug("[%s] Item info chk over with result is 0.", self.item_id ) return False # got item info if not self.initialised: self.track_dict = self.db.item_get_top_n(self.item_id,counter,self.track_type) self.initialised = True for item_info in item_list: if item_info not in self.track_dict: self.track_dict[item_info]=item_info #??? result = self._new_data(item_info) if result == -1: self.log.warn("[%s] Item info already in db, track_time=%s" ,self.name, str(item_info.track_time) ) elif result == 2: self.log.info("[%s] Tracking ended for track_time=%s" ,self.name, str(item_info.track_time) ) #break self.log.debug("[%s] Item info chk over.", self.item_id ) return True except Exception as e: self._error_handle(str(e)) return False def _new_data(self,item): """ result_ 1 new data stored 2 tracking ended -1 already in db """ self.log.info("[%s] New track info coming:[type=%s|delivered=%d] ITEM=%s,%s,%s,%s", self.name, self.track_type, item.is_ended, item.name, item.description, item.location, item.track_time ) if self.db: return self.db.sp_insert_new_item(self.track_type, item.is_ended, item.name, item.track_time, item.description, item.location) else: raise Exception("db did not initialised") def _error_handle(self, msg): if msg: self.log.error("[%s] error happend:%s", self.item_id, msg ) if self.db: return self.db.sp_update_item_status(self.track_type,self.item_id) else: self.log.error("[%s] error happend: db access error at meantime...",self.item_id) if issubclass(msg,BaseException): self.log.exception(msg) def _dump_error(self,item_id,fetch_url,text="",e=None): SAVEPATH=r'ex-pages' """ save exception page""" from time import localtime,time import codecs,os if text is None or text == "": return t=localtime(time()) t_str="%d%d%d%d%d%d"%(t.tm_year,t.tm_mon , t.tm_mday, t.tm_hour, t.tm_min, t.tm_sec) filename=r'./%s/%s_%s.html'%(self.SAVEPATH,item_id,t_str) vavava.util.assure_path("./%s"%self.SAVEPATH) f=codecs.open(filename,"w",'utf-8') file_full_name=os.path.abspath(filename) if f: f.writelines('<!--' + fetch_url + '--!>'+os.linesep) f.write(text) f.close() self.log.error(r"page saved at %s",file_full_name)