def __init__(self): self.failed_ids = [] self.failed_entrys = [] self.table = 'refined_info' self.db = DB_1 self.convertor = FieldConvertor()
class submitter(): """to submit data to resource base""" def __init__(self): self.failed_ids = [] self.failed_entrys = [] self.table = 'refined_info' self.db = DB_1 self.convertor = FieldConvertor() def post(self,url, data): data = urllib.urlencode(data) #enable cookie resp = urllib2.urlopen(url, data, timeout=10) return resp.read() def construct_json_data(self,entry): json_data = {"resourceFk":None,\ "submitWay":"6",\ "partnerId":"1000",\ "specials":None } entry_json = {} #convert the db field to dmg field entry = self.convertor.db_to_dmg(entry) for key,value in entry.items(): #空值的字段不提交到资源库 if value is None: continue if isinstance(value,basestring): if value.strip() == "": continue if isinstance(value,datetime.datetime): value = value.strftime(DATEFORMAT) #print "[%s],corespongding value is [%10s]"%(key,value) entry_json[key] = str(value) #print entry_json json_data['specials'] = entry_json json_data['resourceFk'] = entry_json['info_id'] return json_data def submit_to_RB(self,entrys,posturl=POSTURL): """ submit data to dmg """ failed_count = 0 suc_count = 0 suc_ids = [] failed_ids = [] failed_entrys = [] count = 0 state = "" res = None for entry in entrys[:]: #just for test should delet after testing count += 1 info_id = entry[INFO_ID] entry_json = self.construct_json_data(entry) resjson = {"resjson":json.dumps(entry_json)} try: state =self.post(posturl,resjson) res = json.loads(state)['result'] if (res == 0): LOG.warning("[%s] Submitted Error!" %(info_id)) LOG.warning(state) failed_count += 1 if (res == 1): suc_count += 1 except Exception as e: LOG.error(e) failed_count += 1 LOG.error(state) failed_ids.append(info_id) failed_entrys.append(entry) # LOG.debug("Post one entry into RB the result is: [%s],Suc_Count=\ # [%s],Fail_count=[%s]" % (res,suc_count,failed_count)) if (suc_count + 1) % 1000 == 0 or (failed_count + 1 ) % 100 == 0: LOG.info("Post Entrys To DMG,Suc:[%s] Failed:[%s]" %(suc_count,failed_count)) self.failed_ids = failed_ids self.failed_entrys = failed_entrys LOG.info("Successfully Submitted To DMG [%s],Failed:[%s]" %(suc_count,failed_count)) def resubmit_failed_data(self,url=POSTURL,retry_time=3): """ resubmit the failed data to RB, will try 3 times, before every new try system will sleep 1 minute.Finally after 3 trys,the all-the-time failed data will be submitted with next program call """ db_helper = DBHelper() try_count = 0 while(len(self.failed_ids)>0 and try_count <retry_time): time.sleep(1) LOG.info("\nResubmit Retry Round [%s]." %try_count) LOG.info("[%s] Entrys Are To Re-Submit"%(len(self.failed_ids))) self.submit_to_RB(self.failed_entrys,url) try_count += 1 LOG.info("Finish Resubmitting ,[%s] Remains UnSubmitted." %len(self.failed_ids)) if len(self.failed_ids) != 0: self.write_ids_to_rec() def write_ids_to_rec(self): LOG.info("Recording Failed IDs Into File [%s]" %(FAILED_INFOID_REC)) if not (len(self.failed_ids)>0): return # record the failed_ids to file # check the record file dir if exists ,if not create it dir= os.path.dirname(FAILED_INFOID_REC) file = None if not os.path.isdir(dir): os.mkdir(dir) try: file = open(FAILED_INFOID_REC,'w') file.write(";".join([str(id) for id in self.failed_ids])) LOG.info("Record Successfully Totally [%s] entrys" %(len(self.failed_ids))) except Exception as e: LOG.error(e) finally: if file: file.close() def load_ids_from_rec(self): file = None info_ids = [] try: file = open(FAILED_INFOID_REC,'r') info_ids_str = file.read() info_ids = info_ids_str.split(";") except Exception as e: LOG.error(e) finally: if file:file.close() return info_ids def load_lasttime_failed_entrys(self): LOG.info("Loading lasttime failed entrys") lasttime_info_ids = self.load_ids_from_rec() if len(lasttime_info_ids) == 0: return [] db_helper = DBHelper() entrys = db_helper.get_data_by_infoids(lasttime_info_ids,self.table, self.db,isdict=True) LOG.info("Loaded [%s] lasttime failed entrys" %(len(entrys))) return entrys def deal_submit_req(self,entrys,url=POSTURL,is_retry=True): entrys = list(entrys) failed_entrys = [] if is_retry: failed_entrys = self.load_lasttime_failed_entrys() entrys.extend(failed_entrys) self.submit_to_RB(entrys,url) if len(self.failed_ids) != 0: self.resubmit_failed_data(url)