def __init__(self):
self.failed_ids = []
self.failed_entrys = []
self.table = 'refined_info'
self.db = DB_1
self.convertor = FieldConvertor()
class submitter():
"""to submit data to resource base"""
def __init__(self):
self.failed_ids = []
self.failed_entrys = []
self.table = 'refined_info'
self.db = DB_1
self.convertor = FieldConvertor()
def post(self,url, data):
data = urllib.urlencode(data)
#enable cookie
resp = urllib2.urlopen(url, data, timeout=10)
return resp.read()
def construct_json_data(self,entry):
json_data = {"resourceFk":None,\
"submitWay":"6",\
"partnerId":"1000",\
"specials":None
}
entry_json = {}
#convert the db field to dmg field
entry = self.convertor.db_to_dmg(entry)
for key,value in entry.items():
#空值的字段不提交到资源库
if value is None:
continue
if isinstance(value,basestring):
if value.strip() == "":
continue
if isinstance(value,datetime.datetime):
value = value.strftime(DATEFORMAT)
#print "[%s],corespongding value is [%10s]"%(key,value)
entry_json[key] = str(value)
#print entry_json
json_data['specials'] = entry_json
json_data['resourceFk'] = entry_json['info_id']
return json_data
def submit_to_RB(self,entrys,posturl=POSTURL):
""" submit data to dmg
"""
failed_count = 0
suc_count = 0
suc_ids = []
failed_ids = []
failed_entrys = []
count = 0
state = ""
res = None
for entry in entrys[:]:
#just for test should delet after testing
count += 1
info_id = entry[INFO_ID]
entry_json = self.construct_json_data(entry)
resjson = {"resjson":json.dumps(entry_json)}
try:
state =self.post(posturl,resjson)
res = json.loads(state)['result']
if (res == 0):
LOG.warning("[%s] Submitted Error!" %(info_id))
LOG.warning(state)
failed_count += 1
if (res == 1): suc_count += 1
except Exception as e:
LOG.error(e)
failed_count += 1
LOG.error(state)
failed_ids.append(info_id)
failed_entrys.append(entry)
# LOG.debug("Post one entry into RB the result is: [%s],Suc_Count=\
# [%s],Fail_count=[%s]" % (res,suc_count,failed_count))
if (suc_count + 1) % 1000 == 0 or (failed_count + 1 ) % 100 == 0:
LOG.info("Post Entrys To DMG,Suc:[%s] Failed:[%s]"
%(suc_count,failed_count))
self.failed_ids = failed_ids
self.failed_entrys = failed_entrys
LOG.info("Successfully Submitted To DMG [%s],Failed:[%s]"
%(suc_count,failed_count))
def resubmit_failed_data(self,url=POSTURL,retry_time=3):
""" resubmit the failed data to RB, will try 3 times, before every new
try system will sleep 1 minute.Finally after 3 trys,the all-the-time
failed data will be submitted with next program call
"""
db_helper = DBHelper()
try_count = 0
while(len(self.failed_ids)>0 and try_count <retry_time):
time.sleep(1)
LOG.info("\nResubmit Retry Round [%s]." %try_count)
LOG.info("[%s] Entrys Are To Re-Submit"%(len(self.failed_ids)))
self.submit_to_RB(self.failed_entrys,url)
try_count += 1
LOG.info("Finish Resubmitting ,[%s] Remains UnSubmitted."
%len(self.failed_ids))
if len(self.failed_ids) != 0:
self.write_ids_to_rec()
def write_ids_to_rec(self):
LOG.info("Recording Failed IDs Into File [%s]" %(FAILED_INFOID_REC))
if not (len(self.failed_ids)>0):
return
# record the failed_ids to file
# check the record file dir if exists ,if not create it
dir= os.path.dirname(FAILED_INFOID_REC)
file = None
if not os.path.isdir(dir):
os.mkdir(dir)
try:
file = open(FAILED_INFOID_REC,'w')
file.write(";".join([str(id) for id in self.failed_ids]))
LOG.info("Record Successfully Totally [%s] entrys"
%(len(self.failed_ids)))
except Exception as e:
LOG.error(e)
finally:
if file: file.close()
def load_ids_from_rec(self):
file = None
info_ids = []
try:
file = open(FAILED_INFOID_REC,'r')
info_ids_str = file.read()
info_ids = info_ids_str.split(";")
except Exception as e:
LOG.error(e)
finally:
if file:file.close()
return info_ids
def load_lasttime_failed_entrys(self):
LOG.info("Loading lasttime failed entrys")
lasttime_info_ids = self.load_ids_from_rec()
if len(lasttime_info_ids) == 0:
return []
db_helper = DBHelper()
entrys = db_helper.get_data_by_infoids(lasttime_info_ids,self.table,
self.db,isdict=True)
LOG.info("Loaded [%s] lasttime failed entrys" %(len(entrys)))
return entrys
def deal_submit_req(self,entrys,url=POSTURL,is_retry=True):
entrys = list(entrys)
failed_entrys = []
if is_retry:
failed_entrys = self.load_lasttime_failed_entrys()
entrys.extend(failed_entrys)
self.submit_to_RB(entrys,url)
if len(self.failed_ids) != 0:
self.resubmit_failed_data(url)
