def make_key(entry, key_fields=None):
""" based on the key_fields generate key for a entry"""
if key_fields is None:
LOG.warning("Compared Fileds Are None!")
key_fields = [MEETING_TIME, MEETING_LOCATION, RECRUIT_URL]
key = ""
for field in key_fields:
if entry.get(field) is None:
continue
key += str(entry[field])
return key
def submit_to_RB(self,entrys,posturl=POSTURL):
""" submit data to dmg
"""
failed_count = 0
suc_count = 0
suc_ids = []
failed_ids = []
failed_entrys = []
count = 0
state = ""
res = None
for entry in entrys[:]:
#just for test should delet after testing
count += 1
info_id = entry[INFO_ID]
entry_json = self.construct_json_data(entry)
resjson = {"resjson":json.dumps(entry_json)}
try:
state =self.post(posturl,resjson)
res = json.loads(state)['result']
if (res == 0):
LOG.warning("[%s] Submitted Error!" %(info_id))
LOG.warning(state)
failed_count += 1
if (res == 1): suc_count += 1
except Exception as e:
LOG.error(e)
failed_count += 1
LOG.error(state)
failed_ids.append(info_id)
failed_entrys.append(entry)
# LOG.debug("Post one entry into RB the result is: [%s],Suc_Count=\
# [%s],Fail_count=[%s]" % (res,suc_count,failed_count))
if (suc_count + 1) % 1000 == 0 or (failed_count + 1 ) % 100 == 0:
LOG.info("Post Entrys To DMG,Suc:[%s] Failed:[%s]"
%(suc_count,failed_count))
self.failed_ids = failed_ids
self.failed_entrys = failed_entrys
LOG.info("Successfully Submitted To DMG [%s],Failed:[%s]"
%(suc_count,failed_count))
