def make_key(entry, key_fields=None): """ based on the key_fields generate key for a entry""" if key_fields is None: LOG.warning("Compared Fileds Are None!") key_fields = [MEETING_TIME, MEETING_LOCATION, RECRUIT_URL] key = "" for field in key_fields: if entry.get(field) is None: continue key += str(entry[field]) return key
def submit_to_RB(self,entrys,posturl=POSTURL): """ submit data to dmg """ failed_count = 0 suc_count = 0 suc_ids = [] failed_ids = [] failed_entrys = [] count = 0 state = "" res = None for entry in entrys[:]: #just for test should delet after testing count += 1 info_id = entry[INFO_ID] entry_json = self.construct_json_data(entry) resjson = {"resjson":json.dumps(entry_json)} try: state =self.post(posturl,resjson) res = json.loads(state)['result'] if (res == 0): LOG.warning("[%s] Submitted Error!" %(info_id)) LOG.warning(state) failed_count += 1 if (res == 1): suc_count += 1 except Exception as e: LOG.error(e) failed_count += 1 LOG.error(state) failed_ids.append(info_id) failed_entrys.append(entry) # LOG.debug("Post one entry into RB the result is: [%s],Suc_Count=\ # [%s],Fail_count=[%s]" % (res,suc_count,failed_count)) if (suc_count + 1) % 1000 == 0 or (failed_count + 1 ) % 100 == 0: LOG.info("Post Entrys To DMG,Suc:[%s] Failed:[%s]" %(suc_count,failed_count)) self.failed_ids = failed_ids self.failed_entrys = failed_entrys LOG.info("Successfully Submitted To DMG [%s],Failed:[%s]" %(suc_count,failed_count))