def insertOne(req): global statsCouch from statsMonitoring import parallel_test updatedDoc=parallel_test( [req,[]] ) docid=req["request_name"] if updatedDoc==None or not len(updatedDoc): print "failed to get anything for",docid return False updatedDoc['_id'] = docid statsCouch.create_file(json.dumps(updatedDoc)) #pprint.pprint(updatedDoc) return docid
def updateOne(docid, match_req_list): global logger if "dmason" in docid: logger.info("Its a dmason request: %s" % (docid)) return False # if "anorkus" not in docid: # print "Not anorkus request: %s" % (docid) # return False global statsCouch try: thisDoc = statsCouch.get_file_info(docid) except Exception as ex: logger.error("There was an access crash with %s. Exception %s" % (docid, str(ex))) return False updatedDoc = copy.deepcopy(thisDoc) if not len(match_req_list): # when there is a fake requests in stats. if docid.startswith('fake_'): match_req_list = [{"request_name": docid, "status": "announced", "type": "ReDigi"}] logger.warning("FAKE %s" % (docid)) return False if len(match_req_list) > 1: logger.warning("More than one! %s" % (docid)) req = match_req_list[0] from statsMonitoring import parallel_test global FORCE # print "##DEBUG## sending a request for parallel_test.\nreq:%s\nupdatedDoc:%s" % (req, updatedDoc) updatedDoc = parallel_test([req, [updatedDoc]], force=FORCE) if updatedDoc == {}: logger.error("Updating %s returned an empty dict" % (docid)) return False if updatedDoc is None: logger.info("Deleting %s" % (docid)) pprint.pprint(thisDoc) statsCouch.delete_file_info(docid, thisDoc['_rev']) return False if worthTheUpdate(updatedDoc, thisDoc): to_get = ['pdmv_monitor_time', 'pdmv_evts_in_DAS', 'pdmv_open_evts_in_DAS', 'pdmv_dataset_statuses'] if 'pdmv_monitor_history' not in updatedDoc: # do the migration thisDoc_bis = statsCouch.get_file_info_withrev(docid) revs = thisDoc_bis['_revs_info'] history = [] for rev in revs: try: nextOne = statsCouch.get_file_info_rev(docid, rev['rev']) except: continue history.append({}) for g in to_get: if g not in nextOne: continue history[-1][g] = copy.deepcopy(nextOne[g]) updatedDoc['pdmv_monitor_history'] = history if 'pdmv_monitor_history' in updatedDoc: rev = {} for g in to_get: if g not in updatedDoc: continue rev[g] = copy.deepcopy(updatedDoc[g]) old_history = copy.deepcopy(updatedDoc['pdmv_monitor_history'][0]) new_history = copy.deepcopy(rev) del old_history["pdmv_monitor_time"] # compare history without monitor time del new_history["pdmv_monitor_time"] if not compare_dictionaries(old_history, new_history): # it is worth to fill history updatedDoc['pdmv_monitor_history'].insert(0, rev) while {} in updatedDoc['pdmv_monitor_history']: updatedDoc['pdmv_monitor_history'].remove({}) try: statsCouch.update_file(docid, json.dumps(updatedDoc)) logger.info("Something has changed %s" % (docid)) return docid except Exception as ex: logger.error("Failed to update %s. Exception %s" % (docid, str(ex))) return False else: logger.info("%s not worth the update (nothing changed)" % (docid)) return False
def updateOne(docid, req_list): if "dmason" in docid: print "Its a dmason request: %s" % (docid) return False global statsCouch match_req_list=filter (lambda r: r["request_name"]==docid, req_list) try: thisDoc=statsCouch.get_file_info(docid) except: print "There was an access crash with",docid return False today_str = "{d.year}-{d.month}-{d.day}".format(d=date.today()) today_str_split = today_str.split("-") year_ago_str = "%s-%s-%s" % (int(today_str_split[0])-1, today_str_split[1], today_str_split[2]) year_ago_split = year_ago_str.split("-") print "Working on: %s" % (docid) try: if int(thisDoc["pdmv_submission_date"]) < int(datelist_to_str(year_ago_split)): print "Document: %s is too old (> 1year) to be updated" % (docid) return False except Exception as ex: print "issues with: %s" % (docid) raise ValueError("issues with converting submission_date to int: %s" % (docid)) updatedDoc=copy.deepcopy(thisDoc) if not len(match_req_list): ## when there is a fake requests in stats. if docid.startswith('fake_'): match_req_list=[{"request_name":docid, "status":"announced","type":"ReDigi"}] print "FAKE" return False if len(match_req_list)>1 : print "more than one !!" req=match_req_list[0] from statsMonitoring import parallel_test global FORCE updatedDoc=parallel_test( [req,[updatedDoc]] ,force=FORCE) if updatedDoc=={}: print "updating",docid,"returned an empty dict" return False if updatedDoc==None: print "deleting",docid pprint.pprint(thisDoc) statsCouch.delete_file_info(docid,thisDoc['_rev']) return False if worthTheUpdate(updatedDoc,thisDoc): to_get=['pdmv_monitor_time','pdmv_evts_in_DAS','pdmv_open_evts_in_DAS','pdmv_dataset_statuses'] if 'pdvm_monitor_history' in updatedDoc: updatedDoc['pdvm_monitor_history'] == updatedDoc.pop( 'pdvm_monitor_history' ) if not 'pdmv_monitor_history' in updatedDoc: ## do the migration thisDoc_bis = statsCouch.get_file_info_withrev(docid) revs = thisDoc_bis['_revs_info'] history=[] for rev in revs: try: nextOne=statsCouch.get_file_info_rev(docid, rev['rev']) except: continue history.append({}) for g in to_get: if not g in nextOne: continue history[-1][g] = copy.deepcopy(nextOne[g]) updatedDoc['pdmv_monitor_history'] = history if 'pdmv_monitor_history' in updatedDoc: rev = {} for g in to_get: if not g in updatedDoc: continue rev[g] = copy.deepcopy(updatedDoc[g]) old_history = copy.deepcopy(updatedDoc['pdmv_monitor_history'][0]) new_history = copy.deepcopy(rev) del old_history["pdmv_monitor_time"] ##compare history without monitor time del new_history["pdmv_monitor_time"] if not compare_dictionaries(old_history, new_history): # it is worth to fill history updatedDoc['pdmv_monitor_history'].insert(0, rev) try: statsCouch.update_file(docid,json.dumps(updatedDoc)) print docid,"something has changed" return docid except: print "Failed to update",docid print traceback.format_exc() return False else: print docid,"nothing changed" return False
def updateOne(docid,req_list): global statsCouch match_req_list=filter (lambda r: r["request_name"]==docid, req_list) try: thisDoc=statsCouch.get_file_info(docid) except: print "There was an access crash with",docid return False updatedDoc=copy.deepcopy(thisDoc) #print "before update" #pprint.pprint(thisDoc) if not len(match_req_list): ## when there is a fake requests in stats. if docid.startswith('fake_'): match_req_list=[{"request_name":docid, "status":"announced","type":"ReDigi"}] print "FAKE" return False if len(match_req_list)>1 : print "more than one !!" req=match_req_list[0] from statsMonitoring import parallel_test global FORCE updatedDoc=parallel_test( [req,[updatedDoc]] ,force=FORCE) #print "updated" #pprint.pprint(updatedDoc) if updatedDoc=={}: print "updating",docid,"returned an empty dict" return False if updatedDoc==None: print "deleting",docid pprint.pprint(thisDoc) statsCouch.delete_file_info(docid,thisDoc['_rev']) return False #if pprint.pformat(updatedDoc)!=pprint.pformat(thisDoc): if worthTheUpdate(updatedDoc,thisDoc): to_get=['pdmv_monitor_time','pdmv_evts_in_DAS','pdmv_open_evts_in_DAS','pdmv_dataset_statuses'] if 'pdvm_monitor_history' in updatedDoc: updatedDoc['pdvm_monitor_history'] == updatedDoc.pop( 'pdvm_monitor_history' ) if not 'pdmv_monitor_history' in updatedDoc: ## do the migration thisDoc_bis = statsCouch.get_file_info_withrev(docid) revs = thisDoc_bis['_revs_info'] history=[] for rev in revs: try: nextOne=statsCouch.get_file_info_rev(docid, rev['rev']) except: continue history.append({}) for g in to_get: if not g in nextOne: continue history[-1][g] = copy.deepcopy(nextOne[g]) updatedDoc['pdmv_monitor_history'] = history if 'pdmv_monitor_history' in updatedDoc: rev = {} for g in to_get: if not g in updatedDoc: continue rev[g] = copy.deepcopy(updatedDoc[g]) old_history = copy.deepcopy(updatedDoc['pdmv_monitor_history'][0]) new_history = copy.deepcopy(rev) del old_history["pdmv_monitor_time"] ##compare history without monitor time del new_history["pdmv_monitor_time"] if not compare_dictionaries(old_history, new_history): # it is worth to fill history updatedDoc['pdmv_monitor_history'].insert(0, rev) try: statsCouch.update_file(docid,json.dumps(updatedDoc)) #pprint.pprint(updatedDoc) #print updatedDoc['pdmv_at_T2'] print docid,"something has changed" return docid except: print "Failed to update",docid print traceback.format_exc() return False else: print docid,"nothing changed" return False