def insertOne(req):
global statsCouch
from statsMonitoring import parallel_test
updatedDoc=parallel_test( [req,[]] )
docid=req["request_name"]
if updatedDoc==None or not len(updatedDoc):
print "failed to get anything for",docid
return False
updatedDoc['_id'] = docid
statsCouch.create_file(json.dumps(updatedDoc))
#pprint.pprint(updatedDoc)
return docid
def updateOne(docid, match_req_list):
global logger
if "dmason" in docid:
logger.info("Its a dmason request: %s" % (docid))
return False
# if "anorkus" not in docid:
# print "Not anorkus request: %s" % (docid)
# return False
global statsCouch
try:
thisDoc = statsCouch.get_file_info(docid)
except Exception as ex:
logger.error("There was an access crash with %s. Exception %s" % (docid, str(ex)))
return False
updatedDoc = copy.deepcopy(thisDoc)
if not len(match_req_list):
# when there is a fake requests in stats.
if docid.startswith('fake_'):
match_req_list = [{"request_name": docid, "status": "announced", "type": "ReDigi"}]
logger.warning("FAKE %s" % (docid))
return False
if len(match_req_list) > 1:
logger.warning("More than one! %s" % (docid))
req = match_req_list[0]
from statsMonitoring import parallel_test
global FORCE
# print "##DEBUG## sending a request for parallel_test.\nreq:%s\nupdatedDoc:%s" % (req, updatedDoc)
updatedDoc = parallel_test([req, [updatedDoc]], force=FORCE)
if updatedDoc == {}:
logger.error("Updating %s returned an empty dict" % (docid))
return False
if updatedDoc is None:
logger.info("Deleting %s" % (docid))
pprint.pprint(thisDoc)
statsCouch.delete_file_info(docid, thisDoc['_rev'])
return False
if worthTheUpdate(updatedDoc, thisDoc):
to_get = ['pdmv_monitor_time',
'pdmv_evts_in_DAS',
'pdmv_open_evts_in_DAS',
'pdmv_dataset_statuses']
if 'pdmv_monitor_history' not in updatedDoc:
# do the migration
thisDoc_bis = statsCouch.get_file_info_withrev(docid)
revs = thisDoc_bis['_revs_info']
history = []
for rev in revs:
try:
nextOne = statsCouch.get_file_info_rev(docid, rev['rev'])
except:
continue
history.append({})
for g in to_get:
if g not in nextOne:
continue
history[-1][g] = copy.deepcopy(nextOne[g])
updatedDoc['pdmv_monitor_history'] = history
if 'pdmv_monitor_history' in updatedDoc:
rev = {}
for g in to_get:
if g not in updatedDoc:
continue
rev[g] = copy.deepcopy(updatedDoc[g])
old_history = copy.deepcopy(updatedDoc['pdmv_monitor_history'][0])
new_history = copy.deepcopy(rev)
del old_history["pdmv_monitor_time"] # compare history without monitor time
del new_history["pdmv_monitor_time"]
if not compare_dictionaries(old_history, new_history): # it is worth to fill history
updatedDoc['pdmv_monitor_history'].insert(0, rev)
while {} in updatedDoc['pdmv_monitor_history']:
updatedDoc['pdmv_monitor_history'].remove({})
try:
statsCouch.update_file(docid, json.dumps(updatedDoc))
logger.info("Something has changed %s" % (docid))
return docid
except Exception as ex:
logger.error("Failed to update %s. Exception %s" % (docid, str(ex)))
return False
else:
logger.info("%s not worth the update (nothing changed)" % (docid))
return False
def updateOne(docid, req_list):
if "dmason" in docid:
print "Its a dmason request: %s" % (docid)
return False
global statsCouch
match_req_list=filter (lambda r: r["request_name"]==docid, req_list)
try:
thisDoc=statsCouch.get_file_info(docid)
except:
print "There was an access crash with",docid
return False
today_str = "{d.year}-{d.month}-{d.day}".format(d=date.today())
today_str_split = today_str.split("-")
year_ago_str = "%s-%s-%s" % (int(today_str_split[0])-1, today_str_split[1], today_str_split[2])
year_ago_split = year_ago_str.split("-")
print "Working on: %s" % (docid)
try:
if int(thisDoc["pdmv_submission_date"]) < int(datelist_to_str(year_ago_split)):
print "Document: %s is too old (> 1year) to be updated" % (docid)
return False
except Exception as ex:
print "issues with: %s" % (docid)
raise ValueError("issues with converting submission_date to int: %s" % (docid))
updatedDoc=copy.deepcopy(thisDoc)
if not len(match_req_list):
## when there is a fake requests in stats.
if docid.startswith('fake_'):
match_req_list=[{"request_name":docid, "status":"announced","type":"ReDigi"}]
print "FAKE"
return False
if len(match_req_list)>1 :
print "more than one !!"
req=match_req_list[0]
from statsMonitoring import parallel_test
global FORCE
updatedDoc=parallel_test( [req,[updatedDoc]] ,force=FORCE)
if updatedDoc=={}:
print "updating",docid,"returned an empty dict"
return False
if updatedDoc==None:
print "deleting",docid
pprint.pprint(thisDoc)
statsCouch.delete_file_info(docid,thisDoc['_rev'])
return False
if worthTheUpdate(updatedDoc,thisDoc):
to_get=['pdmv_monitor_time','pdmv_evts_in_DAS','pdmv_open_evts_in_DAS','pdmv_dataset_statuses']
if 'pdvm_monitor_history' in updatedDoc:
updatedDoc['pdvm_monitor_history'] == updatedDoc.pop( 'pdvm_monitor_history' )
if not 'pdmv_monitor_history' in updatedDoc:
## do the migration
thisDoc_bis = statsCouch.get_file_info_withrev(docid)
revs = thisDoc_bis['_revs_info']
history=[]
for rev in revs:
try:
nextOne=statsCouch.get_file_info_rev(docid, rev['rev'])
except:
continue
history.append({})
for g in to_get:
if not g in nextOne: continue
history[-1][g] = copy.deepcopy(nextOne[g])
updatedDoc['pdmv_monitor_history'] = history
if 'pdmv_monitor_history' in updatedDoc:
rev = {}
for g in to_get:
if not g in updatedDoc: continue
rev[g] = copy.deepcopy(updatedDoc[g])
old_history = copy.deepcopy(updatedDoc['pdmv_monitor_history'][0])
new_history = copy.deepcopy(rev)
del old_history["pdmv_monitor_time"] ##compare history without monitor time
del new_history["pdmv_monitor_time"]
if not compare_dictionaries(old_history, new_history): # it is worth to fill history
updatedDoc['pdmv_monitor_history'].insert(0, rev)
try:
statsCouch.update_file(docid,json.dumps(updatedDoc))
print docid,"something has changed"
return docid
except:
print "Failed to update",docid
print traceback.format_exc()
return False
else:
print docid,"nothing changed"
return False
def updateOne(docid,req_list):
global statsCouch
match_req_list=filter (lambda r: r["request_name"]==docid, req_list)
try:
thisDoc=statsCouch.get_file_info(docid)
except:
print "There was an access crash with",docid
return False
updatedDoc=copy.deepcopy(thisDoc)
#print "before update"
#pprint.pprint(thisDoc)
if not len(match_req_list):
## when there is a fake requests in stats.
if docid.startswith('fake_'):
match_req_list=[{"request_name":docid, "status":"announced","type":"ReDigi"}]
print "FAKE"
return False
if len(match_req_list)>1 :
print "more than one !!"
req=match_req_list[0]
from statsMonitoring import parallel_test
global FORCE
updatedDoc=parallel_test( [req,[updatedDoc]] ,force=FORCE)
#print "updated"
#pprint.pprint(updatedDoc)
if updatedDoc=={}:
print "updating",docid,"returned an empty dict"
return False
if updatedDoc==None:
print "deleting",docid
pprint.pprint(thisDoc)
statsCouch.delete_file_info(docid,thisDoc['_rev'])
return False
#if pprint.pformat(updatedDoc)!=pprint.pformat(thisDoc):
if worthTheUpdate(updatedDoc,thisDoc):
to_get=['pdmv_monitor_time','pdmv_evts_in_DAS','pdmv_open_evts_in_DAS','pdmv_dataset_statuses']
if 'pdvm_monitor_history' in updatedDoc:
updatedDoc['pdvm_monitor_history'] == updatedDoc.pop( 'pdvm_monitor_history' )
if not 'pdmv_monitor_history' in updatedDoc:
## do the migration
thisDoc_bis = statsCouch.get_file_info_withrev(docid)
revs = thisDoc_bis['_revs_info']
history=[]
for rev in revs:
try:
nextOne=statsCouch.get_file_info_rev(docid, rev['rev'])
except:
continue
history.append({})
for g in to_get:
if not g in nextOne: continue
history[-1][g] = copy.deepcopy(nextOne[g])
updatedDoc['pdmv_monitor_history'] = history
if 'pdmv_monitor_history' in updatedDoc:
rev = {}
for g in to_get:
if not g in updatedDoc: continue
rev[g] = copy.deepcopy(updatedDoc[g])
old_history = copy.deepcopy(updatedDoc['pdmv_monitor_history'][0])
new_history = copy.deepcopy(rev)
del old_history["pdmv_monitor_time"] ##compare history without monitor time
del new_history["pdmv_monitor_time"]
if not compare_dictionaries(old_history, new_history): # it is worth to fill history
updatedDoc['pdmv_monitor_history'].insert(0, rev)
try:
statsCouch.update_file(docid,json.dumps(updatedDoc))
#pprint.pprint(updatedDoc)
#print updatedDoc['pdmv_at_T2']
print docid,"something has changed"
return docid
except:
print "Failed to update",docid
print traceback.format_exc()
return False
else:
print docid,"nothing changed"
return False