def delnew(sess, clusterurl, qitem):
try:
logging.warn("{View Migrater} DeleteTemps Step (New): Starting")
newurl = clusterurl + '/' + qitem['db'] + '/_design/' + qitem[
'ddid'] + '_NEW'
new = '_design/' + qitem['ddid'] + '_NEW'
newrev = api_utils.get_doc_rev(sess, clusterurl + '/' + qitem['db'],
new)
delnew_response = sess.delete(newurl + '?rev=' + newrev)
if delnew_response is None:
logging.warn(
"{View Migrater} Deltemps Step (New) Error: No response")
update_queue_entry(sess, clusterurl, qitem, 'apimigratequeue',
"failed", "Deltemps Step Error : No response",
None)
return False
elif delnew_response.status_code > 250:
logging.warn(
"{View Migrater} Deltemps Step Session (New) Error : " +
str(delnew_response.status_code))
update_queue_entry(
sess, clusterurl, qitem, 'apimigratequeue', "failed",
"Deltemps Step Session (New) Error : " +
str(delnew_response.status_code), None)
return False
else:
return True
except Exception as e:
logging.warn("{View Migrater} Deltemps Step (New) Error : " + str(e))
return False
def copyold(sess, clusterurl, qitem):
try:
fromurl = clusterurl + '/' + qitem['db'] + '/_design/' + qitem['ddid']
old = '_design/' + qitem['ddid'] + '_OLD'
logging.warn("{View Migrater} CopyOld Step : Starting")
copyold_response = sess.request('COPY',
fromurl,
headers={'Destination': old},
verify=False)
if copyold_response is None:
logging.warn("{View Migrater} CopyOld Step Error: No response")
update_queue_entry(sess, clusterurl, qitem, 'apimigratequeue',
"failed", "CopyOld Step Error : No response",
None)
return False
elif copyold_response.status_code > 250:
logging.warn("{View Migrater} CopyOld Step Session Error : " +
str(copyold_response.status_code))
update_queue_entry(
sess, clusterurl, qitem, 'apimigratequeue', "failed",
"CopyOld Step Session Error : " +
str(copyold_response.status_code), None)
return False
else:
return True
except Exception as e:
logging.warn("{View Migrater} CopyOld Step Error : " + str(e))
return False
def delold(sess, clusterurl, qitem):
try:
logging.warn("{View Migrater} DeleteTemps Step (Old): Starting")
old = '_design/' + qitem['ddid'] + '_OLD'
oldurl = clusterurl + '/' + qitem['db'] + '/_design/' + qitem[
'ddid'] + '_OLD'
oldrev = api_utils.get_doc_rev(sess, clusterurl + '/' + qitem['db'],
old)
delold_response = sess.delete(oldurl + '?rev=' + oldrev)
if delold_response is None:
logging.warn(
"{View Migrater} Deltemps Step (Old) Error: No response")
update_queue_entry(sess, clusterurl, qitem, 'apimigratequeue',
"failed",
"Deltemps Step (Old) Error : No response", None)
return False
elif delold_response.status_code > 250:
logging.warn(
"{View Migrater} Deltemps Step (Old) Session Error : " +
str(delold_response.status_code))
update_queue_entry(
sess, clusterurl, qitem, 'apimigratequeue', "failed",
"Deltemps Step (Old) Session Error : " +
str(delold_response.status_code), None)
return False
else:
return True
except Exception as e:
logging.warn("{View Migrater} Deltemps Step (Old) Error : " + str(e))
return False
def copyfinal(sess, clusterurl, qitem):
try:
# note issue 97653 : Destination should not include the db or a leading /
newurl = clusterurl + '/' + qitem['db'] + '/_design/' + qitem[
'ddid'] + '_NEW'
finalrev = api_utils.get_doc_rev(sess, clusterurl + '/' + qitem['db'],
'_design/' + qitem['ddid'])
final = '_design/' + qitem['ddid'] + '?rev=' + finalrev
logging.warn("{View Migrater} CopyFinal Step : Starting")
copyfinal_response = sess.request('COPY',
newurl,
headers={'Destination': final},
verify=False)
if copyfinal_response is None:
logging.warn("{View Migrater} CopyFinal Step Error: No response")
update_queue_entry(sess, clusterurl, qitem, 'apimigratequeue',
"failed", "CopyFinal Step Error : No response",
None)
return False
elif copyfinal_response.status_code > 250:
logging.warn("{View Migrater} CopyFinal Step Session Error : " +
str(copyfinal_response.status_code))
update_queue_entry(
sess, clusterurl, qitem, 'apimigratequeue', "failed",
"CopyFinal Step Session Error : " +
str(copyfinal_response.status_code), None)
return False
else:
return True
except Exception as e:
logging.warn("{View Migrater} CopyFinal Step Error : " + str(e))
return False
def putddoc(sess, clusterurl, qitem, ddid, stepmarker):
try:
url = clusterurl + '/' + qitem['db'] + '/_design/' + ddid
logging.warn("{View Migrater} " + str(stepmarker) + " : Starting")
put_response = sess.put(url,
data=json.dumps(qitem['ddoc']),
headers={'content-type': 'application/json'})
if put_response is None:
logging.warn("{View Migrater} " + str(stepmarker) +
" Error: No response")
update_queue_entry(sess, clusterurl, qitem, 'apimigratequeue',
"failed",
str(stepmarker) + " Error : No Response", None)
return False
elif put_response.status_code > 250:
logging.warn("{View Migrater} " + str(stepmarker) +
" Session Error : " + str(put_response.status_code))
update_queue_entry(
sess, clusterurl, qitem, 'apimigratequeue', "failed",
str(stepmarker) + " Session Error : " +
str(put_response.status_code), None)
return False
else:
return True
except Exception as e:
logging.warn("{View Migrater} " + str(stepmarker) + " Error : " +
str(e))
return False
def execute_index_create(sess, clusterurl, qitem):
try:
postindex(sess, clusterurl, qitem, 'Create')
except Exception as e:
logging.warn("{Index Migrater} Execute Create Unexpected Error : " +
str(e))
update_queue_entry(sess, clusterurl, qitem, 'apimigratequeue',
"failed", "Create Unexpected Error : " + str(e),
None)
return False
def postindex(sess, clusterurl, qitem, stepmarker):
try:
url = clusterurl + '/' + qitem['db'] + '/_index'
logging.warn("{Index Migrater} " + str(stepmarker) + " : Starting")
post_response = sess.post(url,
data=json.dumps(qitem['index']),
headers={'content-type': 'application/json'})
if post_response is None:
logging.warn("{Index Migrater} " + str(stepmarker) +
" Error: No response")
update_queue_entry(sess, clusterurl, qitem, 'apimigratequeue',
"failed",
str(stepmarker) + " Error : No Response", None)
return False
elif post_response.status_code > 250:
logging.warn("{Index Migrater} " + str(stepmarker) +
" Session Error : " + str(put_response.status_code))
update_queue_entry(
sess, clusterurl, qitem, 'apimigratequeue', "failed",
str(stepmarker) + " Session Error : " +
str(put_response.status_code), None)
return False
else:
data = post_response.json()
if 'result' in data:
if data['result'] == 'created':
update_queue_entry(sess, clusterurl, qitem,
'apimigratequeue', "completed", data,
None)
logging.warn("{Index Migrater} : Completed Successully")
else:
update_queue_entry(sess, clusterurl, qitem,
'apimigratequeue', "failed", data, None)
logging.warn("{Index Migrater} : Failed")
else:
update_queue_entry(sess, clusterurl, qitem, 'apimigratequeue',
"failed", "Empty result", None)
logging.warn("{Index Migrater} : Failed")
return True
except Exception as e:
logging.warn("{Index Migrater} " + str(stepmarker) + " Error : " +
str(e))
return False
def execute_moveandswitch(sess, clusterurl, index_wait_timeout, qitem):
try:
if copyold(sess, clusterurl, qitem):
time.sleep(2.0)
if putddoc(sess, clusterurl, qitem,
str(qitem['ddid']) + '_NEW', 'PutNew'):
time.sleep(2.0)
testview = getview(sess, clusterurl, qitem,
str(qitem['ddid']) + '_NEW')
if testview is not None:
time.sleep(30.0)
if wait_view_complete(sess, clusterurl, qitem,
str(qitem['ddid'] + '_NEW'),
testview, index_wait_timeout,
'PutNew'):
time.sleep(10.0)
if copyfinal(sess, clusterurl, qitem):
time.sleep(5.0)
if delnew(sess, clusterurl, qitem):
time.sleep(2.0)
if delold(sess, clusterurl, qitem):
logging.warn(
"{View Migrater} Complete for Entry ["
+ str(qitem['_id']) + "]")
update_queue_entry(sess, clusterurl, qitem,
'apimigratequeue',
"success", None, None)
return True
time.sleep(5.0)
logging.warn("{View Migrater} Failures during Migration for Entry [" +
str(qitem['_id']) + "]")
logging.warn("{View Migrater} Cleanup : Starting")
delnew(sess, clusterurl, qitem)
delold(sess, clusterurl, qitem)
logging.warn("{View Migrater} Cleanup Complete for Entry [" +
str(qitem['_id']) + "]")
return False
except Exception as e:
logging.warn(
"{View Migrater} Execute Move and Switch Unexpected Error : " +
str(e))
return False
def execute_create(sess, clusterurl, qitem):
try:
if putddoc(sess, clusterurl, qitem, str(qitem['ddid']), 'Create'):
time.sleep(2.0)
testview = getview(sess, clusterurl, qitem, str(qitem['ddid']))
if testview is not None:
if wait_view_complete(sess, clusterurl, qitem,
str(qitem['ddid']), testview,
index_wait_timeout, 'Create'):
logging.warn("{View Migrater} Complete for Entry [" +
str(qitem['_id']) + "]")
update_queue_entry(sess, clusterurl, qitem,
'apimigratequeue', "success", None,
None)
return True
else:
update_queue_entry(sess, clusterurl, qitem, 'apimigratequeue',
"failed",
"Create error : no view found inside doc",
None)
time.sleep(1.0)
logging.warn("{View Migrater} Failures during Migration for Entry [" +
str(qitem['_id']) + "]")
return False
except Exception as e:
logging.warn("{View Migrater} Execute Create Unexpected Error : " +
str(e))
update_queue_entry(sess, clusterurl, qitem, 'apimigratequeue',
"failed", "Create Unexpected Error : " + str(e),
None)
return False
def acquire_ownership_item(sess, clusterurl, qitem):
try:
itemurl = clusterurl + '/apimigratequeue/' + qitem['_id']
status_response = api_utils.get_with_retries(sess, itemurl, 2, None)
if status_response is None:
logging.warn(
"{View Migrater} Acquire Ownership Error for Entry [" +
str(qitem['_id']) + "] : No response")
return False
elif status_response.status_code > 250:
logging.warn(
"{View Migrater} Acquire Ownership Session for Entry [" +
str(qitem['_id']) + "] Error : " +
str(status_response.status_code))
return False
else:
data = status_response.json()
if 'status' in data:
if data['status'] == 'submitted':
if update_queue_entry(sess, clusterurl, qitem,
'apimigratequeue', "processing",
None, None):
logging.warn(
"{View Migrater} Acquire Ownership Success for Entry ["
+ str(qitem['_id']) + "] ")
return True
else:
logging.warn(
"{View Migrater} Acquire Ownership Failed for Entry ["
+ str(qitem['_id']) + "] : could not update item")
return False
else:
logging.warn(
"{View Migrater} Acquire Ownership Failed for Entry ["
+ str(qitem['_id']) +
"] : item owned by another worker")
return False
else:
logging.warn(
"{View Migrater} Acquire Ownership Failed for Entry [" +
str(qitem['_id']) + "] : Failed to read item")
return False
except Exception as e:
logging.warn(
"{View Migrater} Acquire Ownership Unexpected Error for Entry [" +
str(qitem['_id']) + "] : " + str(e))
return False
while True:
sess, serr, scode = api_utils.create_adminsession(
clusterurl, adminuser, adminpwd)
if serr != None:
logging.warn(str(serr) + '/' + str(scode))
else:
qdata, qcode = process_queue(sess, clusterurl,
'apiperfagentqueue')
if qdata is not None:
if not 'error' in qdata:
for qitem in qdata:
logging.warn(
'{perfagent worker} Executing Performance Agent Processing for Entry ['
+ qitem['_id'] + ']')
update_queue_entry(sess, clusterurl, qitem,
'apiperfagentqueue',
"processing", None, None)
if 'opts' in qitem:
opts = qitem['opts']
s_url, s_credentials, s_username, s_password, p_url = process_connection_info(
opts['connectioninfo'])
if s_url is not None and s_credentials is not None and s_username is not None and s_password is not None:
response = execute_collect(opts['scope'],s_url,s_credentials,s_username,s_password,p_url,opts['certverif'],\
opts['inputlogfile'],opts['thresholdsfile'],opts['eventsexclusionsfile'],opts['statsexclusionsfile'],\
opts['fromtime'],opts['totime'],opts['granularity'],opts['performercount'],opts['resultslocation'],\
qitem['_id'],opts['outputformat'],opts['logfilehost'],clusterurl)
if response is not None:
if 'stats' in response:
if response['stats'] == []:
update_queue_entry(
sess, clusterurl,
logging.warn(
'{View Migrater} Executing DesignDoc Creation for Entry ['
+ qitem['_id'] + ']')
execute_create(
sess, clusterurl, qitem)
elif 'index' in qitem:
logging.warn(
'{Index Migrater} Executing Index Creation for Entry ['
+ qitem['_id'] + ']')
execute_index_create(
sess, clusterurl, qitem)
else:
logging.warn(
'{View Migrater} Unrecognised Queue command for Entry ['
+ qitem['_id'] + ']')
update_queue_entry(
sess, clusterurl, qitem,
'apimigratequeue', "failed",
"Invalid Queue Item", None)
elif 'error' in qdata and qdata[
'error'] != "Queue Error = Empty":
logging.warn(
"{View Migrater} Error in processing queue [" +
str(qdata) + "] Code [" + str(qcode) + "]")
api_utils.close_cluster_session(sess, clusterurl)
time.sleep(10)
except Exception as e:
logging.warn(
"Cloudant View Migrater : Unexpected Shutdown : Reason [" +
str(e) + "]")