def cleantarget():
debug("ACTION: Cleaning destination directory %s" % restoredest)
for root, dirs, files in os.walk(restoredest, topdown=False):
for name in files:
os.remove(os.path.join(root, name))
for name in dirs:
os.rmdir(os.path.join(root, name))
def _createinitora(self):
filename = self._initfile
with open(filename, 'w') as f:
contents = self._restoretemplate.get('autoinitora')
if 'cdb' in Configuration.substitutions and Configuration.substitutions['cdb'].upper() == 'TRUE':
contents+= self._restoretemplate.get('cdbinitora')
debug("ACTION: Generated init file %s\n%s" % (filename, contents))
f.write(contents)
def post(self, url, payload):
debug("Sending POST to %s" % url)
r = requests.post("%s/%s" % (self._baseurl, url), cookies=self._cookies, verify=False, data=payload, timeout=self._timeout, allow_redirects=False)
self._cookies = r.cookies
debug("Return code: %d" % r.status_code)
try:
j = json.loads(r.text)
except ValueError:
j = {}
return j, r.status_code
def delete(self, urlarray):
url = self._array2url(urlarray)
debug("Sending DELETE to %s" % url)
r = requests.delete("%s/%s" % (self._baseurl, url),
auth=self._auth,
headers=self._jsonheader,
verify=False,
timeout=self._timeout)
debug("Return code: %d" % r.status_code)
return r.status_code
def verify(self, tolerancechecking=True):
debug('ACTION: opening database to verify the result')
if tolerancechecking:
maxtolerance = timedelta(minutes=int(
Configuration.get('autorestoremaxtoleranceminutes',
'autorestore')))
Configuration.substitutions.update({
'customverifydate':
Configuration.get('customverifydate', self._configname),
})
output = self._exec_sqlplus(self._restoretemplate.get('openandverify'),
returnoutput=True)
for line in output.splitlines():
if line.startswith('CUSTOM VERIFICATION TIME:'):
self.verifytime = datetime.strptime(
line.split(':', 1)[1].strip(), '%Y-%m-%d %H:%M:%S')
if self.verifytime is None:
raise Exception('restore', 'Reading verification time failed.')
self.verifydiff = self._dbparams['restoretarget'].replace(
tzinfo=None) - self.verifytime
self.verifyseconds = int(self.verifydiff.seconds +
self.verifydiff.days * 24 * 3600)
debug("Expected time: %s" % self._dbparams['restoretarget'])
debug("Verified time: %s" % self.verifytime)
debug("VERIFY: Time difference %s" % self.verifydiff)
if tolerancechecking and self.verifydiff > maxtolerance:
raise Exception(
'restore',
"Verification time difference %s is larger than allowed tolerance %s"
% (verifydiff, maxtolerance))
def _run_restore(self):
debug('ACTION: startup nomount')
self._exec_sqlplus(self._restoretemplate.get('startupnomount'))
debug('ACTION: mount database and catalog files')
self._exec_rman(self._restoretemplate.get('mountandcatalog'))
if self._dbparams['bctfile']:
debug('ACTION: disable block change tracking')
self._exec_sqlplus(self._restoretemplate.get('disablebct'))
debug('ACTION: create missing datafiles')
output = self._exec_sqlplus(self._restoretemplate.get('switchdatafiles'), returnoutput=True)
switchdfscript = ""
for line in output.splitlines():
if line.startswith('RENAMEDF-'):
switchdfscript+= "%s\n" % line.strip()[9:]
debug('ACTION: switch and recover')
self._exec_rman("%s\n%s" % (switchdfscript, self._restoretemplate.get('recoverdatafiles')))
def put(self, urlarray, payload):
url = self._array2url(urlarray)
debug("Sending PUT to %s" % url)
r = requests.put("%s/%s" % (self._baseurl, url),
auth=self._auth,
headers=self._jsonheader,
verify=False,
data=json.dumps(payload),
timeout=self._timeout)
debug("Return code: %d" % r.status_code)
if r.status_code == 201:
j = json.loads(r.text)
else:
error("PUT to %s returned %d" % (url, r.status_code))
j = {}
return r.status_code, j
def cleanup(self):
try:
debug('ACTION: In case instance is still running, aborting it')
self._exec_sqlplus(self._restoretemplate.get('shutdownabort'))
except:
pass
if self._successful_mount:
try:
self._unmount()
except:
exception("Error unmounting")
if self._successful_clone:
try:
self._snap.dropautoclone()
except:
exception("Error dropping clone")
self.endtime = datetime.now()
def get(self, urlarray, return_json=True):
url = self._array2url(urlarray)
debug("Sending GET to %s" % url)
r = requests.get("%s/%s" % (self._baseurl, url),
auth=self._auth,
headers=self._jsonheader,
verify=False,
timeout=self._timeout)
debug("Return code: %d" % r.status_code)
if r.status_code != 200:
error("GET to %s returned %d" % (url, r.status_code))
raise Exception(
'zfssareturncode',
"GET request return code is not 200 (%s)" % r.status_code)
if return_json:
j = json.loads(r.text)
return j
else:
return None
def sqlldr(self, login, finalscript):
self._setenv()
debug("SQLLDR execution starts")
f1 = mkstemp(suffix=".ctl")
ftmp = os.fdopen(f1[0], "w")
ftmp.write(finalscript)
ftmp.close()
f2 = mkstemp(suffix=".log")
os.close(f2[0])
with TemporaryFile() as f:
p = Popen([os.path.join(self.oraclehome, 'bin', 'sqlldr'), login, "control=%s" % f1[1], "log=%s" % f2[1], "errors=0", "silent=all"], stdout=f, stderr=None, stdin=None)
p.communicate()
if p.returncode != 0:
error("SQLLDR exited with code %d" % p.returncode)
raise Exception('sqlldr', "sqlldr exited with code %d" % p.returncode)
else:
debug("SQLLDR execution successful")
os.unlink(f1[1])
os.unlink(f2[1])
def sqlplus(self, finalscript, silent=False):
self._setenv()
with TemporaryFile() as f:
args = [os.path.join(self.oraclehome, 'bin', 'sqlplus')]
if silent:
args.append('-S')
args.append('/nolog')
debug("SQL*Plus execution starts")
BackupLogger.close()
p = Popen(args, stdout=f, stderr=f, stdin=PIPE)
p.communicate(input=finalscript)
BackupLogger.init()
if p.returncode != 0:
error("SQL*Plus exited with code %d" % p.returncode)
raise Exception('sqlplus', "sqlplus exited with code %d" % p.returncode)
else:
debug("SQL*Plus execution successful")
if silent:
f.seek(0,0)
return f.read()
def imagecopywithsnap():
starttime = datetime.now()
restoreparamfile = os.path.join(backupdest, 'autorestore.cfg')
#
info("Check if there are missing archivelogs")
backup_missing_archlog()
#
info("Switch current log")
output = exec_sqlplus(rmantemplateconfig.get('archivecurrentlogs'),
silent=True)
if os.path.isfile(restoreparamfile):
with open(restoreparamfile, 'a') as f:
for line in output.splitlines():
if line.startswith('CURRENT DATABASE SCN:'):
f.write("lastscn: %s\n" % line.strip()[22:])
elif line.startswith('CURRENT DATABASE TIME:'):
f.write("lasttime: %s\n" % line.strip()[23:])
elif line.startswith('BCT FILE:'):
f.write("bctfile: %s\n" % line.strip()[10:])
#
if dosnapshot:
info("Snap the current backup area")
snapid = snap.snap()
debug("Created snapshot: %s" % snapid)
#
info("Checking for expired datafile copies")
delete_expired_datafilecopy()
#
info("Refresh imagecopy")
backup('imagecopy')
exec_sqlplus(rmantemplateconfig.get('archivecurrentlogs'))
#
if dosnapshot:
info("Clean expired snapshots")
cleaningresult = snap.clean()
for r in cleaningresult:
debug(r['infostring'])
#
if gimanaged:
p = Popen([os.path.join(scriptpath, 'dbinfo.py'), configsection],
stdout=PIPE,
stderr=None,
stdin=None)
output, outerr = p.communicate()
debug(output)
#
info("Write database parameters for autorestore")
with open(restoreparamfile, 'w') as f:
f.write("[dbparams]\n")
output = exec_sqlplus(rmantemplateconfig.get('autorestoreparameters'),
silent=True)
for line in output.splitlines():
if line.startswith('dbconfig-'):
f.write("%s\n" % line[9:])
#
endtime = datetime.now()
info("------------ TOTAL ------------")
info("Total execution time: %s" % (endtime - starttime))
info("Execution started: %s" % starttime)
info("Execution finished: %s" % endtime)
def rman(self, finalscript):
self._setenv()
debug("RMAN execution starts")
BackupLogger.close()
starttime = datetime.now()
with TemporaryFile() as f:
p = Popen([os.path.join(self.oraclehome, 'bin', 'rman'), "log", BackupLogger.logfile, "append"], stdout=f, stderr=f, stdin=PIPE)
# Send the script to RMAN
p.communicate(input=finalscript)
endtime = datetime.now()
BackupLogger.init()
debug("RMAN execution time %s" % (endtime-starttime))
# If RMAN exists with any code except 0, then there was some error
if p.returncode != 0:
error("RMAN execution failed with code %d" % p.returncode)
raise Exception('rman', "RMAN exited with code %d" % p.returncode)
else:
debug("RMAN execution successful")
def __init__(self, oraclehome, tnspath, sid=None):
self.oraclehome = oraclehome
self.tnspath = tnspath
if sid is not None:
self.oraclesid = sid
debug("Oracle home: %s" % self.oraclehome)
def runrestore(database):
global exitstatus
#
Configuration.defaultsection = database
#
restoredest = Configuration.get('autorestoredestination', 'autorestore')
mountdest = Configuration.get('autorestoremountpoint', 'autorestore')
logdir = Configuration.get('autorestorelogdir', 'autorestore')
Configuration.substitutions.update({
'logdir':
logdir,
'autorestorecatalog':
Configuration.get('autorestorecatalog', 'autorestore')
})
if restoredest is None or not os.path.exists(
restoredest) or not os.path.isdir(restoredest):
print "Restore directory %s not found or is not a proper directory" % restoredest
sys.exit(2)
if mountdest is None or not os.path.exists(mountdest) or not os.path.isdir(
mountdest):
print "Clone mount directory %s not found or is not a proper directory" % mountdest
sys.exit(2)
#
validatechance = int(
Configuration.get('autorestorevalidatechance', 'autorestore'))
validatemodulus = int(
Configuration.get('autorestoremodulus', 'autorestore'))
# Reinitialize logging
BackupLogger.init(
os.path.join(
logdir, "%s-%s.log" %
(datetime.now().strftime('%Y%m%dT%H%M%S'), database)), database)
BackupLogger.clean()
#
restore = RestoreDB(database)
restore.set_mount_path(mountdest)
restore.set_restore_path(restoredest)
#
info("Logfile: %s" % BackupLogger.logfile)
Configuration.substitutions.update({'logfile': BackupLogger.logfile})
cleantarget(restoredest)
#
success = False
#
if validatemodulus > 0:
# Validation based on modulus
validationinfo = validationdate(database)
validatecorruption = validationinfo[0]
if not validatecorruption:
debug("Next database validation in %d days: %s" %
(validationinfo[1], validationinfo[2]))
else:
# Validation based on random
validatecorruption = (validatechance > 0) and (randint(
1, validatechance) == validatechance)
if validatecorruption:
debug("Database will be validated during this restore session")
# Start restore
try:
restore.run()
restore.verify()
if validatecorruption:
restore.blockcheck()
success = True
except:
exitstatus = 1
exception(
"Error happened, but we can continue with the next database.")
finally:
restore.cleanup()
# Log result to catalog
Configuration.substitutions.update({
'log_dbname':
database,
'log_start':
restore.starttime.strftime('%Y-%m-%d %H-%M-%S'),
'log_stop':
restore.endtime.strftime('%Y-%m-%d %H-%M-%S'),
'log_success':
'1' if success else '0',
'log_diff':
restore.verifyseconds,
'log_snapid':
restore.sourcesnapid,
'log_validated':
'1' if validatecorruption else '0'
})
debug('Logging the result to catalog.')
try:
oexec.sqlldr(Configuration.get('autorestorecatalog', 'autorestore'),
restoretemplate.get('sqlldrlog'))
except:
debug("Sending the logfile to catalog failed.")
try:
oexec.sqlplus(restoretemplate.get('insertlog'), silent=False)
except:
debug("Logging the result to catalog failed.")
# Finish up
info("Restore %s, elapsed time: %s" %
('successful' if success else 'failed',
restore.endtime - restore.starttime))
# Run ADRCI to clean up diag
adrage = int(Configuration.get('logretention', 'generic')) * 1440
f1 = mkstemp(suffix=".adi")
ftmp = os.fdopen(f1[0], "w")
ftmp.write("set base %s\n" % logdir)
ftmp.write("show homes\n")
ftmp.close()
f2 = mkstemp(suffix=".adi")
ftmp2 = os.fdopen(f2[0], "w")
ftmp2.write("set base %s\n" % logdir)
with TemporaryFile() as f:
try:
oexec.adrci(f1[1], f)
f.seek(0, 0)
output = f.read()
startreading = False
for line in output.splitlines():
if line.startswith('ADR Homes:'):
startreading = True
elif startreading:
ftmp2.write("set home %s\n" % line.strip())
ftmp2.write("purge -age %d\n" % adrage)
ftmp2.close()
oexec.adrci(f2[1], f)
except:
print "Executing ADRCI failed."
finally:
os.unlink(f1[1])
os.unlink(f2[1])
#
BackupLogger.close(True)
def get(self, url):
debug("Sending GET to %s" % url)
r = requests.get("%s/%s" % (self._baseurl, url), cookies=self._cookies, verify=False, timeout=self._timeout, allow_redirects=False)
debug("Return code: %d" % r.status_code)
return r.status_code
def _set_parameters(self):
# Detect if mountpath is actually a namespace containing multiple volumes
orig_mount_dest = self._mountdest
if not os.path.isfile(os.path.join(orig_mount_dest,
'autorestore.cfg')):
location_correct = False
for item in os.listdir(orig_mount_dest):
item_full_path = os.path.join(orig_mount_dest, item)
if os.path.isdir(item_full_path):
self._mountpaths.append(item_full_path)
if os.path.isfile(
os.path.join(item_full_path, 'autorestore.cfg')):
location_correct = True
self._mountdest = item_full_path
if not location_correct:
raise Exception(
'restore',
'Mount path is not correct, autorestore.cfg was not found')
else:
self._mountpaths.append(self._mountdest)
debug("All datafile mountpaths: %s" % self._mountpaths)
debug("Main datafile mountpath: %s" % self._mountdest)
debug("Location for temporary init.ora and other files: %s" %
self._restoredest)
#
dbconfig = SafeConfigParser()
dbconfig.read(os.path.join(self._mountdest, 'autorestore.cfg'))
self._dbparams['dbname'] = dbconfig.get('dbparams', 'db_name')
if self.targettime is None:
self._dbparams['restoretarget'] = datetime.strptime(
dbconfig.get('dbparams', 'lasttime'), '%Y-%m-%d %H:%M:%S')
else:
self._dbparams['restoretarget'] = self.targettime.astimezone(
get_localzone())
self._dbparams['bctfile'] = dbconfig.get('dbparams', 'bctfile')
catalogstatements = []
for item in self._mountpaths:
catalogstatements.append(
"catalog start with '%s/archivelog/' noprompt;" % item)
catalogstatements.append(
"catalog start with '%s/data_' noprompt;" % item)
Configuration.substitutions.update({
'db_name':
self._dbparams['dbname'],
'db_compatible':
dbconfig.get('dbparams', 'compatible'),
'db_files':
dbconfig.get('dbparams', 'db_files'),
'db_undotbs':
dbconfig.get('dbparams', 'undo_tablespace'),
'db_block_size':
dbconfig.get('dbparams', 'db_block_size'),
# 'lastscn': dbconfig.get('dbparams','lastscn'),
'lasttime':
self._dbparams['restoretarget'].strftime('%Y-%m-%d %H:%M:%S'),
'dbid':
Configuration.get('dbid', self._configname),
'instancenumber':
Configuration.get('autorestoreinstancenumber', self._configname),
'thread':
Configuration.get('autorestorethread', self._configname),
'backupfinishedtime':
dbconfig.get('dbparams', 'backup-finished'),
'bctfile':
self._dbparams['bctfile'],
'autorestoredestination':
self._restoredest,
'mountdestination':
self._mountdest,
'catalogstatements':
"\n".join(catalogstatements)
})
try:
Configuration.substitutions.update(
{'cdb': dbconfig.get('dbparams', 'enable_pluggable_database')})
except:
Configuration.substitutions.update({'cdb': 'FALSE'})
self._initfile = os.path.join(self._restoredest, 'init.ora')
Configuration.substitutions.update({
'initora': self._initfile,
})
def exec_restore():
global exitvalue
restore.clone(False)
print "Please execute the following command as root to mount the backup volume:"
print ""
print "mount -t nfs -o rw,bg,hard,nointr,rsize=32768,wsize=32768,tcp,vers=3,timeo=600 %s %s" % (restore.mountstring, restoreparams['mountpath'])
print ""
while ask_yn("Did you execute it") == "N":
print "Please execute it then."
# Verify that clone is mounted
autorestorefile = os.path.join(restoreparams['mountpath'], 'autorestore.cfg')
if not os.path.isfile(autorestorefile):
print "The mounted path does not look correct, file %s not found" % autorestorefile
exitvalue = 1
return
#
debug("Oracle home: %s" % Configuration.get("oraclehome", "generic"))
debug("Clone mount path: %s" % restoreparams['mountpath'])
debug("Target instance SID: %s" % restoreparams['sid'])
debug("Restore target time UTC: %s" % restoreparams['timepoint'].astimezone(pytz.utc))
debug("Restore target time local: %s" % restoreparams['timepoint'].astimezone(get_localzone()))
debug("Restored from snapshot: %s" % restore.sourcesnapid)
info("Starting database restore")
#
try:
restore.pit_restore(restoreparams['mountpath'], restoreparams['sid'])
restore.verify(False)
info("Database restore complete")
info("SID: %s" % restoreparams['sid'])
info("Requested target time: %s" % restoreparams['timepoint'].astimezone(get_localzone()))
info("Verified restored database time: %s" % restore.verifytime)
info("Difference from target: %s" % restore.verifydiff)
except:
exception("Database restore failed")
exitvalue = 1
print ""
print "Commands to clean up:"
print "1. Shut down database instance %s" % restoreparams['sid']
print "2. Execute as root: umount %s" % restoreparams['mountpath']
print "3. Drop clone: BACKUPCONFIG=%s %s %s dropclone %s" % (os.path.basename(Configuration.configfilename),
os.path.join(scriptpath(), 'zsnapper.py'), configname, restore.clonename)
def runrestore(database):
global exitstatus
#
Configuration.defaultsection = database
# Reinitialize logging
BackupLogger.init(
os.path.join(
logdir, "%s-%s.log" %
(datetime.now().strftime('%Y%m%dT%H%M%S'), database)), database)
BackupLogger.clean()
#
restore = RestoreDB(database)
restore.set_mount_path(mountdest)
restore.set_restore_path(restoredest)
#
info("Logfile: %s" % BackupLogger.logfile)
Configuration.substitutions.update({'logfile': BackupLogger.logfile})
cleantarget()
#
success = False
#
if validatemodulus > 0:
# Validation based on modulus
validationinfo = validationdate(database)
validatecorruption = validationinfo[0]
if not validatecorruption:
debug("Next database validation in %d days: %s" %
(validationinfo[1], validationinfo[2]))
else:
# Validation based on random
validatecorruption = (validatechance > 0) and (randint(
1, validatechance) == validatechance)
if validatecorruption:
debug("Database will be validated during this restore session")
# Start restore
try:
restore.run()
restore.verify()
if validatecorruption:
restore.blockcheck()
success = True
except:
exitstatus = 1
exception(
"Error happened, but we can continue with the next database.")
finally:
restore.cleanup()
# Log result to catalog
Configuration.substitutions.update({
'log_dbname':
database,
'log_start':
restore.starttime.strftime('%Y-%m-%d %H-%M-%S'),
'log_stop':
restore.endtime.strftime('%Y-%m-%d %H-%M-%S'),
'log_success':
'1' if success else '0',
'log_diff':
restore.verifyseconds,
'log_snapid':
restore.sourcesnapid,
'log_validated':
'1' if validatecorruption else '0'
})
debug('Logging the result to catalog.')
try:
oexec.sqlldr(Configuration.get('autorestorecatalog', 'autorestore'),
restoretemplate.get('sqlldrlog'))
except:
debug("Sending the logfile to catalog failed.")
try:
oexec.sqlplus(restoretemplate.get('insertlog'), silent=False)
except:
debug("Logging the result to catalog failed.")
# Finish up
info("Restore %s, elapsed time: %s" %
('successful' if success else 'failed',
restore.endtime - restore.starttime))
BackupLogger.close(True)
