def add_2_ftp(self, fl, ctx):
# Add filename to ftp file list
# Extract subdir if supplied
p = os.path.dirname(fl)
fn = os.path.basename(fl) # Just filename
# No dir supplied
if p == "":
# If prior subdir ..
if self.ftp_dir != "":
# .. return to root dir
self.set_chdir(self.ftp_rootdir)
# Forget prior subdir
self.ftp_dir = ""
fqn = self.ftp_rootdir + '/' + fl
elif p[0:1] == '/' or p[0:1] == '.':
fqn = p + '/' + fn
fqn = self.ftp_rootdir + '/' + fn
# Set local directory
self.set_lchdir(p)
self.ftp_dir = p
else:
fqn = self.ftp_rootdir + '/' + p + '/' + fl
if p != self.ftp_dir: # Change subdir
# Make full dir path (root + sub)
lpath = self.ftp_rootdir + '/' + p
self.set_chdir(lpath) # Set directory
self.ftp_dir = p # Remember subdir
if ctx['x_db_name'] and re.match(ctx['x_db_name'], fn):
os0.wlog("DB {0} not replicated on dev host".format(fn))
else:
self.ls_fd.write("{0}\n".format(fqn))
self.ftp_fd.write("put {0}\n".format(fn))
def __init__(self,
module_id,
module_version,
ctr=None,
abort_on_fail=None):
self.module_id = module_id
self.module_version = module_version
if ctr is None:
self.test_ctr = 0
else:
self.test_ctr = ctr
if abort_on_fail is None:
self.abort_on_fail = False
else:
self.abort_on_fail = abort_on_fail
# Test if running in travis-ci emulator (DEV_ENVIRONMENT)
if 'DEV_ENVIRONMENT' in os.environ \
and os.environ['DEV_ENVIRONMENT'].find(self.module_id):
LOCAL_ECHO = False
else:
LOCAL_ECHO = True
tlog_fn = './' + self.module_id + "_test.log"
os0.set_tlog_file(tlog_fn, echo=LOCAL_ECHO, dir4debug=True)
title = "%s regression test. Version: %s" % (self.module_id,
self.module_version)
os0.wlog(title)
def main():
"""Tool main"""
sts = 0
# pdb.set_trace()
ctx = parse_args(sys.argv[1:], version=version(), doc=__doc__)
if ctx['do_list']:
print ctx['saveset_list']
return sts
BM = Backup_Mirror(ctx)
#
# Backup postgres database
if BM.pgdir:
# BM.chdir(BM.pgdir)
dblist = BM.gen_db_list("psql", "odoo", "psql", ctx)
for db in dblist:
BM.init_bck(BM.chdir(BM.pgdir))
BM.bck_db("psql", [db], "odoo", "pg_dump", ctx)
BM.exec_bck()
#
# Backup mysql database
if BM.mysqldir:
# BM.chdir(BM.mysqldir)
dblist = BM.gen_db_list("mysql", "root", "mysql", ctx)
for db in dblist:
BM.init_bck(BM.chdir(BM.mysqldir))
BM.bck_db("mysql", [db], "root", "mysqldump", ctx)
BM.exec_bck()
os0.wlog("Backup DB ended.")
return sts
def add_2_ftp(self, fl, prm):
# Add filename to ftp file list
p = os.path.dirname(fl)
fn = os.path.basename(fl)
if p == "" and self.ftp_dir != "":
p = self.ftp_dir
if p == "":
p = os.getcwd()
if p[0:1] == '/' or p[0:1] == '.':
fqn = p + '/' + fn
else:
fqn = self.ftp_dir + '/' + p + '/' + fl
p = os.path.dirname(fqn)
if p != self.ftp_dir:
self.chdir(p)
os0.wlog(self.pid, " backup", fn)
if prm['cp_mode'] == "overlap":
self.ftp_fd.write("put {0}\n".format(fn))
elif prm['cp_mode'] == "keepold":
self.ftp_fd.write("-rm {0}.bak\n".format(fn))
self.ftp_fd.write("-rename {0} {0}.bak\n".format(fn))
self.ftp_fd.write("put {0}\n".format(fn))
else:
prm['cp_mode'] = "save"
self.ftp_fd.write("put {0} {0}.new\n".format(fn))
self.ls_fd.write("{0}\t{1}\n".format(fqn, prm['cp_mode']))
return
def chdir(self, path):
# Change root dir
lpath = os0.setlfilename(path)
os0.wlog(" [{0}]".format(lpath))
self.set_chdir(lpath)
self.ftp_rootdir = lpath
self.ftp_dir = ""
def test_01(self, test_num):
if not self.open_conf(test_num):
os0.wlog('test_%02d not defined in configuration file %s' %
(test_num, self.test_conf_fn))
self.close_conf()
return TEST_FAILED
self.msg_new_test(test_num)
self.msg_new_test(test_num)
return TEST_SUCCESS
def msg_test(self, newline, test_num):
# if test_ctr >= 39 and test_ctr <= 40: # debug
# import pdb
# pdb.set_trace()
txt = "Test {0:>2}){1:>3}/{2:3}".format(test_num, self.test_ctr,
self.tot_test2exec)
if not newline:
print "\x1b[2A"
os0.wlog(txt)
def test_result(self, test_brief, expected_value, result_value):
self.test_ctr += 1
test_msg = "Test %d %s" % (self.test_ctr, test_brief)
os0.wlog(test_msg)
if expected_value != result_value:
os0.wlog("Test failed: expected '%s', found '%s'" %
(expected_value, result_value))
if self.abort_on_fail:
raise ValueError("Test Failed")
return TEST_FAILED
return TEST_SUCCESS
def test_01(self, ctx):
self.SetUp(ctx)
fzero = False
fexts = False
if not ctx.get('dry_run', False):
os0.set_tlog_file(FLOGTMP)
tlog_pathname = os.path.join(os.environ['HOME'], FLOGTMP)
sts = self.Z.test_result(ctx, "set logfile", tlog_pathname,
os0.tlog_fn)
if not ctx.get('dry_run', False):
tlog_fd = open(os0.tlog_fn, 'w')
tlog_fd.close()
os0.set_tlog_file(FLOGTMP, new=True, echo=ctx['opt_echo'])
try:
tlog_fd = open(os0.tlog_fn, 'r')
fexts = True
tlog_fd.seek(0, os.SEEK_END)
if tlog_fd.tell() == 0:
fzero = True
tlog_fd.close()
except:
pass
sts = self.Z.test_result(ctx, "new logfile (1)", True, fexts)
sts = self.Z.test_result(ctx, "new logfile (2)", True, fzero)
if os.path.isfile(tlog_pathname):
os.remove(tlog_pathname)
tlog_pathname = os.path.join('.', FLOGTMP)
fzero = False
fexts = False
if not ctx.get('dry_run', False):
os0.set_tlog_file(tlog_pathname, new=True, echo=ctx['opt_echo'])
try:
tlog_fd = open(os0.tlog_fn, 'r')
fexts = True
tlog_fd.seek(0, os.SEEK_END)
if tlog_fd.tell() == 0:
fzero = True
tlog_fd.close()
except:
pass
sts = self.Z.test_result(ctx, "new logfile (3)", True, fexts)
sts = self.Z.test_result(ctx, "new logfile (4)", True, fzero)
if os.path.isfile(tlog_pathname):
os.remove(tlog_pathname)
tlog_pathname = os.path.join(self.Z.test_dir, FLOGTMP)
fzero = False
fexts = False
if not ctx.get('dry_run', False):
os0.set_tlog_file(ctx['logfn'],
new=ctx['opt_new'],
echo=ctx['opt_echo'])
if ctx.get('dry_run', False):
os0.wlog("Since now, test messages are store in", os0.tlog_fn)
return sts
def msg_log(ctx, level, text):
"""Log a message and show if needed"""
ident = ' ' * level
if ctx:
if 'test_unit_mode' in ctx:
return
elif ctx['dry_run'] and level > 0:
txt = u"{0}({1})".format(ident, tounicode(text))
else:
txt = u"{0}{1}".format(ident, tounicode(text))
else:
txt = u"{0}{1}".format(ident, tounicode(text))
os0.wlog(txt)
def all_tests(self):
sts = 0
for i in range(MAX_TEST_NUM):
tname = "test_{0:02}".format(i)
if hasattr(self, tname):
sts = getattr(self, tname)()
if sts:
break
# Result
if sts == 0:
os0.wlog("Ran {0} {1} tests".format(self.test_ctr, self.module_id))
else:
os0.wlog("****** Test {0} failed ******".format(self.module_id))
return sts
def check_4_tkn_in_stdout(self, token):
# Now search for this program name in output;
# if found muteshell worked right!
found_chunk = False
try:
stdout_fd = open(os0.setlfilename(os0.bgout_fn, 'r'))
f = stdout_fd.read()
if f.find(token) >= 0:
found_chunk = True
stdout_fd.close()
except:
pass
if not found_chunk:
os0.wlog("Test failed: muteshell did not work!!!")
raise Exception("Test failed: muteshell did not work!!!")
def debug_msg_log(ctx, level, text):
"""Log a debug message and show if needed"""
global db_msg_sp, db_msg_stack
# if level == -999:
# db_msg_sp += 1
# return
ident = ' ' * abs(level)
if ctx.get('dbg_mode', False):
if 'test_unit_mode' in ctx:
return
elif ctx['dry_run'] and level > 0:
txt = u">{0}({1})".format(ident, tounicode(text))
else:
txt = u">{0}{1}".format(ident, tounicode(text))
print txt
os0.wlog(txt)
def create_dict(self):
self.dict = {}
self.xtl = {}
self.seed = 0
try:
cnf_fd = open(self.fconf, "r")
line = cnf_fd.readline()
while line != "":
i = line.rfind('\n')
if i >= 0 and line[0:1] != "#":
line = line.replace("\\ ", "\\b")
line = re.sub('\\s+', ' ', line).strip()
f = string.split(line, ' ')
self.add_dict_entr(f[0], f[1], f[2])
line = cnf_fd.readline()
except:
os0.wlog("No dictionary file", self.fconf, "found!")
def change_file_ext(self, f):
os0.wlog(" changing extension files")
fsql = f + "-????????" + self.pre_ext
f_ids = glob.glob(fsql)
for f in f_ids:
l = len(f) - len(self.pre_ext)
fsql = f[0:l] + self.sql_ext
if not os.path.isfile(fsql):
ftmp = f
if os.path.isfile(ftmp):
try:
os0.wlog("$ mv", ftmp, fsql)
if not self.dry_run:
# Rename old ext -> nex ext
os.rename(ftmp, fsql)
except:
pass
def test_03(self, ctx):
if not ctx.get('dry_run', False):
wchar_string = u"- Unicode string àèìòù"
os0.wlog(wchar_string)
# if wlog fails follow statemente is not executed
sts = self.Z.test_result(ctx, "Check for unicode support", True, True)
if not ctx.get('dry_run', False):
x = unichr(0x3b1) + unichr(0x3b2) + unichr(0x3b3)
os0.wlog("- Greek letters", x)
sts = self.Z.test_result(ctx, "- Greek letters", True, True)
res = None
ustr = None
bstr = None
if not ctx.get('dry_run', False):
res = os0.str2bool('true', None)
sts = self.Z.test_result(ctx, "str2bool(true)", True, res)
if not ctx.get('dry_run', False):
res = os0.str2bool('0', None)
sts = self.Z.test_result(ctx, "str2bool(0)", False, res)
if not ctx.get('dry_run', False):
res = os0.str2bool(False, None)
sts = self.Z.test_result(ctx, "str2bool(0)", False, res)
if not ctx.get('dry_run', False):
res = os0.str2bool('invalid', False)
sts = self.Z.test_result(ctx, "str2bool(0)", False, res)
if not ctx.get('dry_run', False):
res = os0.nakedname('myfile')
sts = self.Z.test_result(ctx, "nakedname(myfile)", 'myfile', res)
if not ctx.get('dry_run', False):
res = os0.nakedname('myfile.py')
sts = self.Z.test_result(ctx, "nakedname(myfile.py)", 'myfile', res)
if not ctx.get('dry_run', False):
bstr = 'text àèìòù'
ustr = u"text àèìòù"
res = os0.u(bstr)
sts = self.Z.test_result(ctx, "unicode(string)", ustr, res)
sts = self.Z.test_result(ctx, "unicode(string)", ustr, os0.u(ustr))
if not ctx.get('dry_run', False):
bstr = 'text àèìòù'
ustr = u"text àèìòù"
res = os0.b(ustr)
sts = self.Z.test_result(ctx, "bstring(string)", bstr, res)
sts = self.Z.test_result(ctx, "bstring(string)", os0.b(bstr), res)
return sts
def gen_db_list(self, dbtype, user, sqlcmd, ctx):
# pdb.set_trace()
dblist = []
os0.wlog(" Creating", dbtype, "db list")
if dbtype == "psql":
cmd = sqlcmd + " -U" + user + " -l"
cmdlog = cmd
elif dbtype == "mysql":
cmd = sqlcmd + " -u " + user + \
" --password="******" -e \"show databases;\" mysql"
cmdlog = sqlcmd + " -u " + user + " -e \"show databases;\" mysql"
else:
cmd = ""
cmdlog = cmd
os0.trace_debug("$", cmdlog)
os0.muteshell(cmd, simulate=self.dry_run, keepout=True)
if ctx['db_name']:
sel_db = ctx['db_name']
else:
sel_db = '.*'
if os0.debug_mode:
os0.wlog("> DB selection", sel_db)
stdinp_fd = open(os0.setlfilename(os0.bgout_fn), 'r')
line = stdinp_fd.readline()
while line != "":
i = line.rfind('\n')
if i >= 0:
if dbtype == "psql":
if line[0:1] == ' ' and line[1:2] != ' ':
x = line.split('|')
dbname = x[0].strip()
if re.match("z[ei].*|demo.*", dbname) and \
re.match(sel_db, dbname):
dblist.append(dbname)
if os0.debug_mode:
os0.wlog("> dblist.append({0})".format(dbname))
elif dbtype == "mysql":
dbname = line.strip()
if re.match("w.*|mg.*|assioma.*", dbname) and \
re.match(sel_db, dbname):
dblist.append(dbname)
if os0.debug_mode:
os0.wlog("> dblist.append({0})".format(dbname))
line = stdinp_fd.readline()
stdinp_fd.close()
if not os0.debug_mode and not self.dry_run and os0.bgout_fn != "":
os.remove(os0.setlfilename(os0.bgout_fn, 'r'))
return dblist
def test_02(self, ctx):
fzero = False
fexts = False
if not ctx.get('dry_run', False):
title = TITLE
os0.trace_debug("- Inititializing ...")
os0.set_debug_mode(False)
os0.trace_debug("TEST FAILED!!!!")
os0.set_debug_mode(True)
os0.trace_debug("- Inititializing (2) ...")
os0.wlog(title)
os0.wlog("- os0 version:", os0.version)
os0.wlog("- platform:", _platform)
os0.set_tlog_file("os0_test.log", echo=ctx['opt_echo'])
try:
tlog_fd = open(os0.tlog_fn, 'r')
fexts = True
tlog_fd.seek(0, os.SEEK_END)
if tlog_fd.tell() == 0:
fzero = True
tlog_fd.close()
except:
pass
sts = self.Z.test_result(ctx, "Check for empty tracelog", True, fexts)
if sts == TEST_SUCCESS:
sts = self.Z.test_result(ctx, "Check for empty tracelog", True,
fzero)
return sts
def restore_file(self, fqn):
# pdb.set_trace()
dbtype = ""
# Extract dir if supplied
p = os.path.dirname(fqn)
f = os.path.basename(fqn) # Just filename
# No dir supplied
if p == "":
p = self.ftp_dir
elif p == "/var/lib/pgsql/backups":
dbtype = "psql"
elif p == "/var/lib/mysql/backups":
dbtype = "mysql"
if dbtype != self.dbtype:
if dbtype == "psql":
cmd = "service postgresql restart"
os0.trace_debug("$", cmd)
os0.muteshell(cmd,
simulate=self.dry_run,
keepout=os0.debug_mode)
elif dbtype == "mysql":
cmd = "service mysqld restart"
os0.trace_debug("$", cmd)
os0.muteshell(cmd,
simulate=self.dry_run,
keepout=os0.debug_mode)
if p != self.ftp_dir: # Change dir
self.chdir(p) # Set directory
l = len(self.sql_ext) + 9
# i = len(f) - l
# Extract dbname from XXXXX-YYYYMMDD.SQL
dbname = f[0:-l]
# if dbname == "wp-zi-it":
# os0.wlog(" db", dbname, "not upgradable!!!")
if os.path.isfile(f):
self.restore_db(dbtype, dbname, fqn)
else:
os0.wlog(" file", f, "not found!!!")
def main():
"""Tool main"""
sts = 0
# pdb.set_trace()
ctx = parse_args(sys.argv[1:], version=version(), doc=__doc__)
if ctx['do_list']:
print ctx['saveset_list']
return sts
RI = Restore_Image(ctx)
f_alrdy_run = check_if_running(ctx, RI.pid)
if f_alrdy_run:
os0.wlog("({0}) ***Another instance is running!!!".format(RI.pid))
try:
ls_fd = open(RI.flist, "r")
except:
raise Exception("Command aborted: file list not found!!!")
file_2_restore = []
fl = ls_fd.readline()
while fl != "":
i = fl.rfind('\n')
if i >= 0 and fl[0:1] != '#':
f = fl[0:i]
file_2_restore.append(f)
fl = ls_fd.readline()
# Restore files
for fl in file_2_restore:
fn = os.path.basename(fl)
if fn != "restconf" \
and fn != "restconf.py" \
and fn != "restconf.ini" \
and fn != "restconf-0.ini":
f = "{0}.new".format(fl)
if os.path.isfile(f):
RI.restore_file(fl)
else:
os0.wlog(" file", fl, "not found!!!")
return sts
def main():
"""Tool main"""
sts = 0
# pdb.set_trace()
ctx = parse_args(sys.argv[1:], version=version(), doc=__doc__)
if ctx['do_list']:
print ctx['saveset_list']
return sts
RI = Restore_Image(ctx)
f_alrdy_run = check_if_running(ctx, RI.pid)
if f_alrdy_run:
os0.wlog("({0}) ***Another instance is running!!!".format(RI.pid))
# Restore files
file_r_ctr = 0
file_u_ctr = 0
time_wait = 60
wait_loop = 3
if not f_alrdy_run:
fl = RI.extract_fn_2_restore()
loop_ctr = wait_loop
while loop_ctr > 0:
if fl != "":
file_r_ctr = file_r_ctr + 1
if os.path.isfile(fl):
RI.restore_file(fl)
file_u_ctr += 1
if file_u_ctr > 1:
wait_loop = 60
loop_ctr = wait_loop
else:
os0.wlog(" file", fl, "not found!!!")
RI.commit_fn_restored()
fl = RI.extract_fn_2_restore()
if fl == "":
os0.wlog(" wait for next db")
time.sleep(time_wait)
loop_ctr -= 1
if not ctx['dbg_mode'] and os.path.isfile(os0.setlfilename(os0.bgout_fn)):
os.remove(os0.setlfilename(os0.bgout_fn))
if not f_alrdy_run:
os0.wlog("Restore DB ended."
" {0} DB to restore, {1} DB restored ({2}).".format(
file_u_ctr, file_u_ctr, RI.pid))
return sts
def main(self):
"""Unit test main."""
# Check for target software version
required_version = os.environ.get(self.module_id.upper() + '_VERSION')
if not required_version:
required_version = REQ_TEST_VERSION
pkg_version = build.version()
if required_version and pkg_version.find(required_version) < 0:
print "Test not executable: invalid version!"
print "Required version:", required_version
print "Package version:", pkg_version
return TEST_FAILED
# Need debug mode to avoid security fault in Linux
os0.set_debug_mode(True)
title = "%s regression test. Version: %s" % (self.module_id,
REQ_TEST_VERSION)
# Remove test log file if executed previous crashed test
tlog_fn = self.module_id + "_test.log"
os0.set_tlog_file(tlog_fn)
tlog_pathname = os0.tlog_fn
# Set no file log
os0.set_tlog_file('', echo=True)
if os.path.isfile(tlog_pathname):
os.remove(tlog_pathname)
# Test if running in travis-ci emulator (DEV_ENVIRONMENT)
if 'DEV_ENVIRONMENT' in os.environ \
and os.environ['DEV_ENVIRONMENT'].find(self.module_id):
LOCAL_ECHO = False
else:
LOCAL_ECHO = True
tlog_fn = './' + self.module_id + "_test.log"
os0.set_tlog_file(tlog_fn, new=True, echo=LOCAL_ECHO)
os0.wlog(title)
# Test execution body
test_num = 0
sts = 0
for i in range(MAX_TEST_NUM):
tname = "test_{0:02}".format(test_num)
if hasattr(self, tname):
sts = getattr(self, tname)(test_num)
if sts:
break
test_num += 1
# Result
if sts == 0:
os0.wlog("Ran {0} {1} tests".format(self.test_ctr, MODULE_ID))
else:
os0.wlog("****** Test {0} failed ******".format(MODULE_ID))
return sts
def Backup_files(self, cfg_obj, sset, prm):
path_list = cfg_obj.get(sset, "path")
if path_list == "":
os0.wlog(self.pid, "No file declared!!")
if not prm['mute']:
print "***No file declared file!!!"
return
prm['cmd_bck'] = cfg_obj.get(sset, "cmd_bck").replace('{}', '{0}')
prm['cp_mode'] = cfg_obj.get(sset, "cp_mode")
path_list = path_list.split(',')
self.init_bck(prm, path_list[0])
file_2_backup = []
for path in path_list:
if os.path.isdir(path):
path = os.path.join(path, "*")
if prm['specimen'] == "file":
file_2_backup = file_2_backup + glob.glob(path)
elif prm['specimen'] == "dir":
for root, dirs, files in os.walk(path):
for f in files:
file_2_backup = file_2_backup + os.path.join(root, f)
else:
pass
if prm['cmd_bck'] != "":
cmd = prm['cmd_bck'].replace('{0}', path)
os0.trace_debug("> ", cmd)
os0.muteshell(cmd, simulate=prm['simulate'])
self.set_chdir(path_list[0])
for f in file_2_backup:
if os.path.isfile(f):
self.add_2_ftp(f, prm)
elif os.path.isdir(f):
os0.wlog(self.pid, " dir", f, "skipped")
else:
os0.wlog(self.pid, " file", f, "not found!!!")
return
def restore_file(self, fqn):
# Extract dir if supplied
p = os.path.dirname(fqn)
f = os.path.basename(fqn) # Just filename
# No dir supplied
if p == "":
p = self.ftp_dir
if p != self.ftp_dir: # Change dir
self.chdir(p) # Set directory
fzero = False
fexts = False
fn = "{0}.new".format(f)
ftmp = "{0}.tmp".format(f)
fbak = "{0}.bak".format(f)
try:
fn_fd = open(fn, 'r')
# Go to end of file
fn_fd.seek(0, os.SEEK_END)
# File len = 0 ?
if fn_fd.tell() == 0:
fzero = True
# Go to begin of file
fn_fd.seek(0, 0)
# Read entire file
fn_str = fn_fd.read()
fn_fd.close()
except:
fzero = True
if fzero:
os0.wlog(" file", fn, "empty!!!")
else:
os0.wlog(" ", fn, "->", fqn)
# Search for text substitution
key_ids = self.search4item(f)
if key_ids:
# Text couples for substitution
for key in key_ids:
src = self.xtl[key][0]
tgt = self.xtl[key][1]
# print "[{0} >subst/{1}/{2}/".format(fqn, src, tgt)
os0.wlog(" ", fqn, ":", src, "->", tgt)
# Substitute text in file
fn_str = fn_str.replace(src, tgt)
ftmp_fd = open(ftmp, 'w')
# write file with substitutions
ftmp_fd.write(fn_str)
ftmp_fd.close()
else:
# Rename file.new -> file.tmp
os.rename(fn, ftmp)
try:
f_fd = open(f, 'r')
f_fd.close()
fexts = True
with open(os.devnull, "w") as fdnull:
cmd = "diff"
p1 = f
p2 = ftmp
sts = call([cmd, p1, p2], stdout=fdnull, stderr=fdnull)
# New file for upgrade
if sts > 0:
os0.wlog(" file", f, "upgraded!!!")
else:
os0.wlog(" file", f, "not changed.")
fzero = True
fdnull.close()
except:
os0.wlog(" file", f, "new!!!")
try:
# Delete file.new (if exist)
os.remove(fn)
except:
pass
if fzero: # No upgrade
try:
# Delete file.tmp (if exist)
os.remove(ftmp)
except:
pass
else: # Upgrade
try:
# Delete file.bak (if exist)
os.remove(fbak)
except:
pass
if fexts:
# Rename file -> file.bak
os.rename(f, fbak)
if not self.dry_run:
# Rename file.tmp -> file
os.rename(ftmp, f)
def main():
"""Tool main"""
sts = 0
# pdb.set_trace()
ctx = parse_args(sys.argv[1:],
version=version(),
doc=__doc__)
if ctx['do_list']:
print ctx['saveset_list']
return sts
BM = Backup_Mirror(ctx)
# Backup files in root directory
BM.chdir("/root")
file_2_backup = ["av_php", "bckconf", "bckconf.py",
"bckdb", "bckdb.py", "bckwww",
"cldb", "purgedb",
"restconf", "restconf.py",
"restconf.ini", "restconf-0.ini",
"restdb", "restdb.py", "restwww",
"ssl_certificate", "statdb",
"zarlib.py"]
for fl in file_2_backup:
if os.path.isfile(fl):
os0.wlog(" ", fl)
BM.add_2_ftp(fl)
else:
os0.wlog(" file", fl, "not found!!!")
# Backup configuration file for http server
BM.chdir("/etc/httpd/conf/sites-enabled")
file_2_backup = glob.glob("/etc/httpd/conf/sites-enabled/*.conf")
for f in file_2_backup:
fl = os.path.basename(f) # Just filename
if os.path.isfile(fl):
os0.wlog(" ", fl)
BM.add_2_ftp(fl)
else:
os0.wlog(" file", fl, "not found!!!")
# Backup web files
BM.chdir("/var/www/html")
file_2_backup = ["openerp/index.html"]
for fl in file_2_backup:
if os.path.isfile(fl):
os0.wlog(" ", fl)
BM.add_2_ftp(fl)
else:
os0.wlog(" file", fl, "not found!!!")
# Backup web certificates
BM.chdir("/var/www/certs")
file_2_backup = ["shop.zeroincombenze.it.crt",
"shop.zeroincombenze.it.CA.crt",
"shop.zeroincombenze.it.pem",
"19340476.crt",
"19340476.ca-bundle",
"19340476.pem",
"zi-oe.csr",
"zi-oe.pem",
"www.shs-av.com.csr",
"shs-av.com.pem",
"StartCom.CA.csr"]
for fl in file_2_backup:
if os.path.isfile(fl):
os0.wlog(" ", fl)
BM.add_2_ftp(fl)
else:
os0.wlog(" file", fl, "not found!!!")
# Backup postgres configuration files
BM.chdir("/var/lib/pgsql/data")
# file_2_backup = ["pg_hba.conf", ]
# for fl in file_2_backup:
# if os.path.isfile(fl):
# os0.wlog(" ", fl)
# BM.add_2_ftp(fl)
# else:
# os0.wlog(" file", fl, "not found!!!")
BM.exec_bck()
return sts
def repl_data(self, dbname, fqn):
fzero = False
try:
fqn_fd = open(fqn, 'r')
# Go to end of file
fqn_fd.seek(0, os.SEEK_END)
# File len = 0 ?
if fqn_fd.tell() == 0:
fzero = True
# Go to begin of file
fqn_fd.seek(0, 0)
# Read entire file
fqn_str = fqn_fd.read()
fqn_fd.close()
except:
fzero = True
if fzero:
os0.wlog(" file", fqn, "empty!!!")
else:
fxch = False
# Search for text substitution (Wordpress)
f = dbname + "->wp"
ctx = self.get_params(f)
if ctx['prefix'] != "" and ctx['siteURL'] != "":
fxch = True
fqn_str = self.repl_data_wp(ctx, fqn_str)
# Search for sql command to append
f = dbname + "/"
key_ids = self.search4item(f)
if key_ids:
fxch = True
# Text couples for substitution
for key in key_ids:
src = self.xtl[key][0]
src = src.replace("\\b", " ")
tgt = self.xtl[key][1]
tgt = tgt.replace("\\b", " ")
os0.trace_debug(">", src, tgt, ";")
fqn_str = fqn_str + src + " " + tgt + ";\n"
# Search for text substitution in SQL statements
f = dbname + self.sql_ext
key_ids = self.search4item(f)
if key_ids:
fxch = True
# Text couples for substitution
for key in key_ids:
src = self.xtl[key][0]
src = src.replace("\\b", " ")
tgt = self.xtl[key][1]
tgt = tgt.replace("\\b", " ")
os0.trace_debug("> sed|{0}|{1}|".format(src, tgt))
fqn_str = fqn_str.replace(src, tgt)
if fxch:
fqn_fd = open(fqn, 'w')
fqn_fd.write(fqn_str)
fqn_fd.close()
f = dbname + "->wiki"
ctx = self.get_params(f)
if ctx['siteURL'] != "":
fqns = ctx['conf_file'].split(',')
for fqn in fqns:
self.replace_file(ctx, f, fqn)
if ctx['conf_file2']:
fqns = ctx['conf_file2'].split(',')
for fqn in fqns:
self.replace_file(ctx, f, fqn)
if ctx['conf_file3']:
fqns = ctx['conf_file3'].split(',')
for fqn in fqns:
self.replace_file(ctx, f, fqn)
fqn = ctx['index_html']
self.replace_file(ctx, f, fqn)
def restore_db(self, dbtype, dbname, fqn):
# pdb.set_trace()
os0.wlog(" restoring", dbname, " ({0})".format(fqn))
homedir = os.path.expanduser("~")
tar_ext = self.tar_ext
tar_opt = self.tar_opt
fzip_fn = dbname + tar_ext
if not os.path.isfile(fzip_fn):
if self.tar_ext == ".gz":
tar_ext = ".bz2"
tar_opt = "j"
fzip_fn = dbname + tar_ext
if not os.path.isfile(fzip_fn):
tar_ext = self.tar_ext
tar_opt = self.tar_opt
# No compressed file found
fzip_fn = ""
elif self.tar_ext == ".bz2":
tar_ext = ".gz"
tar_opt = "z"
fzip_fn = dbname + tar_ext
if not os.path.isfile(fzip_fn):
tar_ext = self.tar_ext
tar_opt = self.tar_opt
# No compressed file found
fzip_fn = ""
f = os.path.basename(fqn) # Just filename
l = len(self.sql_ext) + 9
i = len(f) - l
# Extract date (YYYYMMDD) from XXXXX-YYYYMMDD.SQL
dts = f[i + 1:i + 9]
if dbtype == "psql":
cmd = "chown " + self.psql_uu + ":" + self.psql_uu + " " + fqn
elif dbtype == "mysql":
cmd = "chown " + self.mysql_uu + ":" + self.mysql_uu + " " + fqn
os0.trace_debug("$", cmd)
os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode)
sql_fn = homedir + "/restdb.sql"
cmd = "cp " + fqn + " " + sql_fn
os0.trace_debug("$", cmd)
os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode)
# cmd = "sed -i -e \"s|Owner: openerp|Owner: odoo|g\""\
# " -e \"s|OWNER TO openerp|OWNER TO odoo|g\" ~/restdb.sql"
# os0.trace_debug("$", cmd)
# os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode)
if dbtype == "psql":
cmd = "chown " + self.psql_uu + ":" + self.psql_uu + " " + sql_fn
elif dbtype == "mysql":
cmd = "chown " + self.mysql_uu + ":" + self.mysql_uu + " " + sql_fn
os0.trace_debug("$", cmd)
os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode)
self.repl_data(dbname, sql_fn)
psh_fn = homedir + "/restdb.psh"
psh_fd = open(psh_fn, "w")
if dbtype == "psql":
user = self.psql_uu
defdb = self.psql_db
psh_fd.write("\\c {0}\n".format(defdb))
psh_fd.write("DROP DATABASE IF EXISTS \"{0}-{1}\";\n".format(
dbname, dts))
psh_fd.write("DROP DATABASE IF EXISTS \"{0}\";\n".format(dbname))
psh_fd.write(
"CREATE DATABASE \"{0}\" TEMPLATE template1;\n".format(dbname))
psh_fd.write("\\c \"{0}\"\n".format(dbname))
psh_fd.write("\\i {0}\n".format(sql_fn))
psh_fd.write(
"ALTER DATABASE \"{0}\" OWNER TO odoo;\n".format(dbname))
cmd = "psql -f " + psh_fn + " -U" + user + " " + defdb
psh_fd.close()
os0.trace_debug("$", cmd)
os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode)
elif dbtype == "mysql":
user = "******"
pwd = "SHS13mgr"
# defdb = self.psql_db
psh_fd.write(
"mysqladmin -u{0} --password={1} -f drop \"{2}-{3}\" ||true\n".
format(user, pwd, dbname, dts))
psh_fd.write(
"mysqladmin -u{0} --password={1} -f drop \"{2}\" || true\n".
format(user, pwd, dbname))
psh_fd.write(
"mysqladmin -u{0} --password={1} -f create \"{2}\"\n".format(
user, pwd, dbname))
psh_fd.write(
"mysql -u{0} --password=SHS13mgr -G -e \"source {1}\" {2}\n".
format(user, sql_fn, dbname))
psh_fd.close()
cmd = "chmod +x " + psh_fn
os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode)
cmd = psh_fn
os0.trace_debug("$", cmd)
os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode)
else:
os0.wlog(" unknown", dbname, "database type!!!")
cmd = "echo Error"
# Compressed file found
if fzip_fn != "":
if dbtype == "psql":
cmd = "chown " + self.psql_uu + \
":" + self.psql_uu + " " + fzip_fn
elif dbtype == "mysql":
cmd = "chown " + self.mysql_uu + \
":" + self.mysql_uu + " " + fzip_fn
os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode)
cmd = "tar --keep-newer-files -x" + tar_opt + "f " + fzip_fn
os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode)
if not self.dry_run:
os.remove(fzip_fn)
self.purge_db(dbtype, dbname)
def purge_db(self, dbtype, f):
# pdb.set_trace()
if self.sql_ext != self.pre_ext:
self.change_file_ext(f)
dtc = date.today() - timedelta(90)
os0.wlog(" removing file older than", dtc.strftime("%Y-%m-%d"))
fzip_fn = f + self.tar_ext
force_change_ext = False
for i in range(180, 120, -1):
dtc = datetime.today() - timedelta(i)
dts = dtc.strftime("%Y%m%d")
fsql = f + "-" + dts + self.sql_ext
if not os.path.isfile(fsql) and self.sql_ext != self.pre_ext:
ftmp = f + "-" + dts + self.pre_ext
if os.path.isfile(ftmp):
try:
os0.wlog("$ mv", ftmp, fsql)
if not self.dry_run:
# Rename old ext -> nex ext
os.rename(ftmp, fsql)
# Force change sql file extension
force_change_ext = True
except:
pass
if dtc.day != 1:
if not self.remove_sql_file(fsql) \
and self.sql_ext != self.pre_ext:
fsql = f + "-" + dts + self.pre_ext
self.remove_sql_file(fsql)
if force_change_ext:
self.change_file_ext(f)
fsql = f + "-????????" + self.sql_ext
if dbtype == "psql":
cmd = "chown " + self.psql_uu + ":" + self.psql_uu + " " + fsql
elif dbtype == "mysql":
cmd = "chown " + self.mysql_uu + ":" + self.mysql_uu + " " + fsql
os0.trace_debug("$ ", cmd)
os0.muteshell(cmd, simulate=self.dry_run)
cmd = "tar --remove-files -c" + \
self.tar_opt + "f " + fzip_fn + " " + fsql
os0.trace_debug("$ ", cmd)
os0.muteshell(cmd, simulate=self.dry_run)
if dbtype == "psql":
cmd = "chown " + self.psql_uu + ":" + self.psql_uu + " " + fzip_fn
elif dbtype == "mysql":
cmd = "chown " + self.mysql_uu + \
":" + self.mysql_uu + " " + fzip_fn
os0.trace_debug("$ ", cmd)
os0.muteshell(cmd, simulate=self.dry_run)
os0.wlog(" removing archived files")
fsql = f + "-????????" + self.sql_ext
f_ids = sorted(glob.glob(fsql))
for fsql in f_ids:
self.remove_sql_file(fsql)
def __init__(self, ctx):
self.hostname = ctx['hostname']
os0.set_debug_mode(ctx['dbg_mode'])
self.prodhost = ctx['production_host']
self.devhost = ctx['development_host']
self.mirrorhost = ctx['mirror_host']
self.pgdir = ctx['pg_dir']
self.mysqldir = ctx['mysql_dir']
homedir = os.path.expanduser("~")
self.ftp_cfn = homedir + "/" + ctx['ftp_script']
self.flist = homedir + "/" + ctx['list_file']
os0.set_tlog_file(ctx['logfn'])
# Log begin execution
os0.wlog("Restore configuration files", __version__)
# Simulate backup
self.dry_run = ctx['dry_run']
if ctx['saveset'] == "bckdb" or \
ctx['saveset'] == "bckconf" or \
ctx['saveset'] == "bckwww":
if self.hostname == self.prodhost:
os0.wlog("Running on production machine")
if ctx['alt']:
self.bck_host = self.mirrorhost
self.fconf = homedir + "/" + \
ctx['no_translation']
else:
self.bck_host = self.devhost
self.fconf = homedir + "/" + \
ctx['data_translation']
elif self.hostname == self.mirrorhost:
os0.wlog("Running on mirror machine")
if ctx['alt']:
self.bck_host = self.prodhost
self.fconf = homedir + "/" + \
ctx['no_translation']
else:
self.bck_host = self.devhost
self.fconf = homedir + "/" + \
ctx['data_translation']
elif self.hostname == self.devhost:
os0.wlog("This command cannot run on development machine")
if not ctx['dry_run']:
raise Exception("Command aborted due invalid machine")
else:
os0.wlog("Unknown machine - Command aborted")
if not ctx['dry_run']:
raise Exception("Command aborted due unknown machine")
elif ctx['saveset'] == "restdb" or \
ctx['saveset'] == "restconf" or \
ctx['saveset'] == "restwww":
if self.hostname == self.prodhost:
os0.wlog("This command cannot run on production machine")
if not ctx['dry_run']:
raise Exception("Command aborted due production machine")
elif self.hostname == self.mirrorhost:
os0.wlog("Running on mirror machine")
if ctx['alt']:
self.bck_host = self.prodhost
self.fconf = homedir + "/" + \
ctx['no_translation']
else:
self.bck_host = self.devhost
self.fconf = homedir + "/" + \
ctx['data_translation']
elif self.hostname == self.devhost:
os0.wlog("Running on development machine")
if ctx['alt']:
self.bck_host = self.mirrorhost
self.fconf = homedir + "/" + \
ctx['data_translation']
else:
self.bck_host = self.devhost
self.fconf = homedir + "/" + \
ctx['data_translation']
else:
os0.wlog("Unknown machine - Command aborted")
if not ctx['dry_run']:
raise Exception("Command aborted due unknown machine")
# May be (.gz or .bz2)
self.tar_ext = ctx['tar_ext']
# May be (z or j)
self.tar_opt = ctx['tar_opt']
# May be (null or .sql)
self.pre_ext = ctx['pre_ext']
# May be (null or .sql)
self.sql_ext = ctx['sql_ext']
self.psql_uu = ctx['pgsql_user']
self.psql_db = ctx['pgsql_def_db']
self.mysql_uu = ctx['mysql_user']
self.mysql_db = ctx['mysql_def_db']
self.pid = os.getpid()
self.ftp_rootdir = ""
self.ftp_dir = ""
self.dbtype = ""
self.create_dict()
def test_00(self, test_num):
if not self.open_conf(test_num):
os0.wlog('test_%02d not defined in configuration file %s' %
(test_num, self.test_conf_fn))
self.close_conf()
return TEST_FAILED
self.msg_new_test(test_num)
prm = build.parse_args(['-btvT', 'pypi'])
if 'template' not in prm or prm['template'] != 'pypi':
os0.wlog('Test failed: option -T')
return TEST_FAILED
self.msg_new_test(test_num)
if 'modname' not in prm or prm['modname'] != 'midea':
os0.wlog('Test failed: default option -n')
return TEST_FAILED
self.msg_new_test(test_num)
prm = build.parse_args(['-btvT', 'pypi', '-n', 'testcode'])
if 'modname' not in prm or prm['modname'] != 'testcode':
os0.wlog('Test failed: ption -n')
return TEST_FAILED
self.msg_new_test(test_num)
conf_fn = self.module_id + '.conf'
prm = build.parse_args(['-btvT', 'pypi', '-c', conf_fn], True)
if os.path.isfile('/etc/odoo-server.conf'):
if 'conf_fn' not in prm or\
prm['conf_fn'] != ['/etc/odoo-server.conf', conf_fn]:
os0.wlog('Test failed: option -c')
return TEST_FAILED
elif os.path.isfile('/etc/openerp-server.conf'):
if 'conf_fn' not in prm or\
prm['conf_fn'] != ['/etc/openerp-server.conf', conf_fn]:
os0.wlog('Test failed: option -c')
return TEST_FAILED
self.close_conf()
return TEST_SUCCESS
