def remove_sql_file(self, fsql):
try:
fzip_fd = open(fsql, "r")
fzip_fd.close()
os0.trace_debug("$ rm", fsql)
if not self.dry_run:
os.remove(fsql)
sts = True
except:
sts = False
return sts
def gen_db_list(self, dbtype, user, sqlcmd, ctx):
# pdb.set_trace()
dblist = []
os0.wlog(" Creating", dbtype, "db list")
if dbtype == "psql":
cmd = sqlcmd + " -U" + user + " -l"
cmdlog = cmd
elif dbtype == "mysql":
cmd = sqlcmd + " -u " + user + \
" --password="******" -e \"show databases;\" mysql"
cmdlog = sqlcmd + " -u " + user + " -e \"show databases;\" mysql"
else:
cmd = ""
cmdlog = cmd
os0.trace_debug("$", cmdlog)
os0.muteshell(cmd, simulate=self.dry_run, keepout=True)
if ctx['db_name']:
sel_db = ctx['db_name']
else:
sel_db = '.*'
if os0.debug_mode:
os0.wlog("> DB selection", sel_db)
stdinp_fd = open(os0.setlfilename(os0.bgout_fn), 'r')
line = stdinp_fd.readline()
while line != "":
i = line.rfind('\n')
if i >= 0:
if dbtype == "psql":
if line[0:1] == ' ' and line[1:2] != ' ':
x = line.split('|')
dbname = x[0].strip()
if re.match("z[ei].*|demo.*", dbname) and \
re.match(sel_db, dbname):
dblist.append(dbname)
if os0.debug_mode:
os0.wlog("> dblist.append({0})".format(dbname))
elif dbtype == "mysql":
dbname = line.strip()
if re.match("w.*|mg.*|assioma.*", dbname) and \
re.match(sel_db, dbname):
dblist.append(dbname)
if os0.debug_mode:
os0.wlog("> dblist.append({0})".format(dbname))
line = stdinp_fd.readline()
stdinp_fd.close()
if not os0.debug_mode and not self.dry_run and os0.bgout_fn != "":
os.remove(os0.setlfilename(os0.bgout_fn, 'r'))
return dblist
def deduplicate_db(self, fsql):
f_ids = sorted(glob.glob(fsql))
f_prior = ""
for fsql in f_ids:
if f_prior == "":
f_prior = fsql
continue
if filecmp.cmp(f_prior, fsql):
os0.trace_debug(
"Files",
f_prior,
fsql,
"are identical",
)
f_prior = fsql
def replace_file(self, ctx, f, fqn):
os0.trace_debug("> replace file", fqn)
try:
fn_fd = open(fqn, 'r')
fn_str = fn_fd.read()
fn_fd.close()
key_ids = self.search4item(f)
if key_ids:
src = ctx['siteURL']
tgt = ctx['testURL']
fn_str = fn_str.replace(src, tgt)
src = ctx['siteURI']
tgt = ctx['testURI']
fn_str = fn_str.replace(src, tgt)
fn_fd = open(fqn, 'w')
fn_fd.write(fn_str)
fn_fd.close()
except:
pass
def commit_fn_restored(self):
ftmp = self.flist + ".lst"
fbak = self.flist + ".bak"
if os.path.isfile(ftmp):
fn_fd = open(ftmp, 'r')
fzero = True
fl = fn_fd.readline()
while fl != "" and fzero:
i = fl.rfind('\n')
if i >= 0:
fzero = False
fl = fn_fd.readline()
fn_fd.close()
if not fzero:
cmd = "rm -f {2}; mv {0} {2}; mv {1} {0}".format(
self.flist, ftmp, fbak)
os0.trace_debug("$ ", cmd)
os0.muteshell(cmd, simulate=self.dry_run)
else:
if not self.dry_run:
os.remove(ftmp)
def test_02(self, ctx):
fzero = False
fexts = False
if not ctx.get('dry_run', False):
title = TITLE
os0.trace_debug("- Inititializing ...")
os0.set_debug_mode(False)
os0.trace_debug("TEST FAILED!!!!")
os0.set_debug_mode(True)
os0.trace_debug("- Inititializing (2) ...")
os0.wlog(title)
os0.wlog("- os0 version:", os0.version)
os0.wlog("- platform:", _platform)
os0.set_tlog_file("os0_test.log", echo=ctx['opt_echo'])
try:
tlog_fd = open(os0.tlog_fn, 'r')
fexts = True
tlog_fd.seek(0, os.SEEK_END)
if tlog_fd.tell() == 0:
fzero = True
tlog_fd.close()
except:
pass
sts = self.Z.test_result(ctx, "Check for empty tracelog", True, fexts)
if sts == TEST_SUCCESS:
sts = self.Z.test_result(ctx, "Check for empty tracelog", True,
fzero)
return sts
def restore_file(self, fqn):
# pdb.set_trace()
dbtype = ""
# Extract dir if supplied
p = os.path.dirname(fqn)
f = os.path.basename(fqn) # Just filename
# No dir supplied
if p == "":
p = self.ftp_dir
elif p == "/var/lib/pgsql/backups":
dbtype = "psql"
elif p == "/var/lib/mysql/backups":
dbtype = "mysql"
if dbtype != self.dbtype:
if dbtype == "psql":
cmd = "service postgresql restart"
os0.trace_debug("$", cmd)
os0.muteshell(cmd,
simulate=self.dry_run,
keepout=os0.debug_mode)
elif dbtype == "mysql":
cmd = "service mysqld restart"
os0.trace_debug("$", cmd)
os0.muteshell(cmd,
simulate=self.dry_run,
keepout=os0.debug_mode)
if p != self.ftp_dir: # Change dir
self.chdir(p) # Set directory
l = len(self.sql_ext) + 9
# i = len(f) - l
# Extract dbname from XXXXX-YYYYMMDD.SQL
dbname = f[0:-l]
# if dbname == "wp-zi-it":
# os0.wlog(" db", dbname, "not upgradable!!!")
if os.path.isfile(f):
self.restore_db(dbtype, dbname, fqn)
else:
os0.wlog(" file", f, "not found!!!")
def Backup_files(self, cfg_obj, sset, prm):
path_list = cfg_obj.get(sset, "path")
if path_list == "":
os0.wlog(self.pid, "No file declared!!")
if not prm['mute']:
print "***No file declared file!!!"
return
prm['cmd_bck'] = cfg_obj.get(sset, "cmd_bck").replace('{}', '{0}')
prm['cp_mode'] = cfg_obj.get(sset, "cp_mode")
path_list = path_list.split(',')
self.init_bck(prm, path_list[0])
file_2_backup = []
for path in path_list:
if os.path.isdir(path):
path = os.path.join(path, "*")
if prm['specimen'] == "file":
file_2_backup = file_2_backup + glob.glob(path)
elif prm['specimen'] == "dir":
for root, dirs, files in os.walk(path):
for f in files:
file_2_backup = file_2_backup + os.path.join(root, f)
else:
pass
if prm['cmd_bck'] != "":
cmd = prm['cmd_bck'].replace('{0}', path)
os0.trace_debug("> ", cmd)
os0.muteshell(cmd, simulate=prm['simulate'])
self.set_chdir(path_list[0])
for f in file_2_backup:
if os.path.isfile(f):
self.add_2_ftp(f, prm)
elif os.path.isdir(f):
os0.wlog(self.pid, " dir", f, "skipped")
else:
os0.wlog(self.pid, " file", f, "not found!!!")
return
def add_2_ftp(self, fl):
# Add filename to ftp file list
lx = ("cldb", "bckconf",
"bckdb", "bckdb.py", "bckwww",
"purgedb", "restconf", "restconf.py",
"restdb", "restdb.py", "restwww",
"statdb", ".zar.conf")
# Extract subdir if supplied
p = os.path.dirname(fl)
fn = os.path.basename(fl) # Just filename
if p == "" and os.path.dirname(__file__) != "" \
and (fn == "restconf.py" or fn == "restdb.py"):
os0.trace_debug("Development & Testing software execution!")
p = os.path.dirname(__file__)
# No dir supplied
if p == "":
# If prior subdir ..
if self.ftp_dir != "":
# .. return to root dir
self.set_chdir(self.ftp_rootdir)
# Forget prior subdir
self.ftp_dir = ""
fqn = self.ftp_rootdir + '/' + fl
elif p[0:1] == '/' or p[0:1] == '.':
fqn = p + '/' + fn
if fn in lx:
cmd = "chmod +x {0}".format(fqn)
os0.trace_debug("$ ", cmd)
os0.muteshell(cmd, simulate=self.dry_run)
fqn = self.ftp_rootdir + '/' + fn
# Set local directory
self.set_lchdir(p)
self.ftp_dir = p
else:
fqn = self.ftp_rootdir + '/' + p + '/' + fl
if p != self.ftp_dir: # Change subdir
# Make full dir path (root + sub)
lpath = self.ftp_rootdir + '/' + p
self.set_chdir(lpath) # Set directory
self.ftp_dir = p # Remember subdir
self.ls_fd.write("{0}\n".format(fqn))
if fn == "restconf.py" or fn == "restdb.py":
cmd = "chmod +x {0}".format(fqn)
os0.trace_debug("$ ", cmd)
os0.muteshell(cmd, simulate=self.dry_run)
if fn == "restconf" or fn == "restconf.py":
self.ftp_fd.write("-rm {0}.bak\n".format(fn))
self.ftp_fd.write("-rename {0} {0}.bak\n".format(fn))
self.ftp_fd.write("put {0}\n".format(fn))
elif fn == "restconf.ini" or fn == "restconf-0.ini":
self.ftp_fd.write("-rm {0}.bak\n".format(fn))
self.ftp_fd.write("-rename {0} {0}.bak\n".format(fn))
self.ftp_fd.write("put {0}\n".format(fn))
else:
self.ftp_fd.write("put {0} {0}.new\n".format(fn))
def purge_db(self, dbtype, f):
# pdb.set_trace()
if self.sql_ext != self.pre_ext:
self.change_file_ext(f)
dtc = date.today() - timedelta(90)
os0.wlog(" removing file older than", dtc.strftime("%Y-%m-%d"))
fzip_fn = f + self.tar_ext
force_change_ext = False
for i in range(180, 120, -1):
dtc = datetime.today() - timedelta(i)
dts = dtc.strftime("%Y%m%d")
fsql = f + "-" + dts + self.sql_ext
if not os.path.isfile(fsql) and self.sql_ext != self.pre_ext:
ftmp = f + "-" + dts + self.pre_ext
if os.path.isfile(ftmp):
try:
os0.wlog("$ mv", ftmp, fsql)
if not self.dry_run:
# Rename old ext -> nex ext
os.rename(ftmp, fsql)
# Force change sql file extension
force_change_ext = True
except:
pass
if dtc.day != 1:
if not self.remove_sql_file(fsql) \
and self.sql_ext != self.pre_ext:
fsql = f + "-" + dts + self.pre_ext
self.remove_sql_file(fsql)
if force_change_ext:
self.change_file_ext(f)
fsql = f + "-????????" + self.sql_ext
if dbtype == "psql":
cmd = "chown " + self.psql_uu + ":" + self.psql_uu + " " + fsql
elif dbtype == "mysql":
cmd = "chown " + self.mysql_uu + ":" + self.mysql_uu + " " + fsql
os0.trace_debug("$ ", cmd)
os0.muteshell(cmd, simulate=self.dry_run)
cmd = "tar --remove-files -c" + \
self.tar_opt + "f " + fzip_fn + " " + fsql
os0.trace_debug("$ ", cmd)
os0.muteshell(cmd, simulate=self.dry_run)
if dbtype == "psql":
cmd = "chown " + self.psql_uu + ":" + self.psql_uu + " " + fzip_fn
elif dbtype == "mysql":
cmd = "chown " + self.mysql_uu + \
":" + self.mysql_uu + " " + fzip_fn
os0.trace_debug("$ ", cmd)
os0.muteshell(cmd, simulate=self.dry_run)
os0.wlog(" removing archived files")
fsql = f + "-????????" + self.sql_ext
f_ids = sorted(glob.glob(fsql))
for fsql in f_ids:
self.remove_sql_file(fsql)
def restore_db(self, dbtype, dbname, fqn):
# pdb.set_trace()
os0.wlog(" restoring", dbname, " ({0})".format(fqn))
homedir = os.path.expanduser("~")
tar_ext = self.tar_ext
tar_opt = self.tar_opt
fzip_fn = dbname + tar_ext
if not os.path.isfile(fzip_fn):
if self.tar_ext == ".gz":
tar_ext = ".bz2"
tar_opt = "j"
fzip_fn = dbname + tar_ext
if not os.path.isfile(fzip_fn):
tar_ext = self.tar_ext
tar_opt = self.tar_opt
# No compressed file found
fzip_fn = ""
elif self.tar_ext == ".bz2":
tar_ext = ".gz"
tar_opt = "z"
fzip_fn = dbname + tar_ext
if not os.path.isfile(fzip_fn):
tar_ext = self.tar_ext
tar_opt = self.tar_opt
# No compressed file found
fzip_fn = ""
f = os.path.basename(fqn) # Just filename
l = len(self.sql_ext) + 9
i = len(f) - l
# Extract date (YYYYMMDD) from XXXXX-YYYYMMDD.SQL
dts = f[i + 1:i + 9]
if dbtype == "psql":
cmd = "chown " + self.psql_uu + ":" + self.psql_uu + " " + fqn
elif dbtype == "mysql":
cmd = "chown " + self.mysql_uu + ":" + self.mysql_uu + " " + fqn
os0.trace_debug("$", cmd)
os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode)
sql_fn = homedir + "/restdb.sql"
cmd = "cp " + fqn + " " + sql_fn
os0.trace_debug("$", cmd)
os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode)
# cmd = "sed -i -e \"s|Owner: openerp|Owner: odoo|g\""\
# " -e \"s|OWNER TO openerp|OWNER TO odoo|g\" ~/restdb.sql"
# os0.trace_debug("$", cmd)
# os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode)
if dbtype == "psql":
cmd = "chown " + self.psql_uu + ":" + self.psql_uu + " " + sql_fn
elif dbtype == "mysql":
cmd = "chown " + self.mysql_uu + ":" + self.mysql_uu + " " + sql_fn
os0.trace_debug("$", cmd)
os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode)
self.repl_data(dbname, sql_fn)
psh_fn = homedir + "/restdb.psh"
psh_fd = open(psh_fn, "w")
if dbtype == "psql":
user = self.psql_uu
defdb = self.psql_db
psh_fd.write("\\c {0}\n".format(defdb))
psh_fd.write("DROP DATABASE IF EXISTS \"{0}-{1}\";\n".format(
dbname, dts))
psh_fd.write("DROP DATABASE IF EXISTS \"{0}\";\n".format(dbname))
psh_fd.write(
"CREATE DATABASE \"{0}\" TEMPLATE template1;\n".format(dbname))
psh_fd.write("\\c \"{0}\"\n".format(dbname))
psh_fd.write("\\i {0}\n".format(sql_fn))
psh_fd.write(
"ALTER DATABASE \"{0}\" OWNER TO odoo;\n".format(dbname))
cmd = "psql -f " + psh_fn + " -U" + user + " " + defdb
psh_fd.close()
os0.trace_debug("$", cmd)
os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode)
elif dbtype == "mysql":
user = "******"
pwd = "SHS13mgr"
# defdb = self.psql_db
psh_fd.write(
"mysqladmin -u{0} --password={1} -f drop \"{2}-{3}\" ||true\n".
format(user, pwd, dbname, dts))
psh_fd.write(
"mysqladmin -u{0} --password={1} -f drop \"{2}\" || true\n".
format(user, pwd, dbname))
psh_fd.write(
"mysqladmin -u{0} --password={1} -f create \"{2}\"\n".format(
user, pwd, dbname))
psh_fd.write(
"mysql -u{0} --password=SHS13mgr -G -e \"source {1}\" {2}\n".
format(user, sql_fn, dbname))
psh_fd.close()
cmd = "chmod +x " + psh_fn
os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode)
cmd = psh_fn
os0.trace_debug("$", cmd)
os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode)
else:
os0.wlog(" unknown", dbname, "database type!!!")
cmd = "echo Error"
# Compressed file found
if fzip_fn != "":
if dbtype == "psql":
cmd = "chown " + self.psql_uu + \
":" + self.psql_uu + " " + fzip_fn
elif dbtype == "mysql":
cmd = "chown " + self.mysql_uu + \
":" + self.mysql_uu + " " + fzip_fn
os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode)
cmd = "tar --keep-newer-files -x" + tar_opt + "f " + fzip_fn
os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode)
if not self.dry_run:
os.remove(fzip_fn)
self.purge_db(dbtype, dbname)
def repl_data(self, dbname, fqn):
fzero = False
try:
fqn_fd = open(fqn, 'r')
# Go to end of file
fqn_fd.seek(0, os.SEEK_END)
# File len = 0 ?
if fqn_fd.tell() == 0:
fzero = True
# Go to begin of file
fqn_fd.seek(0, 0)
# Read entire file
fqn_str = fqn_fd.read()
fqn_fd.close()
except:
fzero = True
if fzero:
os0.wlog(" file", fqn, "empty!!!")
else:
fxch = False
# Search for text substitution (Wordpress)
f = dbname + "->wp"
ctx = self.get_params(f)
if ctx['prefix'] != "" and ctx['siteURL'] != "":
fxch = True
fqn_str = self.repl_data_wp(ctx, fqn_str)
# Search for sql command to append
f = dbname + "/"
key_ids = self.search4item(f)
if key_ids:
fxch = True
# Text couples for substitution
for key in key_ids:
src = self.xtl[key][0]
src = src.replace("\\b", " ")
tgt = self.xtl[key][1]
tgt = tgt.replace("\\b", " ")
os0.trace_debug(">", src, tgt, ";")
fqn_str = fqn_str + src + " " + tgt + ";\n"
# Search for text substitution in SQL statements
f = dbname + self.sql_ext
key_ids = self.search4item(f)
if key_ids:
fxch = True
# Text couples for substitution
for key in key_ids:
src = self.xtl[key][0]
src = src.replace("\\b", " ")
tgt = self.xtl[key][1]
tgt = tgt.replace("\\b", " ")
os0.trace_debug("> sed|{0}|{1}|".format(src, tgt))
fqn_str = fqn_str.replace(src, tgt)
if fxch:
fqn_fd = open(fqn, 'w')
fqn_fd.write(fqn_str)
fqn_fd.close()
f = dbname + "->wiki"
ctx = self.get_params(f)
if ctx['siteURL'] != "":
fqns = ctx['conf_file'].split(',')
for fqn in fqns:
self.replace_file(ctx, f, fqn)
if ctx['conf_file2']:
fqns = ctx['conf_file2'].split(',')
for fqn in fqns:
self.replace_file(ctx, f, fqn)
if ctx['conf_file3']:
fqns = ctx['conf_file3'].split(',')
for fqn in fqns:
self.replace_file(ctx, f, fqn)
fqn = ctx['index_html']
self.replace_file(ctx, f, fqn)
def repl_data_wp(self, ctx, fqn_str):
os0.trace_debug("> update URL (wp) {0}->{1}".format(
ctx['siteURL'], ctx['testURL']))
stmt = "update {0}options set option_value='{1}'"\
" where option_name='{2}'"\
.format(ctx['prefix'], ctx['testURL'], "siteurl")
fqn_str = fqn_str + stmt + ";\n"
stmt = "update {0}options set option_value='{1}'"\
" where option_name='{2}'"\
.format(ctx['prefix'], ctx['testURL'], "home")
fqn_str = fqn_str + stmt + ";\n"
stmt = "update {0}options set option_value='{1}/'"\
" where option_name='{2}'"\
.format(ctx['prefix'], ctx['testURL'], "ga_default_domain")
fqn_str = fqn_str + stmt + ";\n"
stmt = "update {0}options set option_value='{1}'"\
" where option_name='{2}'"\
.format(ctx['prefix'], ctx['siteURI'], "ga_root_domain")
fqn_str = fqn_str + stmt + ";\n"
stmt = "update {0}options set option_value='{1}'"\
" where option_name='{2}'"\
.format(ctx['prefix'], ctx['admin_email'], "admin_email")
fqn_str = fqn_str + stmt + ";\n"
stmt = "update {0}options set option_value='{1}'"\
" where option_name='{2}'"\
.format(ctx['prefix'], "0", "blog_public")
fqn_str = fqn_str + stmt + ";\n"
src_str = ctx['siteURL']
ix = fqn_str.find(src_str)
while ix >= 0:
l = len(ctx['siteURL'])
j = ix - 1
sep = ' '
while sep == ' ':
while fqn_str[j] != '\"' and fqn_str[j] != '\'':
j = j - 1
sep = fqn_str[j]
j = j - 1
if fqn_str[j] == '\\':
if sep == '\'':
sep = ' '
else:
j = j - 1
if fqn_str[j] != ':':
sep = ' '
else:
j = j - 1
if sep == '\"':
ix1 = j + 1
while fqn_str[j].isdigit():
j = j - 1
n = fqn_str[j + 1:ix1]
i = int(n)
if i >= l:
src = fqn_str[j + 1:ix] + ctx['siteURL']
j = len(ctx['testURL'])
n = str(i + j - l)
tgt = n + fqn_str[ix1:ix] + ctx['testURL']
os0.trace_debug("> sed|{0}|{1}|".format(src, tgt))
fqn_str = fqn_str.replace(src, tgt)
ix = fqn_str.find(src_str, ix + 1)
return fqn_str
def bck_db(self, dbtype, dblist, user, sqlcmd, ctx):
# pdb.set_trace()
# save_ftp_rootdir = self.ftp_rootdir
p = "backups"
# Make full dir path (root + sub)
lpath = self.ftp_rootdir + '/' + p
self.chdir(lpath) # Set directory
for f in dblist:
tar_ext = self.tar_ext
tar_opt = self.tar_opt
fzip_fn = f + tar_ext
if not os.path.isfile(fzip_fn):
if self.tar_ext == ".gz":
tar_ext = ".bz2"
tar_opt = "j"
fzip_fn = f + tar_ext
if not os.path.isfile(fzip_fn):
tar_ext = self.tar_ext
tar_opt = self.tar_opt
# No compressed file found
fzip_fn = ""
elif self.tar_ext == ".bz2":
tar_ext = ".gz"
tar_opt = "z"
fzip_fn = f + tar_ext
if not os.path.isfile(fzip_fn):
tar_ext = self.tar_ext
tar_opt = self.tar_opt
# No compressed file found
fzip_fn = ""
# Compressed file found
if fzip_fn != "":
cmd = "tar -x" + tar_opt + "f " + fzip_fn
os0.trace_debug("$", cmd)
os0.muteshell(cmd, simulate=self.dry_run)
if not self.dry_run:
os.remove(fzip_fn)
dts = date.today().strftime("%Y%m%d")
fsql = f + "-" + dts + self.sql_ext
if dbtype == "psql":
cmd = sqlcmd + " -U" + user + " -F p -f " + fsql + " " + f
cmdlog = cmd
elif dbtype == "mysql":
cmd = sqlcmd + " -u " + user + \
" --password="******" " + f + \
" -r " + fsql
cmdlog = sqlcmd + " -u " + user + " " + f + " -r " + fsql
else:
cmd = ""
cmdlog = cmd
os0.trace_debug("$", cmdlog)
os0.muteshell(cmd, simulate=self.dry_run)
if os.path.isfile(fsql):
os0.wlog(" ", fsql)
self.add_2_ftp(fsql, ctx)
else:
os0.wlog(" file", fsql, "not found!!!")
self.purge_db(dbtype, f)