def add_2_ftp(self, fl, ctx): # Add filename to ftp file list # Extract subdir if supplied p = os.path.dirname(fl) fn = os.path.basename(fl) # Just filename # No dir supplied if p == "": # If prior subdir .. if self.ftp_dir != "": # .. return to root dir self.set_chdir(self.ftp_rootdir) # Forget prior subdir self.ftp_dir = "" fqn = self.ftp_rootdir + '/' + fl elif p[0:1] == '/' or p[0:1] == '.': fqn = p + '/' + fn fqn = self.ftp_rootdir + '/' + fn # Set local directory self.set_lchdir(p) self.ftp_dir = p else: fqn = self.ftp_rootdir + '/' + p + '/' + fl if p != self.ftp_dir: # Change subdir # Make full dir path (root + sub) lpath = self.ftp_rootdir + '/' + p self.set_chdir(lpath) # Set directory self.ftp_dir = p # Remember subdir if ctx['x_db_name'] and re.match(ctx['x_db_name'], fn): os0.wlog("DB {0} not replicated on dev host".format(fn)) else: self.ls_fd.write("{0}\n".format(fqn)) self.ftp_fd.write("put {0}\n".format(fn))
def __init__(self, module_id, module_version, ctr=None, abort_on_fail=None): self.module_id = module_id self.module_version = module_version if ctr is None: self.test_ctr = 0 else: self.test_ctr = ctr if abort_on_fail is None: self.abort_on_fail = False else: self.abort_on_fail = abort_on_fail # Test if running in travis-ci emulator (DEV_ENVIRONMENT) if 'DEV_ENVIRONMENT' in os.environ \ and os.environ['DEV_ENVIRONMENT'].find(self.module_id): LOCAL_ECHO = False else: LOCAL_ECHO = True tlog_fn = './' + self.module_id + "_test.log" os0.set_tlog_file(tlog_fn, echo=LOCAL_ECHO, dir4debug=True) title = "%s regression test. Version: %s" % (self.module_id, self.module_version) os0.wlog(title)
def main(): """Tool main""" sts = 0 # pdb.set_trace() ctx = parse_args(sys.argv[1:], version=version(), doc=__doc__) if ctx['do_list']: print ctx['saveset_list'] return sts BM = Backup_Mirror(ctx) # # Backup postgres database if BM.pgdir: # BM.chdir(BM.pgdir) dblist = BM.gen_db_list("psql", "odoo", "psql", ctx) for db in dblist: BM.init_bck(BM.chdir(BM.pgdir)) BM.bck_db("psql", [db], "odoo", "pg_dump", ctx) BM.exec_bck() # # Backup mysql database if BM.mysqldir: # BM.chdir(BM.mysqldir) dblist = BM.gen_db_list("mysql", "root", "mysql", ctx) for db in dblist: BM.init_bck(BM.chdir(BM.mysqldir)) BM.bck_db("mysql", [db], "root", "mysqldump", ctx) BM.exec_bck() os0.wlog("Backup DB ended.") return sts
def add_2_ftp(self, fl, prm): # Add filename to ftp file list p = os.path.dirname(fl) fn = os.path.basename(fl) if p == "" and self.ftp_dir != "": p = self.ftp_dir if p == "": p = os.getcwd() if p[0:1] == '/' or p[0:1] == '.': fqn = p + '/' + fn else: fqn = self.ftp_dir + '/' + p + '/' + fl p = os.path.dirname(fqn) if p != self.ftp_dir: self.chdir(p) os0.wlog(self.pid, " backup", fn) if prm['cp_mode'] == "overlap": self.ftp_fd.write("put {0}\n".format(fn)) elif prm['cp_mode'] == "keepold": self.ftp_fd.write("-rm {0}.bak\n".format(fn)) self.ftp_fd.write("-rename {0} {0}.bak\n".format(fn)) self.ftp_fd.write("put {0}\n".format(fn)) else: prm['cp_mode'] = "save" self.ftp_fd.write("put {0} {0}.new\n".format(fn)) self.ls_fd.write("{0}\t{1}\n".format(fqn, prm['cp_mode'])) return
def chdir(self, path): # Change root dir lpath = os0.setlfilename(path) os0.wlog(" [{0}]".format(lpath)) self.set_chdir(lpath) self.ftp_rootdir = lpath self.ftp_dir = ""
def test_01(self, test_num): if not self.open_conf(test_num): os0.wlog('test_%02d not defined in configuration file %s' % (test_num, self.test_conf_fn)) self.close_conf() return TEST_FAILED self.msg_new_test(test_num) self.msg_new_test(test_num) return TEST_SUCCESS
def msg_test(self, newline, test_num): # if test_ctr >= 39 and test_ctr <= 40: # debug # import pdb # pdb.set_trace() txt = "Test {0:>2}){1:>3}/{2:3}".format(test_num, self.test_ctr, self.tot_test2exec) if not newline: print "\x1b[2A" os0.wlog(txt)
def test_result(self, test_brief, expected_value, result_value): self.test_ctr += 1 test_msg = "Test %d %s" % (self.test_ctr, test_brief) os0.wlog(test_msg) if expected_value != result_value: os0.wlog("Test failed: expected '%s', found '%s'" % (expected_value, result_value)) if self.abort_on_fail: raise ValueError("Test Failed") return TEST_FAILED return TEST_SUCCESS
def test_01(self, ctx): self.SetUp(ctx) fzero = False fexts = False if not ctx.get('dry_run', False): os0.set_tlog_file(FLOGTMP) tlog_pathname = os.path.join(os.environ['HOME'], FLOGTMP) sts = self.Z.test_result(ctx, "set logfile", tlog_pathname, os0.tlog_fn) if not ctx.get('dry_run', False): tlog_fd = open(os0.tlog_fn, 'w') tlog_fd.close() os0.set_tlog_file(FLOGTMP, new=True, echo=ctx['opt_echo']) try: tlog_fd = open(os0.tlog_fn, 'r') fexts = True tlog_fd.seek(0, os.SEEK_END) if tlog_fd.tell() == 0: fzero = True tlog_fd.close() except: pass sts = self.Z.test_result(ctx, "new logfile (1)", True, fexts) sts = self.Z.test_result(ctx, "new logfile (2)", True, fzero) if os.path.isfile(tlog_pathname): os.remove(tlog_pathname) tlog_pathname = os.path.join('.', FLOGTMP) fzero = False fexts = False if not ctx.get('dry_run', False): os0.set_tlog_file(tlog_pathname, new=True, echo=ctx['opt_echo']) try: tlog_fd = open(os0.tlog_fn, 'r') fexts = True tlog_fd.seek(0, os.SEEK_END) if tlog_fd.tell() == 0: fzero = True tlog_fd.close() except: pass sts = self.Z.test_result(ctx, "new logfile (3)", True, fexts) sts = self.Z.test_result(ctx, "new logfile (4)", True, fzero) if os.path.isfile(tlog_pathname): os.remove(tlog_pathname) tlog_pathname = os.path.join(self.Z.test_dir, FLOGTMP) fzero = False fexts = False if not ctx.get('dry_run', False): os0.set_tlog_file(ctx['logfn'], new=ctx['opt_new'], echo=ctx['opt_echo']) if ctx.get('dry_run', False): os0.wlog("Since now, test messages are store in", os0.tlog_fn) return sts
def msg_log(ctx, level, text): """Log a message and show if needed""" ident = ' ' * level if ctx: if 'test_unit_mode' in ctx: return elif ctx['dry_run'] and level > 0: txt = u"{0}({1})".format(ident, tounicode(text)) else: txt = u"{0}{1}".format(ident, tounicode(text)) else: txt = u"{0}{1}".format(ident, tounicode(text)) os0.wlog(txt)
def all_tests(self): sts = 0 for i in range(MAX_TEST_NUM): tname = "test_{0:02}".format(i) if hasattr(self, tname): sts = getattr(self, tname)() if sts: break # Result if sts == 0: os0.wlog("Ran {0} {1} tests".format(self.test_ctr, self.module_id)) else: os0.wlog("****** Test {0} failed ******".format(self.module_id)) return sts
def check_4_tkn_in_stdout(self, token): # Now search for this program name in output; # if found muteshell worked right! found_chunk = False try: stdout_fd = open(os0.setlfilename(os0.bgout_fn, 'r')) f = stdout_fd.read() if f.find(token) >= 0: found_chunk = True stdout_fd.close() except: pass if not found_chunk: os0.wlog("Test failed: muteshell did not work!!!") raise Exception("Test failed: muteshell did not work!!!")
def debug_msg_log(ctx, level, text): """Log a debug message and show if needed""" global db_msg_sp, db_msg_stack # if level == -999: # db_msg_sp += 1 # return ident = ' ' * abs(level) if ctx.get('dbg_mode', False): if 'test_unit_mode' in ctx: return elif ctx['dry_run'] and level > 0: txt = u">{0}({1})".format(ident, tounicode(text)) else: txt = u">{0}{1}".format(ident, tounicode(text)) print txt os0.wlog(txt)
def create_dict(self): self.dict = {} self.xtl = {} self.seed = 0 try: cnf_fd = open(self.fconf, "r") line = cnf_fd.readline() while line != "": i = line.rfind('\n') if i >= 0 and line[0:1] != "#": line = line.replace("\\ ", "\\b") line = re.sub('\\s+', ' ', line).strip() f = string.split(line, ' ') self.add_dict_entr(f[0], f[1], f[2]) line = cnf_fd.readline() except: os0.wlog("No dictionary file", self.fconf, "found!")
def change_file_ext(self, f): os0.wlog(" changing extension files") fsql = f + "-????????" + self.pre_ext f_ids = glob.glob(fsql) for f in f_ids: l = len(f) - len(self.pre_ext) fsql = f[0:l] + self.sql_ext if not os.path.isfile(fsql): ftmp = f if os.path.isfile(ftmp): try: os0.wlog("$ mv", ftmp, fsql) if not self.dry_run: # Rename old ext -> nex ext os.rename(ftmp, fsql) except: pass
def test_03(self, ctx): if not ctx.get('dry_run', False): wchar_string = u"- Unicode string àèìòù" os0.wlog(wchar_string) # if wlog fails follow statemente is not executed sts = self.Z.test_result(ctx, "Check for unicode support", True, True) if not ctx.get('dry_run', False): x = unichr(0x3b1) + unichr(0x3b2) + unichr(0x3b3) os0.wlog("- Greek letters", x) sts = self.Z.test_result(ctx, "- Greek letters", True, True) res = None ustr = None bstr = None if not ctx.get('dry_run', False): res = os0.str2bool('true', None) sts = self.Z.test_result(ctx, "str2bool(true)", True, res) if not ctx.get('dry_run', False): res = os0.str2bool('0', None) sts = self.Z.test_result(ctx, "str2bool(0)", False, res) if not ctx.get('dry_run', False): res = os0.str2bool(False, None) sts = self.Z.test_result(ctx, "str2bool(0)", False, res) if not ctx.get('dry_run', False): res = os0.str2bool('invalid', False) sts = self.Z.test_result(ctx, "str2bool(0)", False, res) if not ctx.get('dry_run', False): res = os0.nakedname('myfile') sts = self.Z.test_result(ctx, "nakedname(myfile)", 'myfile', res) if not ctx.get('dry_run', False): res = os0.nakedname('myfile.py') sts = self.Z.test_result(ctx, "nakedname(myfile.py)", 'myfile', res) if not ctx.get('dry_run', False): bstr = 'text àèìòù' ustr = u"text àèìòù" res = os0.u(bstr) sts = self.Z.test_result(ctx, "unicode(string)", ustr, res) sts = self.Z.test_result(ctx, "unicode(string)", ustr, os0.u(ustr)) if not ctx.get('dry_run', False): bstr = 'text àèìòù' ustr = u"text àèìòù" res = os0.b(ustr) sts = self.Z.test_result(ctx, "bstring(string)", bstr, res) sts = self.Z.test_result(ctx, "bstring(string)", os0.b(bstr), res) return sts
def gen_db_list(self, dbtype, user, sqlcmd, ctx): # pdb.set_trace() dblist = [] os0.wlog(" Creating", dbtype, "db list") if dbtype == "psql": cmd = sqlcmd + " -U" + user + " -l" cmdlog = cmd elif dbtype == "mysql": cmd = sqlcmd + " -u " + user + \ " --password="******" -e \"show databases;\" mysql" cmdlog = sqlcmd + " -u " + user + " -e \"show databases;\" mysql" else: cmd = "" cmdlog = cmd os0.trace_debug("$", cmdlog) os0.muteshell(cmd, simulate=self.dry_run, keepout=True) if ctx['db_name']: sel_db = ctx['db_name'] else: sel_db = '.*' if os0.debug_mode: os0.wlog("> DB selection", sel_db) stdinp_fd = open(os0.setlfilename(os0.bgout_fn), 'r') line = stdinp_fd.readline() while line != "": i = line.rfind('\n') if i >= 0: if dbtype == "psql": if line[0:1] == ' ' and line[1:2] != ' ': x = line.split('|') dbname = x[0].strip() if re.match("z[ei].*|demo.*", dbname) and \ re.match(sel_db, dbname): dblist.append(dbname) if os0.debug_mode: os0.wlog("> dblist.append({0})".format(dbname)) elif dbtype == "mysql": dbname = line.strip() if re.match("w.*|mg.*|assioma.*", dbname) and \ re.match(sel_db, dbname): dblist.append(dbname) if os0.debug_mode: os0.wlog("> dblist.append({0})".format(dbname)) line = stdinp_fd.readline() stdinp_fd.close() if not os0.debug_mode and not self.dry_run and os0.bgout_fn != "": os.remove(os0.setlfilename(os0.bgout_fn, 'r')) return dblist
def test_02(self, ctx): fzero = False fexts = False if not ctx.get('dry_run', False): title = TITLE os0.trace_debug("- Inititializing ...") os0.set_debug_mode(False) os0.trace_debug("TEST FAILED!!!!") os0.set_debug_mode(True) os0.trace_debug("- Inititializing (2) ...") os0.wlog(title) os0.wlog("- os0 version:", os0.version) os0.wlog("- platform:", _platform) os0.set_tlog_file("os0_test.log", echo=ctx['opt_echo']) try: tlog_fd = open(os0.tlog_fn, 'r') fexts = True tlog_fd.seek(0, os.SEEK_END) if tlog_fd.tell() == 0: fzero = True tlog_fd.close() except: pass sts = self.Z.test_result(ctx, "Check for empty tracelog", True, fexts) if sts == TEST_SUCCESS: sts = self.Z.test_result(ctx, "Check for empty tracelog", True, fzero) return sts
def restore_file(self, fqn): # pdb.set_trace() dbtype = "" # Extract dir if supplied p = os.path.dirname(fqn) f = os.path.basename(fqn) # Just filename # No dir supplied if p == "": p = self.ftp_dir elif p == "/var/lib/pgsql/backups": dbtype = "psql" elif p == "/var/lib/mysql/backups": dbtype = "mysql" if dbtype != self.dbtype: if dbtype == "psql": cmd = "service postgresql restart" os0.trace_debug("$", cmd) os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode) elif dbtype == "mysql": cmd = "service mysqld restart" os0.trace_debug("$", cmd) os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode) if p != self.ftp_dir: # Change dir self.chdir(p) # Set directory l = len(self.sql_ext) + 9 # i = len(f) - l # Extract dbname from XXXXX-YYYYMMDD.SQL dbname = f[0:-l] # if dbname == "wp-zi-it": # os0.wlog(" db", dbname, "not upgradable!!!") if os.path.isfile(f): self.restore_db(dbtype, dbname, fqn) else: os0.wlog(" file", f, "not found!!!")
def main(): """Tool main""" sts = 0 # pdb.set_trace() ctx = parse_args(sys.argv[1:], version=version(), doc=__doc__) if ctx['do_list']: print ctx['saveset_list'] return sts RI = Restore_Image(ctx) f_alrdy_run = check_if_running(ctx, RI.pid) if f_alrdy_run: os0.wlog("({0}) ***Another instance is running!!!".format(RI.pid)) try: ls_fd = open(RI.flist, "r") except: raise Exception("Command aborted: file list not found!!!") file_2_restore = [] fl = ls_fd.readline() while fl != "": i = fl.rfind('\n') if i >= 0 and fl[0:1] != '#': f = fl[0:i] file_2_restore.append(f) fl = ls_fd.readline() # Restore files for fl in file_2_restore: fn = os.path.basename(fl) if fn != "restconf" \ and fn != "restconf.py" \ and fn != "restconf.ini" \ and fn != "restconf-0.ini": f = "{0}.new".format(fl) if os.path.isfile(f): RI.restore_file(fl) else: os0.wlog(" file", fl, "not found!!!") return sts
def main(): """Tool main""" sts = 0 # pdb.set_trace() ctx = parse_args(sys.argv[1:], version=version(), doc=__doc__) if ctx['do_list']: print ctx['saveset_list'] return sts RI = Restore_Image(ctx) f_alrdy_run = check_if_running(ctx, RI.pid) if f_alrdy_run: os0.wlog("({0}) ***Another instance is running!!!".format(RI.pid)) # Restore files file_r_ctr = 0 file_u_ctr = 0 time_wait = 60 wait_loop = 3 if not f_alrdy_run: fl = RI.extract_fn_2_restore() loop_ctr = wait_loop while loop_ctr > 0: if fl != "": file_r_ctr = file_r_ctr + 1 if os.path.isfile(fl): RI.restore_file(fl) file_u_ctr += 1 if file_u_ctr > 1: wait_loop = 60 loop_ctr = wait_loop else: os0.wlog(" file", fl, "not found!!!") RI.commit_fn_restored() fl = RI.extract_fn_2_restore() if fl == "": os0.wlog(" wait for next db") time.sleep(time_wait) loop_ctr -= 1 if not ctx['dbg_mode'] and os.path.isfile(os0.setlfilename(os0.bgout_fn)): os.remove(os0.setlfilename(os0.bgout_fn)) if not f_alrdy_run: os0.wlog("Restore DB ended." " {0} DB to restore, {1} DB restored ({2}).".format( file_u_ctr, file_u_ctr, RI.pid)) return sts
def main(self): """Unit test main.""" # Check for target software version required_version = os.environ.get(self.module_id.upper() + '_VERSION') if not required_version: required_version = REQ_TEST_VERSION pkg_version = build.version() if required_version and pkg_version.find(required_version) < 0: print "Test not executable: invalid version!" print "Required version:", required_version print "Package version:", pkg_version return TEST_FAILED # Need debug mode to avoid security fault in Linux os0.set_debug_mode(True) title = "%s regression test. Version: %s" % (self.module_id, REQ_TEST_VERSION) # Remove test log file if executed previous crashed test tlog_fn = self.module_id + "_test.log" os0.set_tlog_file(tlog_fn) tlog_pathname = os0.tlog_fn # Set no file log os0.set_tlog_file('', echo=True) if os.path.isfile(tlog_pathname): os.remove(tlog_pathname) # Test if running in travis-ci emulator (DEV_ENVIRONMENT) if 'DEV_ENVIRONMENT' in os.environ \ and os.environ['DEV_ENVIRONMENT'].find(self.module_id): LOCAL_ECHO = False else: LOCAL_ECHO = True tlog_fn = './' + self.module_id + "_test.log" os0.set_tlog_file(tlog_fn, new=True, echo=LOCAL_ECHO) os0.wlog(title) # Test execution body test_num = 0 sts = 0 for i in range(MAX_TEST_NUM): tname = "test_{0:02}".format(test_num) if hasattr(self, tname): sts = getattr(self, tname)(test_num) if sts: break test_num += 1 # Result if sts == 0: os0.wlog("Ran {0} {1} tests".format(self.test_ctr, MODULE_ID)) else: os0.wlog("****** Test {0} failed ******".format(MODULE_ID)) return sts
def Backup_files(self, cfg_obj, sset, prm): path_list = cfg_obj.get(sset, "path") if path_list == "": os0.wlog(self.pid, "No file declared!!") if not prm['mute']: print "***No file declared file!!!" return prm['cmd_bck'] = cfg_obj.get(sset, "cmd_bck").replace('{}', '{0}') prm['cp_mode'] = cfg_obj.get(sset, "cp_mode") path_list = path_list.split(',') self.init_bck(prm, path_list[0]) file_2_backup = [] for path in path_list: if os.path.isdir(path): path = os.path.join(path, "*") if prm['specimen'] == "file": file_2_backup = file_2_backup + glob.glob(path) elif prm['specimen'] == "dir": for root, dirs, files in os.walk(path): for f in files: file_2_backup = file_2_backup + os.path.join(root, f) else: pass if prm['cmd_bck'] != "": cmd = prm['cmd_bck'].replace('{0}', path) os0.trace_debug("> ", cmd) os0.muteshell(cmd, simulate=prm['simulate']) self.set_chdir(path_list[0]) for f in file_2_backup: if os.path.isfile(f): self.add_2_ftp(f, prm) elif os.path.isdir(f): os0.wlog(self.pid, " dir", f, "skipped") else: os0.wlog(self.pid, " file", f, "not found!!!") return
def restore_file(self, fqn): # Extract dir if supplied p = os.path.dirname(fqn) f = os.path.basename(fqn) # Just filename # No dir supplied if p == "": p = self.ftp_dir if p != self.ftp_dir: # Change dir self.chdir(p) # Set directory fzero = False fexts = False fn = "{0}.new".format(f) ftmp = "{0}.tmp".format(f) fbak = "{0}.bak".format(f) try: fn_fd = open(fn, 'r') # Go to end of file fn_fd.seek(0, os.SEEK_END) # File len = 0 ? if fn_fd.tell() == 0: fzero = True # Go to begin of file fn_fd.seek(0, 0) # Read entire file fn_str = fn_fd.read() fn_fd.close() except: fzero = True if fzero: os0.wlog(" file", fn, "empty!!!") else: os0.wlog(" ", fn, "->", fqn) # Search for text substitution key_ids = self.search4item(f) if key_ids: # Text couples for substitution for key in key_ids: src = self.xtl[key][0] tgt = self.xtl[key][1] # print "[{0} >subst/{1}/{2}/".format(fqn, src, tgt) os0.wlog(" ", fqn, ":", src, "->", tgt) # Substitute text in file fn_str = fn_str.replace(src, tgt) ftmp_fd = open(ftmp, 'w') # write file with substitutions ftmp_fd.write(fn_str) ftmp_fd.close() else: # Rename file.new -> file.tmp os.rename(fn, ftmp) try: f_fd = open(f, 'r') f_fd.close() fexts = True with open(os.devnull, "w") as fdnull: cmd = "diff" p1 = f p2 = ftmp sts = call([cmd, p1, p2], stdout=fdnull, stderr=fdnull) # New file for upgrade if sts > 0: os0.wlog(" file", f, "upgraded!!!") else: os0.wlog(" file", f, "not changed.") fzero = True fdnull.close() except: os0.wlog(" file", f, "new!!!") try: # Delete file.new (if exist) os.remove(fn) except: pass if fzero: # No upgrade try: # Delete file.tmp (if exist) os.remove(ftmp) except: pass else: # Upgrade try: # Delete file.bak (if exist) os.remove(fbak) except: pass if fexts: # Rename file -> file.bak os.rename(f, fbak) if not self.dry_run: # Rename file.tmp -> file os.rename(ftmp, f)
def main(): """Tool main""" sts = 0 # pdb.set_trace() ctx = parse_args(sys.argv[1:], version=version(), doc=__doc__) if ctx['do_list']: print ctx['saveset_list'] return sts BM = Backup_Mirror(ctx) # Backup files in root directory BM.chdir("/root") file_2_backup = ["av_php", "bckconf", "bckconf.py", "bckdb", "bckdb.py", "bckwww", "cldb", "purgedb", "restconf", "restconf.py", "restconf.ini", "restconf-0.ini", "restdb", "restdb.py", "restwww", "ssl_certificate", "statdb", "zarlib.py"] for fl in file_2_backup: if os.path.isfile(fl): os0.wlog(" ", fl) BM.add_2_ftp(fl) else: os0.wlog(" file", fl, "not found!!!") # Backup configuration file for http server BM.chdir("/etc/httpd/conf/sites-enabled") file_2_backup = glob.glob("/etc/httpd/conf/sites-enabled/*.conf") for f in file_2_backup: fl = os.path.basename(f) # Just filename if os.path.isfile(fl): os0.wlog(" ", fl) BM.add_2_ftp(fl) else: os0.wlog(" file", fl, "not found!!!") # Backup web files BM.chdir("/var/www/html") file_2_backup = ["openerp/index.html"] for fl in file_2_backup: if os.path.isfile(fl): os0.wlog(" ", fl) BM.add_2_ftp(fl) else: os0.wlog(" file", fl, "not found!!!") # Backup web certificates BM.chdir("/var/www/certs") file_2_backup = ["shop.zeroincombenze.it.crt", "shop.zeroincombenze.it.CA.crt", "shop.zeroincombenze.it.pem", "19340476.crt", "19340476.ca-bundle", "19340476.pem", "zi-oe.csr", "zi-oe.pem", "www.shs-av.com.csr", "shs-av.com.pem", "StartCom.CA.csr"] for fl in file_2_backup: if os.path.isfile(fl): os0.wlog(" ", fl) BM.add_2_ftp(fl) else: os0.wlog(" file", fl, "not found!!!") # Backup postgres configuration files BM.chdir("/var/lib/pgsql/data") # file_2_backup = ["pg_hba.conf", ] # for fl in file_2_backup: # if os.path.isfile(fl): # os0.wlog(" ", fl) # BM.add_2_ftp(fl) # else: # os0.wlog(" file", fl, "not found!!!") BM.exec_bck() return sts
def repl_data(self, dbname, fqn): fzero = False try: fqn_fd = open(fqn, 'r') # Go to end of file fqn_fd.seek(0, os.SEEK_END) # File len = 0 ? if fqn_fd.tell() == 0: fzero = True # Go to begin of file fqn_fd.seek(0, 0) # Read entire file fqn_str = fqn_fd.read() fqn_fd.close() except: fzero = True if fzero: os0.wlog(" file", fqn, "empty!!!") else: fxch = False # Search for text substitution (Wordpress) f = dbname + "->wp" ctx = self.get_params(f) if ctx['prefix'] != "" and ctx['siteURL'] != "": fxch = True fqn_str = self.repl_data_wp(ctx, fqn_str) # Search for sql command to append f = dbname + "/" key_ids = self.search4item(f) if key_ids: fxch = True # Text couples for substitution for key in key_ids: src = self.xtl[key][0] src = src.replace("\\b", " ") tgt = self.xtl[key][1] tgt = tgt.replace("\\b", " ") os0.trace_debug(">", src, tgt, ";") fqn_str = fqn_str + src + " " + tgt + ";\n" # Search for text substitution in SQL statements f = dbname + self.sql_ext key_ids = self.search4item(f) if key_ids: fxch = True # Text couples for substitution for key in key_ids: src = self.xtl[key][0] src = src.replace("\\b", " ") tgt = self.xtl[key][1] tgt = tgt.replace("\\b", " ") os0.trace_debug("> sed|{0}|{1}|".format(src, tgt)) fqn_str = fqn_str.replace(src, tgt) if fxch: fqn_fd = open(fqn, 'w') fqn_fd.write(fqn_str) fqn_fd.close() f = dbname + "->wiki" ctx = self.get_params(f) if ctx['siteURL'] != "": fqns = ctx['conf_file'].split(',') for fqn in fqns: self.replace_file(ctx, f, fqn) if ctx['conf_file2']: fqns = ctx['conf_file2'].split(',') for fqn in fqns: self.replace_file(ctx, f, fqn) if ctx['conf_file3']: fqns = ctx['conf_file3'].split(',') for fqn in fqns: self.replace_file(ctx, f, fqn) fqn = ctx['index_html'] self.replace_file(ctx, f, fqn)
def restore_db(self, dbtype, dbname, fqn): # pdb.set_trace() os0.wlog(" restoring", dbname, " ({0})".format(fqn)) homedir = os.path.expanduser("~") tar_ext = self.tar_ext tar_opt = self.tar_opt fzip_fn = dbname + tar_ext if not os.path.isfile(fzip_fn): if self.tar_ext == ".gz": tar_ext = ".bz2" tar_opt = "j" fzip_fn = dbname + tar_ext if not os.path.isfile(fzip_fn): tar_ext = self.tar_ext tar_opt = self.tar_opt # No compressed file found fzip_fn = "" elif self.tar_ext == ".bz2": tar_ext = ".gz" tar_opt = "z" fzip_fn = dbname + tar_ext if not os.path.isfile(fzip_fn): tar_ext = self.tar_ext tar_opt = self.tar_opt # No compressed file found fzip_fn = "" f = os.path.basename(fqn) # Just filename l = len(self.sql_ext) + 9 i = len(f) - l # Extract date (YYYYMMDD) from XXXXX-YYYYMMDD.SQL dts = f[i + 1:i + 9] if dbtype == "psql": cmd = "chown " + self.psql_uu + ":" + self.psql_uu + " " + fqn elif dbtype == "mysql": cmd = "chown " + self.mysql_uu + ":" + self.mysql_uu + " " + fqn os0.trace_debug("$", cmd) os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode) sql_fn = homedir + "/restdb.sql" cmd = "cp " + fqn + " " + sql_fn os0.trace_debug("$", cmd) os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode) # cmd = "sed -i -e \"s|Owner: openerp|Owner: odoo|g\""\ # " -e \"s|OWNER TO openerp|OWNER TO odoo|g\" ~/restdb.sql" # os0.trace_debug("$", cmd) # os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode) if dbtype == "psql": cmd = "chown " + self.psql_uu + ":" + self.psql_uu + " " + sql_fn elif dbtype == "mysql": cmd = "chown " + self.mysql_uu + ":" + self.mysql_uu + " " + sql_fn os0.trace_debug("$", cmd) os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode) self.repl_data(dbname, sql_fn) psh_fn = homedir + "/restdb.psh" psh_fd = open(psh_fn, "w") if dbtype == "psql": user = self.psql_uu defdb = self.psql_db psh_fd.write("\\c {0}\n".format(defdb)) psh_fd.write("DROP DATABASE IF EXISTS \"{0}-{1}\";\n".format( dbname, dts)) psh_fd.write("DROP DATABASE IF EXISTS \"{0}\";\n".format(dbname)) psh_fd.write( "CREATE DATABASE \"{0}\" TEMPLATE template1;\n".format(dbname)) psh_fd.write("\\c \"{0}\"\n".format(dbname)) psh_fd.write("\\i {0}\n".format(sql_fn)) psh_fd.write( "ALTER DATABASE \"{0}\" OWNER TO odoo;\n".format(dbname)) cmd = "psql -f " + psh_fn + " -U" + user + " " + defdb psh_fd.close() os0.trace_debug("$", cmd) os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode) elif dbtype == "mysql": user = "******" pwd = "SHS13mgr" # defdb = self.psql_db psh_fd.write( "mysqladmin -u{0} --password={1} -f drop \"{2}-{3}\" ||true\n". format(user, pwd, dbname, dts)) psh_fd.write( "mysqladmin -u{0} --password={1} -f drop \"{2}\" || true\n". format(user, pwd, dbname)) psh_fd.write( "mysqladmin -u{0} --password={1} -f create \"{2}\"\n".format( user, pwd, dbname)) psh_fd.write( "mysql -u{0} --password=SHS13mgr -G -e \"source {1}\" {2}\n". format(user, sql_fn, dbname)) psh_fd.close() cmd = "chmod +x " + psh_fn os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode) cmd = psh_fn os0.trace_debug("$", cmd) os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode) else: os0.wlog(" unknown", dbname, "database type!!!") cmd = "echo Error" # Compressed file found if fzip_fn != "": if dbtype == "psql": cmd = "chown " + self.psql_uu + \ ":" + self.psql_uu + " " + fzip_fn elif dbtype == "mysql": cmd = "chown " + self.mysql_uu + \ ":" + self.mysql_uu + " " + fzip_fn os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode) cmd = "tar --keep-newer-files -x" + tar_opt + "f " + fzip_fn os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode) if not self.dry_run: os.remove(fzip_fn) self.purge_db(dbtype, dbname)
def purge_db(self, dbtype, f): # pdb.set_trace() if self.sql_ext != self.pre_ext: self.change_file_ext(f) dtc = date.today() - timedelta(90) os0.wlog(" removing file older than", dtc.strftime("%Y-%m-%d")) fzip_fn = f + self.tar_ext force_change_ext = False for i in range(180, 120, -1): dtc = datetime.today() - timedelta(i) dts = dtc.strftime("%Y%m%d") fsql = f + "-" + dts + self.sql_ext if not os.path.isfile(fsql) and self.sql_ext != self.pre_ext: ftmp = f + "-" + dts + self.pre_ext if os.path.isfile(ftmp): try: os0.wlog("$ mv", ftmp, fsql) if not self.dry_run: # Rename old ext -> nex ext os.rename(ftmp, fsql) # Force change sql file extension force_change_ext = True except: pass if dtc.day != 1: if not self.remove_sql_file(fsql) \ and self.sql_ext != self.pre_ext: fsql = f + "-" + dts + self.pre_ext self.remove_sql_file(fsql) if force_change_ext: self.change_file_ext(f) fsql = f + "-????????" + self.sql_ext if dbtype == "psql": cmd = "chown " + self.psql_uu + ":" + self.psql_uu + " " + fsql elif dbtype == "mysql": cmd = "chown " + self.mysql_uu + ":" + self.mysql_uu + " " + fsql os0.trace_debug("$ ", cmd) os0.muteshell(cmd, simulate=self.dry_run) cmd = "tar --remove-files -c" + \ self.tar_opt + "f " + fzip_fn + " " + fsql os0.trace_debug("$ ", cmd) os0.muteshell(cmd, simulate=self.dry_run) if dbtype == "psql": cmd = "chown " + self.psql_uu + ":" + self.psql_uu + " " + fzip_fn elif dbtype == "mysql": cmd = "chown " + self.mysql_uu + \ ":" + self.mysql_uu + " " + fzip_fn os0.trace_debug("$ ", cmd) os0.muteshell(cmd, simulate=self.dry_run) os0.wlog(" removing archived files") fsql = f + "-????????" + self.sql_ext f_ids = sorted(glob.glob(fsql)) for fsql in f_ids: self.remove_sql_file(fsql)
def __init__(self, ctx): self.hostname = ctx['hostname'] os0.set_debug_mode(ctx['dbg_mode']) self.prodhost = ctx['production_host'] self.devhost = ctx['development_host'] self.mirrorhost = ctx['mirror_host'] self.pgdir = ctx['pg_dir'] self.mysqldir = ctx['mysql_dir'] homedir = os.path.expanduser("~") self.ftp_cfn = homedir + "/" + ctx['ftp_script'] self.flist = homedir + "/" + ctx['list_file'] os0.set_tlog_file(ctx['logfn']) # Log begin execution os0.wlog("Restore configuration files", __version__) # Simulate backup self.dry_run = ctx['dry_run'] if ctx['saveset'] == "bckdb" or \ ctx['saveset'] == "bckconf" or \ ctx['saveset'] == "bckwww": if self.hostname == self.prodhost: os0.wlog("Running on production machine") if ctx['alt']: self.bck_host = self.mirrorhost self.fconf = homedir + "/" + \ ctx['no_translation'] else: self.bck_host = self.devhost self.fconf = homedir + "/" + \ ctx['data_translation'] elif self.hostname == self.mirrorhost: os0.wlog("Running on mirror machine") if ctx['alt']: self.bck_host = self.prodhost self.fconf = homedir + "/" + \ ctx['no_translation'] else: self.bck_host = self.devhost self.fconf = homedir + "/" + \ ctx['data_translation'] elif self.hostname == self.devhost: os0.wlog("This command cannot run on development machine") if not ctx['dry_run']: raise Exception("Command aborted due invalid machine") else: os0.wlog("Unknown machine - Command aborted") if not ctx['dry_run']: raise Exception("Command aborted due unknown machine") elif ctx['saveset'] == "restdb" or \ ctx['saveset'] == "restconf" or \ ctx['saveset'] == "restwww": if self.hostname == self.prodhost: os0.wlog("This command cannot run on production machine") if not ctx['dry_run']: raise Exception("Command aborted due production machine") elif self.hostname == self.mirrorhost: os0.wlog("Running on mirror machine") if ctx['alt']: self.bck_host = self.prodhost self.fconf = homedir + "/" + \ ctx['no_translation'] else: self.bck_host = self.devhost self.fconf = homedir + "/" + \ ctx['data_translation'] elif self.hostname == self.devhost: os0.wlog("Running on development machine") if ctx['alt']: self.bck_host = self.mirrorhost self.fconf = homedir + "/" + \ ctx['data_translation'] else: self.bck_host = self.devhost self.fconf = homedir + "/" + \ ctx['data_translation'] else: os0.wlog("Unknown machine - Command aborted") if not ctx['dry_run']: raise Exception("Command aborted due unknown machine") # May be (.gz or .bz2) self.tar_ext = ctx['tar_ext'] # May be (z or j) self.tar_opt = ctx['tar_opt'] # May be (null or .sql) self.pre_ext = ctx['pre_ext'] # May be (null or .sql) self.sql_ext = ctx['sql_ext'] self.psql_uu = ctx['pgsql_user'] self.psql_db = ctx['pgsql_def_db'] self.mysql_uu = ctx['mysql_user'] self.mysql_db = ctx['mysql_def_db'] self.pid = os.getpid() self.ftp_rootdir = "" self.ftp_dir = "" self.dbtype = "" self.create_dict()
def test_00(self, test_num): if not self.open_conf(test_num): os0.wlog('test_%02d not defined in configuration file %s' % (test_num, self.test_conf_fn)) self.close_conf() return TEST_FAILED self.msg_new_test(test_num) prm = build.parse_args(['-btvT', 'pypi']) if 'template' not in prm or prm['template'] != 'pypi': os0.wlog('Test failed: option -T') return TEST_FAILED self.msg_new_test(test_num) if 'modname' not in prm or prm['modname'] != 'midea': os0.wlog('Test failed: default option -n') return TEST_FAILED self.msg_new_test(test_num) prm = build.parse_args(['-btvT', 'pypi', '-n', 'testcode']) if 'modname' not in prm or prm['modname'] != 'testcode': os0.wlog('Test failed: ption -n') return TEST_FAILED self.msg_new_test(test_num) conf_fn = self.module_id + '.conf' prm = build.parse_args(['-btvT', 'pypi', '-c', conf_fn], True) if os.path.isfile('/etc/odoo-server.conf'): if 'conf_fn' not in prm or\ prm['conf_fn'] != ['/etc/odoo-server.conf', conf_fn]: os0.wlog('Test failed: option -c') return TEST_FAILED elif os.path.isfile('/etc/openerp-server.conf'): if 'conf_fn' not in prm or\ prm['conf_fn'] != ['/etc/openerp-server.conf', conf_fn]: os0.wlog('Test failed: option -c') return TEST_FAILED self.close_conf() return TEST_SUCCESS