def test_09(self, z0ctx): sts = TEST_SUCCESS confn = '%s/test_clodoo.conf' % self.Z.test_dir if os.environ.get("HOSTNAME", "") == "shsdef16": for oe_version in ('6.1', '7.0', '8.0', '9.0', '10.0'): if not ctx['dry_run']: cmd = self.cmd xmlrpc_port, dbname = self.param_by_db(oe_version) codefile = """[options] actions=drop_db login_user=%s db_name=%s xmlrpc_port=%s oe_version=%s """ % (self.login_2_test, dbname, xmlrpc_port, oe_version) # DB exists, but due -q switch no response returns False fd = open(confn, 'w') fd.write(codefile) fd.close() if os.environ.get("TRAVIS", "") != "true": os0.muteshell("/opt/odoo/tools/zar/pg_db_active -wa " + dbname) cmd = cmd + ['-c%s' % confn] p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) res, err = p.communicate() res = self.check_4_db(dbname) else: res = False sts = self.Z.test_result(z0ctx, "Drop DB -b%s" % oe_version, False, res) return sts
def analyze(self, prm): cmd = "netstat -nat | awk '{print $6}' | sort | uniq -c | sort -n" os0.muteshell(cmd, simulate=self.simulate, keepout=True) stdinp_fd = open(os0.setlfilename(os0.bgout_fn), 'r') line = stdinp_fd.readline() total_conn = 0 total_dos_conn = 0 total_overload = 0 while line != "": fields = line.strip().split() # Check for DOS attack if fields[1] == "SYS_SENT": total_dos_conn += int(fields[0]) # Check for overload elif fields[1] == "SYN_RECV" or fields[1] == "TIME_WAIT": total_overload += int(fields[0]) total_conn += int(fields[0]) line = stdinp_fd.readline() stdinp_fd.close() try: os.remove(os0.setlfilename(os0.bgout_fn)) except: pass perc_dos_conn = (total_dos_conn * 100) / total_conn perc_overload = (total_overload * 100) / total_conn prm['total_conn'] = total_conn prm['total_dos_conn'] = total_dos_conn prm['total_overload'] = total_overload prm['perc_dos_conn'] = perc_dos_conn prm['perc_overload'] = perc_overload return prm
def add_2_ftp(self, fl): # Add filename to ftp file list lx = ("cldb", "bckconf", "bckdb", "bckdb.py", "bckwww", "purgedb", "restconf", "restconf.py", "restdb", "restdb.py", "restwww", "statdb", ".zar.conf") # Extract subdir if supplied p = os.path.dirname(fl) fn = os.path.basename(fl) # Just filename if p == "" and os.path.dirname(__file__) != "" \ and (fn == "restconf.py" or fn == "restdb.py"): os0.trace_debug("Development & Testing software execution!") p = os.path.dirname(__file__) # No dir supplied if p == "": # If prior subdir .. if self.ftp_dir != "": # .. return to root dir self.set_chdir(self.ftp_rootdir) # Forget prior subdir self.ftp_dir = "" fqn = self.ftp_rootdir + '/' + fl elif p[0:1] == '/' or p[0:1] == '.': fqn = p + '/' + fn if fn in lx: cmd = "chmod +x {0}".format(fqn) os0.trace_debug("$ ", cmd) os0.muteshell(cmd, simulate=self.dry_run) fqn = self.ftp_rootdir + '/' + fn # Set local directory self.set_lchdir(p) self.ftp_dir = p else: fqn = self.ftp_rootdir + '/' + p + '/' + fl if p != self.ftp_dir: # Change subdir # Make full dir path (root + sub) lpath = self.ftp_rootdir + '/' + p self.set_chdir(lpath) # Set directory self.ftp_dir = p # Remember subdir self.ls_fd.write("{0}\n".format(fqn)) if fn == "restconf.py" or fn == "restdb.py": cmd = "chmod +x {0}".format(fqn) os0.trace_debug("$ ", cmd) os0.muteshell(cmd, simulate=self.dry_run) if fn == "restconf" or fn == "restconf.py": self.ftp_fd.write("-rm {0}.bak\n".format(fn)) self.ftp_fd.write("-rename {0} {0}.bak\n".format(fn)) self.ftp_fd.write("put {0}\n".format(fn)) elif fn == "restconf.ini" or fn == "restconf-0.ini": self.ftp_fd.write("-rm {0}.bak\n".format(fn)) self.ftp_fd.write("-rename {0} {0}.bak\n".format(fn)) self.ftp_fd.write("put {0}\n".format(fn)) else: self.ftp_fd.write("put {0} {0}.new\n".format(fn))
def gen_db_list(self, dbtype, user, sqlcmd, ctx): # pdb.set_trace() dblist = [] os0.wlog(" Creating", dbtype, "db list") if dbtype == "psql": cmd = sqlcmd + " -U" + user + " -l" cmdlog = cmd elif dbtype == "mysql": cmd = sqlcmd + " -u " + user + \ " --password="******" -e \"show databases;\" mysql" cmdlog = sqlcmd + " -u " + user + " -e \"show databases;\" mysql" else: cmd = "" cmdlog = cmd os0.trace_debug("$", cmdlog) os0.muteshell(cmd, simulate=self.dry_run, keepout=True) if ctx['db_name']: sel_db = ctx['db_name'] else: sel_db = '.*' if os0.debug_mode: os0.wlog("> DB selection", sel_db) stdinp_fd = open(os0.setlfilename(os0.bgout_fn), 'r') line = stdinp_fd.readline() while line != "": i = line.rfind('\n') if i >= 0: if dbtype == "psql": if line[0:1] == ' ' and line[1:2] != ' ': x = line.split('|') dbname = x[0].strip() if re.match("z[ei].*|demo.*", dbname) and \ re.match(sel_db, dbname): dblist.append(dbname) if os0.debug_mode: os0.wlog("> dblist.append({0})".format(dbname)) elif dbtype == "mysql": dbname = line.strip() if re.match("w.*|mg.*|assioma.*", dbname) and \ re.match(sel_db, dbname): dblist.append(dbname) if os0.debug_mode: os0.wlog("> dblist.append({0})".format(dbname)) line = stdinp_fd.readline() stdinp_fd.close() if not os0.debug_mode and not self.dry_run and os0.bgout_fn != "": os.remove(os0.setlfilename(os0.bgout_fn, 'r')) return dblist
def check_if_running(ctx, pid): f_alrdy_run = False id_str = ctx['caller'] + ".py" cmd = "ps aux|grep " + id_str os0.muteshell(cmd, keepout=True) stdinp_fd = open(os0.setlfilename(os0.bgout_fn), 'r') rxmatch = "root .* python .*" + id_str + ".*" rxnmatch = "root .* {0} .*".format(pid) line = stdinp_fd.readline() while line != "" and not f_alrdy_run: i = line.rfind('\n') if i >= 0: if re.match(rxmatch, line) and not re.match(rxnmatch, line): f_alrdy_run = True line = stdinp_fd.readline() stdinp_fd.close() if os.path.isfile(os0.setlfilename(os0.bgout_fn)): os.remove(os0.setlfilename(os0.bgout_fn)) return f_alrdy_run
def commit_fn_restored(self): ftmp = self.flist + ".lst" fbak = self.flist + ".bak" if os.path.isfile(ftmp): fn_fd = open(ftmp, 'r') fzero = True fl = fn_fd.readline() while fl != "" and fzero: i = fl.rfind('\n') if i >= 0: fzero = False fl = fn_fd.readline() fn_fd.close() if not fzero: cmd = "rm -f {2}; mv {0} {2}; mv {1} {0}".format( self.flist, ftmp, fbak) os0.trace_debug("$ ", cmd) os0.muteshell(cmd, simulate=self.dry_run) else: if not self.dry_run: os.remove(ftmp)
def exec_bck(self): # Close files list self.ls_fd.close() self.set_chdir("/root") # Copy files list too self.ftp_fd.write("put {0} {0}.tmp\n".format( os.path.basename(self.flist))) self.ftp_fd.close() self.ftp_fd = None fn = os.path.basename(self.ftp_cfn) if _platform == "win32": cmd = "ftp" p1 = "-s:" + fn p2 = self.bck_host else: cmd = "sftp" p1 = "-b" + fn p2 = "root@" + self.bck_host cmd = cmd + " " + p1 + " " + p2 os0.muteshell(cmd, simulate=self.dry_run, tlog=True) if not self.dry_run: # os.remove(self.ftp_cfn) # os.remove(self.flist) # Delete # files list cmd = "ssh root@{0} \"cat {1}.tmp>>{1}; rm {1}.tmp\"".format( self.bck_host, self.flist) os0.muteshell(cmd, keepout=os0.debug_mode, tlog=True) cmd = "ssh root@" + self.bck_host + " \"at -f ./restdb now\"" os0.muteshell(cmd, keepout=os0.debug_mode, tlog=True)
def restore_file(self, fqn): # pdb.set_trace() dbtype = "" # Extract dir if supplied p = os.path.dirname(fqn) f = os.path.basename(fqn) # Just filename # No dir supplied if p == "": p = self.ftp_dir elif p == "/var/lib/pgsql/backups": dbtype = "psql" elif p == "/var/lib/mysql/backups": dbtype = "mysql" if dbtype != self.dbtype: if dbtype == "psql": cmd = "service postgresql restart" os0.trace_debug("$", cmd) os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode) elif dbtype == "mysql": cmd = "service mysqld restart" os0.trace_debug("$", cmd) os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode) if p != self.ftp_dir: # Change dir self.chdir(p) # Set directory l = len(self.sql_ext) + 9 # i = len(f) - l # Extract dbname from XXXXX-YYYYMMDD.SQL dbname = f[0:-l] # if dbname == "wp-zi-it": # os0.wlog(" db", dbname, "not upgradable!!!") if os.path.isfile(f): self.restore_db(dbtype, dbname, fqn) else: os0.wlog(" file", f, "not found!!!")
def test_03(self, z0ctx): sts = TEST_SUCCESS if os.environ.get("HOSTNAME", "") == "shsdef16": # or os.getcwd[0:19] != "/opt/odoo/dev/pypi/"): for oe_version in ('6.1', '7.0', '8.0', '9.0', '10.0'): if not ctx['dry_run']: cmd = self.bulk_cmd(oe_version) if os.environ.get("TRAVIS", "") != "true": xmlrpc_port, dbname = self.param_by_db(oe_version) os0.muteshell("/opt/odoo/tools/zar/pg_db_active -wa " + dbname) os0.muteshell("dropdb -Upostgres --if-exists " + dbname) cmd = cmd + ['-Anew_db'] p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) res, err = p.communicate() res = self.check_4_db(dbname) else: res = True sts = self.Z.test_result(z0ctx, "Create DB -b%s" % oe_version, True, res) if not ctx['dry_run']: cmd = self.bulk_cmd(oe_version) cmd = cmd + ['-Ashow_params'] p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) res, err = p.communicate() if oe_version in ('6.1', '7.0', '8.0'): xml_prot = 'xmlrpc' elif oe_version in ('9.0', '10.0'): xml_prot = 'jsonrpc' if re.search('protocol *= *%s' % xml_prot, res): res = True else: res = False sts = self.Z.test_result( z0ctx, "Show db params -b%s" % oe_version, True, res) return sts
def Backup_files(self, cfg_obj, sset, prm): path_list = cfg_obj.get(sset, "path") if path_list == "": os0.wlog(self.pid, "No file declared!!") if not prm['mute']: print "***No file declared file!!!" return prm['cmd_bck'] = cfg_obj.get(sset, "cmd_bck").replace('{}', '{0}') prm['cp_mode'] = cfg_obj.get(sset, "cp_mode") path_list = path_list.split(',') self.init_bck(prm, path_list[0]) file_2_backup = [] for path in path_list: if os.path.isdir(path): path = os.path.join(path, "*") if prm['specimen'] == "file": file_2_backup = file_2_backup + glob.glob(path) elif prm['specimen'] == "dir": for root, dirs, files in os.walk(path): for f in files: file_2_backup = file_2_backup + os.path.join(root, f) else: pass if prm['cmd_bck'] != "": cmd = prm['cmd_bck'].replace('{0}', path) os0.trace_debug("> ", cmd) os0.muteshell(cmd, simulate=prm['simulate']) self.set_chdir(path_list[0]) for f in file_2_backup: if os.path.isfile(f): self.add_2_ftp(f, prm) elif os.path.isdir(f): os0.wlog(self.pid, " dir", f, "skipped") else: os0.wlog(self.pid, " file", f, "not found!!!") return
def exec_bck(self): # Close files list self.ls_fd.close() self.set_chdir("/root") # Copy files list too self.ftp_fd.write("put {0}\n".format(self.flist)) self.ftp_fd.close() fn = os.path.basename(self.ftp_cfn) if _platform == "win32": cmd = "ftp" p1 = "-s:" + fn p2 = self.bck_host else: cmd = "sftp" p1 = "-b" + fn p2 = "root@" + self.bck_host cmd = cmd + " " + p1 + " " + p2 os0.muteshell(cmd, simulate=self.dry_run, tlog=True) if not self.dry_run: os.remove(self.ftp_cfn) # Delete files list os.remove(self.flist) cmd = "ssh root@" + self.bck_host + " \"./restconf.py\"" os0.muteshell(cmd, keepout=os0.debug_mode, tlog=True)
def test_06(self, ctx): if not ctx.get('dry_run', False): if os.path.dirname(__file__) == "": if __file__ == "__main__.py": cmd = "dir " + os0.setlfilename("../") else: cmd = "dir " + os0.setlfilename("./") else: cmd = "dir " + os0.setlfilename(os.path.dirname(__file__)) try: os.remove(os0.setlfilename(os0.bgout_fn)) except: pass os0.muteshell(cmd, keepout=True) self.check_4_tkn_in_stdout(os.path.basename(__file__)) sts = self.Z.test_result(ctx, "os0.dir", True, os.path.isfile(os0.bgout_fn)) if not ctx.get('dry_run', False): if _platform == "win32": cmd = "del" elif _platform == "OpenVMS": cmd = "delete" else: cmd = "rm" cmd = cmd + " " + (os0.setlfilename(os0.bgout_fn)) os0.muteshell(cmd, simulate=True, tlog=True) self.check_4_tkn_in_stdout(cmd) if sts == TEST_SUCCESS: sts = self.Z.test_result(ctx, "os0.del", True, os.path.isfile(os0.bgout_fn)) if not ctx.get('dry_run', False): if not ctx.get('dry_run', False): cmd = "dir" os0.muteshell(cmd) if sts == TEST_SUCCESS: sts = self.Z.test_result(ctx, "os0.dir", False, os.path.isfile(os0.bgout_fn)) return sts
def purge_db(self, dbtype, f): # pdb.set_trace() if self.sql_ext != self.pre_ext: self.change_file_ext(f) dtc = date.today() - timedelta(90) os0.wlog(" removing file older than", dtc.strftime("%Y-%m-%d")) fzip_fn = f + self.tar_ext force_change_ext = False for i in range(180, 120, -1): dtc = datetime.today() - timedelta(i) dts = dtc.strftime("%Y%m%d") fsql = f + "-" + dts + self.sql_ext if not os.path.isfile(fsql) and self.sql_ext != self.pre_ext: ftmp = f + "-" + dts + self.pre_ext if os.path.isfile(ftmp): try: os0.wlog("$ mv", ftmp, fsql) if not self.dry_run: # Rename old ext -> nex ext os.rename(ftmp, fsql) # Force change sql file extension force_change_ext = True except: pass if dtc.day != 1: if not self.remove_sql_file(fsql) \ and self.sql_ext != self.pre_ext: fsql = f + "-" + dts + self.pre_ext self.remove_sql_file(fsql) if force_change_ext: self.change_file_ext(f) fsql = f + "-????????" + self.sql_ext if dbtype == "psql": cmd = "chown " + self.psql_uu + ":" + self.psql_uu + " " + fsql elif dbtype == "mysql": cmd = "chown " + self.mysql_uu + ":" + self.mysql_uu + " " + fsql os0.trace_debug("$ ", cmd) os0.muteshell(cmd, simulate=self.dry_run) cmd = "tar --remove-files -c" + \ self.tar_opt + "f " + fzip_fn + " " + fsql os0.trace_debug("$ ", cmd) os0.muteshell(cmd, simulate=self.dry_run) if dbtype == "psql": cmd = "chown " + self.psql_uu + ":" + self.psql_uu + " " + fzip_fn elif dbtype == "mysql": cmd = "chown " + self.mysql_uu + \ ":" + self.mysql_uu + " " + fzip_fn os0.trace_debug("$ ", cmd) os0.muteshell(cmd, simulate=self.dry_run) os0.wlog(" removing archived files") fsql = f + "-????????" + self.sql_ext f_ids = sorted(glob.glob(fsql)) for fsql in f_ids: self.remove_sql_file(fsql)
def restore_db(self, dbtype, dbname, fqn): # pdb.set_trace() os0.wlog(" restoring", dbname, " ({0})".format(fqn)) homedir = os.path.expanduser("~") tar_ext = self.tar_ext tar_opt = self.tar_opt fzip_fn = dbname + tar_ext if not os.path.isfile(fzip_fn): if self.tar_ext == ".gz": tar_ext = ".bz2" tar_opt = "j" fzip_fn = dbname + tar_ext if not os.path.isfile(fzip_fn): tar_ext = self.tar_ext tar_opt = self.tar_opt # No compressed file found fzip_fn = "" elif self.tar_ext == ".bz2": tar_ext = ".gz" tar_opt = "z" fzip_fn = dbname + tar_ext if not os.path.isfile(fzip_fn): tar_ext = self.tar_ext tar_opt = self.tar_opt # No compressed file found fzip_fn = "" f = os.path.basename(fqn) # Just filename l = len(self.sql_ext) + 9 i = len(f) - l # Extract date (YYYYMMDD) from XXXXX-YYYYMMDD.SQL dts = f[i + 1:i + 9] if dbtype == "psql": cmd = "chown " + self.psql_uu + ":" + self.psql_uu + " " + fqn elif dbtype == "mysql": cmd = "chown " + self.mysql_uu + ":" + self.mysql_uu + " " + fqn os0.trace_debug("$", cmd) os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode) sql_fn = homedir + "/restdb.sql" cmd = "cp " + fqn + " " + sql_fn os0.trace_debug("$", cmd) os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode) # cmd = "sed -i -e \"s|Owner: openerp|Owner: odoo|g\""\ # " -e \"s|OWNER TO openerp|OWNER TO odoo|g\" ~/restdb.sql" # os0.trace_debug("$", cmd) # os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode) if dbtype == "psql": cmd = "chown " + self.psql_uu + ":" + self.psql_uu + " " + sql_fn elif dbtype == "mysql": cmd = "chown " + self.mysql_uu + ":" + self.mysql_uu + " " + sql_fn os0.trace_debug("$", cmd) os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode) self.repl_data(dbname, sql_fn) psh_fn = homedir + "/restdb.psh" psh_fd = open(psh_fn, "w") if dbtype == "psql": user = self.psql_uu defdb = self.psql_db psh_fd.write("\\c {0}\n".format(defdb)) psh_fd.write("DROP DATABASE IF EXISTS \"{0}-{1}\";\n".format( dbname, dts)) psh_fd.write("DROP DATABASE IF EXISTS \"{0}\";\n".format(dbname)) psh_fd.write( "CREATE DATABASE \"{0}\" TEMPLATE template1;\n".format(dbname)) psh_fd.write("\\c \"{0}\"\n".format(dbname)) psh_fd.write("\\i {0}\n".format(sql_fn)) psh_fd.write( "ALTER DATABASE \"{0}\" OWNER TO odoo;\n".format(dbname)) cmd = "psql -f " + psh_fn + " -U" + user + " " + defdb psh_fd.close() os0.trace_debug("$", cmd) os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode) elif dbtype == "mysql": user = "******" pwd = "SHS13mgr" # defdb = self.psql_db psh_fd.write( "mysqladmin -u{0} --password={1} -f drop \"{2}-{3}\" ||true\n". format(user, pwd, dbname, dts)) psh_fd.write( "mysqladmin -u{0} --password={1} -f drop \"{2}\" || true\n". format(user, pwd, dbname)) psh_fd.write( "mysqladmin -u{0} --password={1} -f create \"{2}\"\n".format( user, pwd, dbname)) psh_fd.write( "mysql -u{0} --password=SHS13mgr -G -e \"source {1}\" {2}\n". format(user, sql_fn, dbname)) psh_fd.close() cmd = "chmod +x " + psh_fn os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode) cmd = psh_fn os0.trace_debug("$", cmd) os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode) else: os0.wlog(" unknown", dbname, "database type!!!") cmd = "echo Error" # Compressed file found if fzip_fn != "": if dbtype == "psql": cmd = "chown " + self.psql_uu + \ ":" + self.psql_uu + " " + fzip_fn elif dbtype == "mysql": cmd = "chown " + self.mysql_uu + \ ":" + self.mysql_uu + " " + fzip_fn os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode) cmd = "tar --keep-newer-files -x" + tar_opt + "f " + fzip_fn os0.muteshell(cmd, simulate=self.dry_run, keepout=os0.debug_mode) if not self.dry_run: os.remove(fzip_fn) self.purge_db(dbtype, dbname)
def bck_db(self, dbtype, dblist, user, sqlcmd, ctx): # pdb.set_trace() # save_ftp_rootdir = self.ftp_rootdir p = "backups" # Make full dir path (root + sub) lpath = self.ftp_rootdir + '/' + p self.chdir(lpath) # Set directory for f in dblist: tar_ext = self.tar_ext tar_opt = self.tar_opt fzip_fn = f + tar_ext if not os.path.isfile(fzip_fn): if self.tar_ext == ".gz": tar_ext = ".bz2" tar_opt = "j" fzip_fn = f + tar_ext if not os.path.isfile(fzip_fn): tar_ext = self.tar_ext tar_opt = self.tar_opt # No compressed file found fzip_fn = "" elif self.tar_ext == ".bz2": tar_ext = ".gz" tar_opt = "z" fzip_fn = f + tar_ext if not os.path.isfile(fzip_fn): tar_ext = self.tar_ext tar_opt = self.tar_opt # No compressed file found fzip_fn = "" # Compressed file found if fzip_fn != "": cmd = "tar -x" + tar_opt + "f " + fzip_fn os0.trace_debug("$", cmd) os0.muteshell(cmd, simulate=self.dry_run) if not self.dry_run: os.remove(fzip_fn) dts = date.today().strftime("%Y%m%d") fsql = f + "-" + dts + self.sql_ext if dbtype == "psql": cmd = sqlcmd + " -U" + user + " -F p -f " + fsql + " " + f cmdlog = cmd elif dbtype == "mysql": cmd = sqlcmd + " -u " + user + \ " --password="******" " + f + \ " -r " + fsql cmdlog = sqlcmd + " -u " + user + " " + f + " -r " + fsql else: cmd = "" cmdlog = cmd os0.trace_debug("$", cmdlog) os0.muteshell(cmd, simulate=self.dry_run) if os.path.isfile(fsql): os0.wlog(" ", fsql) self.add_2_ftp(fsql, ctx) else: os0.wlog(" file", fsql, "not found!!!") self.purge_db(dbtype, f)