def get(self, path, target_path): """target_path must be the directory where to put copied files or directories """ try: if not self.local: with fabric.api.settings( host_string=self.host, # destination host key_filename=self.ssh_key, # a path to ssh key timeout=2, # a network connection timeout warn_only=True, # don't exit on error abort_on_prompts=True, # non-interactive mode ): logger.debug("Getting remote file: %s %s", path, target_path) utils.execute('mkdir -p "{0}"'.format(target_path)) try: return fabric.api.get(path, target_path) except SystemExit: logger.error("Fabric aborted this iteration") else: logger.debug("Getting local file: cp -r %s %s", path, target_path) utils.execute('mkdir -p "{0}"'.format(target_path)) return utils.execute('cp -r "{0}" "{1}"'.format(path, target_path)) except Exception as e: logger.error("Error occured: %s", str(e))
def snapshot(self): if not os.path.exists(self.target_path): utils.execute('mkdir -p "{0}"'.format(os.path.dirname( self.target_path))) with open(self.target_path, "w") as f: f.write("Host {0} was offline/unreachable during " "logs obtaining.\n".format(self.host))
def get(self, path, target_path): """Get remote or local file target_path must be the directory where to put copied files or directories """ try: if self.dest_host: with fabric.api.settings( host_string=self.dest_host, # destination host key_filename=self.ssh_key, # a path to ssh key timeout=2, # connection timeout warn_only=True, # don't exit on error abort_on_prompts=True, # non-interactive mode ): logger.debug("Getting remote file: %s %s", path, target_path) utils.execute('mkdir -p "{0}"'.format(target_path)) try: return fabric.api.get(path, target_path) except SystemExit: logger.error("Fabric aborted this iteration") else: logger.debug( "Getting local file: cp -r %s %s", path, target_path) utils.execute('mkdir -p "{0}"'.format(target_path)) return utils.execute( 'cp -r "{0}" "{1}"'.format(path, target_path)) except fabric.exceptions.NetworkError as e: logger.error("NetworkError occured: %s", str(e)) raise except Exception as e: logger.error("Unexpected error occured: %s", str(e))
def snapshot(self): """Example: self.conf.target IS /target self.host IS host.domain.tld self.path IS /var/log/somedir (it can be /var/log/somedir*) self.target_path IS /target/host.domain.tld/var/log 1. we get remote directory host.domain.tld:/var/log/somedir 2. we put it into /target/host.domain.tld/var/log 3. we walk through /target/host.domain.tld/var/log 4. we check fnmatch(/var/log/*, /var/log/somedir) """ # 1. # 2. super(Subs, self).snapshot() # 3. walk = os.walk(self.target_path) for root, _, files in walk: for filename in files: # /target/host.domain.tld/var/log/somedir/1/2 fullfilename = os.path.join(root, filename) # 4. # /target/host.domain.tld tgt_host = os.path.join(self.conf.target, self.host) # var/log/somedir/1/2 rel_tgt_host = os.path.relpath(fullfilename, tgt_host) # /var/log/somedir/1/2 match_orig_path = os.path.join("/", rel_tgt_host) if not fnmatch.fnmatch(match_orig_path, self.path): continue tempfilename = execute("mktemp")[1].strip() self.sed(fullfilename, tempfilename) execute("mv -f %s %s" % (tempfilename, fullfilename))
def snapshot(self): """Example: self.conf.target IS /target self.host IS host.domain.tld self.path IS /var/log/somedir (it can be /var/log/somedir*) self.target_path IS /target/host.domain.tld/var/log 1. we get remote directory host.domain.tld:/var/log/somedir 2. we put it into /target/host.domain.tld/var/log 3. we walk through /target/host.domain.tld/var/log 4. we check fnmatch(/var/log/*, /var/log/somedir) """ # 1. # 2. super(Subs, self).snapshot() # 3. walk = os.walk(self.target_path) for root, _, files in walk: for filename in files: # /target/host.domain.tld/var/log/somedir/1/2 fullfilename = os.path.join(root, filename) # 4. # /target/host.domain.tld tgt_host = os.path.join(self.conf.target, self.host) # var/log/somedir/1/2 rel_tgt_host = os.path.relpath(fullfilename, tgt_host) # /var/log/somedir/1/2 match_orig_path = os.path.join("/", rel_tgt_host) if not fnmatch.fnmatch(match_orig_path, self.path): continue tempfilename = execute("mktemp")[1].strip() self.sed(fullfilename, tempfilename) execute('mv -f "{0}" "{1}"'.format(tempfilename, fullfilename))
def get(self, path, target_path): if not self.local: with fabric.api.settings(host_string=self.host): logger.debug("Getting remote file: %s %s", path, target_path) return fabric.api.get(path, target_path) else: logger.debug("Getting local file: cp -r %s %s", path, target_path) execute("mkdir -p %s" % os.path.dirname(target_path)) return execute("cp -r %s %s" % (path, target_path))
def snapshot(self): out = self.command(self.cmdname) execute("mkdir -p {0}".format(os.path.dirname(self.target_path))) with open(self.target_path, "w") as f: f.write("===== COMMAND =====: {0}\n".format(self.cmdname)) f.write("===== RETURN CODE =====: {0}\n".format(out.return_code)) f.write("===== STDOUT =====:\n") f.write(out.stdout) f.write("\n===== STDERR =====:\n") f.write(out.stderr)
def snapshot(self): super(Subs, self).snapshot() walk = os.walk(self.target_path) if not os.path.isdir(self.target_path): walk = (("/", [], [self.target_path]),) for root, _, files in walk: for filename in files: fullfilename = os.path.join(root, filename) tempfilename = self.command("mktemp").stdout.strip() self.sed(fullfilename, tempfilename) execute("mv %s %s" % (tempfilename, fullfilename))
def snapshot(self): execute('mkdir -p "{0}"'.format(os.path.dirname(self.target_path))) server = xmlrpclib.Server(self.server) with open(self.target_path, "w") as f: for method in self.methods: if hasattr(server, method): response = getattr(server, method)() response = pprint.pformat(response, indent=2) else: response = "no such method on remote server" f.write("===== {0} =====\n{1}\n\n".format(method, response))
def _snapshot_single(self, cmd): out = self.command(cmd) utils.execute('mkdir -p "{0}"'.format(os.path.dirname( self.target_path))) with open(self.target_path, "a") as f: f.write("===== COMMAND =====: {0}\n".format(cmd)) f.write("===== RETURN CODE =====: {0}\n".format(out.return_code)) f.write("===== STDOUT =====:\n") if out.stdout: f.write(out.stdout) f.write("\n===== STDERR =====:\n") if out.stderr: f.write(out.stderr)
def snapshot(self): logger.debug("Making snapshot") utils.execute("rm -rf {0}".format(os.path.dirname(self.conf.target))) for obj_data in self.conf.objects: logger.debug("Dumping: %s", obj_data) driver = Driver.getDriver(obj_data, self.conf) driver.snapshot() logger.debug("Archiving dump directory: %s", self.conf.target) utils.compress(self.conf.target, self.conf.compression_level) with open(self.conf.lastdump, "w") as fo: fo.write("{0}.tar.xz".format(self.conf.target)) return "{0}.tar.xz".format(self.conf.target)
def snapshot(self): logger.debug("Making snapshot") for obj_data in self.conf.objects: logger.debug("Dumping: %s", obj_data) driver = Driver.getDriver(obj_data, self.conf) driver.snapshot() logger.debug("Archiving dump directory: %s", self.conf.target) execute("tar zcf {0}.tgz -C {1} {2}" "".format(self.conf.target, os.path.dirname(self.conf.target), os.path.basename(self.conf.target))) execute("rm -r {0}".format(self.conf.target)) with open(self.conf.lastdump, "w") as fo: fo.write("{0}.tgz".format(self.conf.target)) return "{0}.tgz".format(self.conf.target)
def snapshot(self): logger.debug("Making snapshot") for obj_data in self.conf.objects: logger.debug("Dumping: %s", obj_data) driver = Driver.getDriver(obj_data, self.conf) driver.snapshot() logger.debug("Archiving dump directory: %s", self.conf.target) execute("tar zcf {0}.tgz -C {1} {2}" "".format(self.conf.target, os.path.dirname(self.conf.target), os.path.basename(self.conf.target))) execute("rm -r {0}".format(self.conf.target)) with open(self.conf.lastdump, "w") as fo: fo.write("%s.tgz" % self.conf.target) return "%s.tgz" % self.conf.target
def sed(self, from_filename, to_filename, gz=False): sedscript = tempfile.NamedTemporaryFile() logger.debug("Sed script: %s", sedscript.name) for orig, new in self.subs.iteritems(): logger.debug("Sed script: s/%s/%s/g", orig, new) sedscript.write("s/{0}/{1}/g\n".format(orig, new)) sedscript.flush() command = " | ".join(filter(lambda x: x != "", [ "cat {0}".format(from_filename), self.decompress(from_filename), "sed -f {0}".format(sedscript.name), self.compress(from_filename), ])) execute(command, to_filename=to_filename) sedscript.close()
def get(self, path, target_path): try: if not self.local: with fabric.api.settings(host_string=self.host, timeout=2, warn_only=True): logger.debug("Getting remote file: %s %s", path, target_path) return fabric.api.get(path, target_path) else: logger.debug("Getting local file: cp -r %s %s", path, target_path) execute("mkdir -p %s" % os.path.dirname(target_path)) return execute("cp -r %s %s" % (path, target_path)) except Exception as e: logger.error("Error occured: %s", str(e))
def sed(self, from_filename, to_filename, gz=False): sedscript = tempfile.NamedTemporaryFile() logger.debug("Sed script: %s", sedscript.name) for orig, new in self.subs.iteritems(): logger.debug("Sed script: s/%s/%s/g", orig, new) sedscript.write("s/%s/%s/g\n" % (orig, new)) sedscript.flush() command = " | ".join(filter(lambda x: x != "", [ "cat %s" % from_filename, self.decompress(from_filename), "sed -f %s" % sedscript.name, self.compress(from_filename), ])) execute(command, to_filename=to_filename) sedscript.close()
def snapshot(self): logger.debug("Making snapshot") utils.execute("rm -rf {0}".format(os.path.dirname(self.conf.target))) for obj_data in self.conf.objects: logger.debug("Dumping: %s", obj_data) self.action_single(obj_data, action='snapshot') logger.debug("Dumping shotgun log and archiving dump directory: %s", self.conf.target) self.action_single(self.conf.self_log_object, action='snapshot') utils.compress(self.conf.target, self.conf.compression_level) with open(self.conf.lastdump, "w") as fo: fo.write("{0}.tar.xz".format(self.conf.target)) return "{0}.tar.xz".format(self.conf.target)
def command(self, command): out = CommandOut() try: if not self.local: with fabric.api.settings( host_string=self.host, # destination host key_filename=self.ssh_key, # a path to ssh key timeout=2, # a network connection timeout command_timeout=10, # a command execution timeout warn_only=True, # don't exit on error ): logger.debug( "Running remote command: " "host: %s command: %s", self.host, command) output = fabric.api.run(command, pty=True) out.stdout = output out.return_code = output.return_code out.stderr = output.stderr else: logger.debug("Running local command: %s", command) out.return_code, out.stdout, out.stderr = execute(command) logger.debug("Stderr: %s", out.stderr) except Exception as e: logger.error("Error occured: %s", str(e)) return out
def command(self, command): out = CommandOut() raw_stdout = utils.CCStringIO(writers=sys.stdout) try: if not self.local: with fabric.api.settings( host_string=self.host, # destination host key_filename=self.ssh_key, # a path to ssh key timeout=2, # a network connection timeout command_timeout=10, # a command execution timeout warn_only=True, # don't exit on error abort_on_prompts=True, # non-interactive mode ): logger.debug("Running remote command: " "host: %s command: %s", self.host, command) try: output = fabric.api.run(command, stdout=raw_stdout) except SystemExit: logger.error("Fabric aborted this iteration") # NOTE(prmtl): because of pty=True (default) and # combine_stderr=True (default) stderr is combined # with stdout out.stdout = raw_stdout.getvalue() out.return_code = output.return_code else: logger.debug("Running local command: %s", command) out.return_code, out.stdout, out.stderr = utils.execute( command) except Exception as e: logger.error("Error occured: %s", str(e)) out.stdout = raw_stdout.getvalue() return out
def command(self, command): out = CommandOut() raw_stdout = utils.CCStringIO(writers=sys.stdout) try: if not self.local: with fabric.api.settings( host_string=self.host, # destination host key_filename=self.ssh_key, # a path to ssh key timeout=2, # a network connection timeout command_timeout=10, # a command execution timeout warn_only=True, # don't exit on error abort_on_prompts=True, # non-interactive mode ): logger.debug( "Running remote command: " "host: %s command: %s", self.host, command) try: output = fabric.api.run(command, stdout=raw_stdout) except SystemExit: logger.error("Fabric aborted this iteration") # NOTE(prmtl): because of pty=True (default) and # combine_stderr=True (default) stderr is combined # with stdout out.stdout = raw_stdout.getvalue() out.return_code = output.return_code else: logger.debug("Running local command: %s", command) out.return_code, out.stdout, out.stderr = utils.execute( command) except Exception as e: logger.error("Error occured: %s", str(e)) out.stdout = raw_stdout.getvalue() return out
def get(self, path, target_path): """target_path must be the directory where to put copied files or directories """ try: if not self.local: with fabric.api.settings(host_string=self.host, timeout=2, warn_only=True): logger.debug("Getting remote file: %s %s", path, target_path) execute("mkdir -p %s" % target_path) return fabric.api.get(path, target_path) else: logger.debug("Getting local file: cp -r %s %s", path, target_path) execute("mkdir -p %s" % target_path) return execute("cp -r %s %s" % (path, target_path)) except Exception as e: logger.error("Error occured: %s", str(e))
class Postgres(Driver): def __init__(self, data, conf): super(Postgres, self).__init__(data, conf) self.dbhost = self.data.get("dbhost", "localhost") self.dbname = self.data["dbname"] self.username = self.data.get("username", "postgres") self.password = self.data.get("password") self.target_path = str( os.path.join(self.conf.target, self.host, "pg_dump")) def snapshot(self): if self.password: authline = "{host}:{port}:{dbname}:{username}:{password}".format( host=self.dbhost, port="5432", dbname=self.dbname, username=self.username, password=self.password) home_dir = pwd.getpwuid(os.getuid()).pw_dir pgpass = os.path.join(home_dir, ".pgpass") with open(pgpass, "a+") as fo: fo.seek(0) auth = False for line in fo: if re.search(ur"^{0}$".format(authline), line): auth = True break if not auth: fo.seek(0, 2) fo.write("{0}\n".format(authline)) os.chmod(pgpass, stat.S_IRUSR + stat.S_IWUSR) temp = self.command("mktemp").stdout.strip() self.command("pg_dump -h {dbhost} -U {username} -w " "-f {file} {dbname}".format(dbhost=self.dbhost, username=self.username, file=temp, dbname=self.dbname)) execute('mkdir -p "{0}"'.format(self.target_path)) dump_basename = "{0}_{1}.sql".format(self.dbhost, self.dbname) execute('mv -f "{0}" "{1}"'.format( temp, os.path.join(self.target_path, dump_basename)))
class Postgres(Driver): def __init__(self, data, conf): super(Postgres, self).__init__(data, conf) self.dbhost = self.data.get("dbhost", "localhost") self.dbname = self.data["dbname"] self.username = self.data.get("username", "postgres") self.password = self.data.get("password") self.target_path = str( os.path.join(self.conf.target, self.host, "pg_dump")) def snapshot(self): if self.password: authline = "{host}:{port}:{dbname}:{username}:{password}".format( host=self.host, port="5432", dbname=self.dbname, username=self.username, password=self.password) with open(os.path.expanduser("~/.pgpass"), "a+") as fo: fo.seek(0) auth = False for line in fo: if re.search(ur"^%s$" % authline, line): auth = True break if not auth: fo.seek(0, 2) fo.write("{0}\n".format(authline)) os.chmod(os.path.expanduser("~/.pgpass"), stat.S_IRUSR + stat.S_IWUSR) temp = self.command("mktemp").stdout.strip() self.command("pg_dump -h {dbhost} -U {username} -w " "-f {file} {dbname}".format(dbhost=self.dbhost, username=self.username, file=temp, dbname=self.dbname)) execute("mkdir -p %s" % self.target_path) dump_basename = "%s_%s.sql" % (self.dbhost, self.dbname) execute("mv -f %s %s" % (temp, os.path.join(self.target_path, dump_basename)))
def get(self, path, target_path): """target_path must be the directory where to put copied files or directories """ try: if not self.local: with fabric.api.settings( host_string=self.host, # destination host key_filename=self.ssh_key, # a path to ssh key timeout=2, # a network connection timeout warn_only=True, # don't exit on error ): logger.debug("Getting remote file: %s %s", path, target_path) execute('mkdir -p "{0}"'.format(target_path)) return fabric.api.get(path, target_path) else: logger.debug("Getting local file: cp -r %s %s", path, target_path) execute('mkdir -p "{0}"'.format(target_path)) return execute('cp -r "{0}" "{1}"'.format(path, target_path)) except Exception as e: logger.error("Error occured: %s", str(e))
def command(self, command): out = CommandOut() if not self.local: with fabric.api.settings(host_string=self.host): logger.debug("Running remote command: " "host: %s command: %s", self.host, command) output = fabric.api.run(command, pty=True) out.stdout = output out.return_code = output.return_code out.stderr = output.stderr else: logger.debug("Running local command: %s", command) out.return_code, out.stdout, out.stderr = execute(command) return out
def command(self, command): out = CommandOut() try: if not self.local: with fabric.api.settings(host_string=self.host, timeout=2, warn_only=True): logger.debug("Running remote command: " "host: %s command: %s", self.host, command) output = fabric.api.run(command, pty=True) out.stdout = output out.return_code = output.return_code out.stderr = output.stderr else: logger.debug("Running local command: %s", command) out.return_code, out.stdout, out.stderr = execute(command) logger.debug("Stderr: %s", out.stderr) except Exception as e: logger.error("Error occured: %s", str(e)) return out
def command(self, command): out = CommandOut() try: if not self.local: with fabric.api.settings(host_string=self.host, timeout=2, warn_only=True): logger.debug( "Running remote command: " "host: %s command: %s", self.host, command) output = fabric.api.run(command, pty=True) out.stdout = output out.return_code = output.return_code out.stderr = output.stderr else: logger.debug("Running local command: %s", command) out.return_code, out.stdout, out.stderr = execute(command) logger.debug("Stderr: %s", out.stderr) except Exception as e: logger.error("Error occured: %s", str(e)) return out
def get(self, path, target_path): """Get remote or local file target_path must be the directory where to put copied files or directories """ try: if not os.path.exists(target_path): os.makedirs(target_path) if self.dest_host: with fabric.api.settings( host_string=self.dest_host, # destination host key_filename=self.ssh_key, # a path to ssh key timeout=2, # connection timeout warn_only=True, # don't exit on error abort_on_prompts=True, # non-interactive mode ): logger.debug("Getting remote file: %s %s", path, target_path) try: return fabric.api.get(path, target_path) except SystemExit: logger.error("Fabric aborted this iteration") else: # NOTE(mkwiek): We need to use shell ln instead of os.symlink # because of wildcards used in shotgun settings. ln utility # will nicely handle wildcards and create all needed symlinks # in target_path directory symlink_command = 'ln -s "{}" "{}"'.format(path, target_path) logger.debug( "Symlinking to local file: {}".format(symlink_command)) return utils.execute(symlink_command) except fabric.exceptions.NetworkError as e: logger.error("NetworkError occured: %s", str(e)) raise except Exception as e: logger.error("Unexpected error occured: %s", str(e))
def command(self, command): out = CommandOut() try: if not self.local: with fabric.api.settings( host_string=self.host, # destination host key_filename=self.ssh_key, # a path to ssh key timeout=2, # a network connection timeout command_timeout=10, # a command execution timeout warn_only=True, # don't exit on error ): logger.debug("Running remote command: " "host: %s command: %s", self.host, command) output = fabric.api.run(command, pty=True) out.stdout = output out.return_code = output.return_code out.stderr = output.stderr else: logger.debug("Running local command: %s", command) out.return_code, out.stdout, out.stderr = execute(command) logger.debug("Stderr: %s", out.stderr) except Exception as e: logger.error("Error occured: %s", str(e)) return out