def archive_local_dir(self, local_path: Text):
"""
Generates the archive locally and pipe it to a remote dd to write it
on disk on the other side
"""
tar = subprocess.Popen(
["tar", "-C", local_path, "-c", "-z", "."],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
dd = self.ssh_popen(
["dd", f"of={self.path}"],
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
stdin=tar.stdout,
)
_, dd_err = dd.communicate()
_, tar_err = tar.communicate()
if dd.returncode:
raise LuhError(f"Could not write remote archive: {dd_err}")
if tar.returncode:
raise LuhError(f"Could not create the archive: {tar_err}")
def extract_archive_to_dir(self, target_dir: Text) -> None:
"""
Cat the remote file and pipe it into tar
"""
parse_location(target_dir).ensure_exists_as_dir()
cat = self.ssh_popen(["cat", self.path],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
tar = subprocess.Popen(
["tar", "-C", target_dir, "-x", "-z"],
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
stdin=cat.stdout,
)
_, tar_err = tar.communicate()
_, cat_err = cat.communicate()
if cat.returncode:
raise LuhError(
f"Error while reading the remote archive: {cat_err}")
if tar.returncode:
raise LuhError(f"Error while extracting the archive: {tar_err}")
def ensure_exists_as_dir(self) -> None:
try:
Path(self.path).mkdir(parents=True, exist_ok=True)
except PermissionError:
raise LuhError(f"You don't have the permission the create {self}")
except NotADirectoryError:
raise LuhError(f"Some component of {self} is not a directory")
except OSError:
raise LuhError(f"Unknown error while creating {self}")
def get_content(self) -> Text:
try:
with open(self.path, "r", encoding="utf-8") as f:
return f.read()
except PermissionError:
raise LuhError(f"You don't have the permission to read {self}")
except FileNotFoundError:
raise LuhError(f"The file {self} does not exist")
except OSError:
raise LuhError(f"Unknown error while opening {self}")
def read_config(file_path: Text):
"""
Read configuration from JSON file (extracted from the snapshot)
"""
try:
with open(file_path, "r", encoding="utf-8") as f:
return json.load(f)
except json.JSONDecodeError:
raise LuhError("Configuration file is not valid JSON")
except OSError as e:
raise LuhError(f"Error while opening file: {e}")
def extract_php_constants(file: str):
"""
Parses a PHP file to extract all the declared constants
"""
define = re.compile(r"^\s*define\(")
lines = ["<?php"]
for line in file.splitlines(False):
if define.match(line):
lines.append(line)
lines.extend(
[
"$const = get_defined_constants(true);",
'$user = $const["user"];',
"echo json_encode($user);",
]
)
try:
data = run_php("\n".join(lines))
return json.loads(data)
except (ValueError, TypeError):
raise LuhError("Configuration file has syntax errors")
def get_content(self) -> Text:
"""
Uses a remote cat to get the content
"""
cp = self.ssh_run(["cat", self.path],
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL)
if cp.returncode == 1:
raise LuhError(f"The file {self} does not exist or you don't have "
f"permissions to read it")
elif cp.returncode:
raise LuhError(f"Unknown error while reading {self}")
return cp.stdout
def apply_wp_config(patch: Dict, wp_config: Dict, source: Location) -> Dict:
"""
Given a wp_config and a source, apply them into a patch
- Make sure that the source arg is set
- Put the DB settings in php_define
- Also set wp_config properly
"""
new_patch = deepcopy(patch)
new_patch["args"] = {"source": f"{source}"}
new_patch["wp_config"] = wp_config
if "php_define" not in new_patch:
new_patch["php_define"] = {}
try:
new_patch["php_define"].update({
"DB_HOST": wp_config["db_host"],
"DB_USER": wp_config["db_user"],
"DB_NAME": wp_config["db_name"],
"DB_PASSWORD": wp_config["db_password"],
})
except (KeyError, TypeError, AttributeError) as e:
raise LuhError(f"Generated wp_config is incorrect: {e}")
else:
return new_patch
def sync_files(remote: Location, local: Location, delete: bool = False):
"""
Use rsync to copy files from a location to another
"""
local.ensure_exists_as_dir()
args = [
"rsync",
"-rz",
"--exclude=.git",
"--exclude=.idea",
"--exclude=*.swp",
"--exclude=*.un~",
]
if delete:
args.append("--delete")
args += [remote.rsync_path(True), local.rsync_path(True)]
cp = subprocess.run(args, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
if cp.returncode:
raise LuhError(f"Error while copying files: {cp.stderr}")
def archive_local_dir(self, local_path):
cp = subprocess.run(
["tar", "-C", local_path, "-c", "-z", "-f", self.path, "."],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if cp.returncode:
raise LuhError(f"Could not create archive {self.path}")
def get_wp_config(config: Dict) -> Dict:
"""
Reads the configuration to extract the database configuration
"""
try:
return config["wp_config"]
except KeyError:
raise LuhError("Configuration is incomplete, missing wp_config")
def get_remote(config: Dict) -> Location:
"""
Reads the configuration to extract the address of the remote
"""
try:
return parse_location(config["args"]["source"])
except KeyError:
raise LuhError("Configuration is incomplete, missing args.source")
def set_wp_config_values(values: Dict[Text, Any], file_path: Text):
"""
Makes sure that the key/values set in values are set to this value in
wp-config.php.
- Existing define() are modified to get the new value
- Missing define() are added at the top of the file
Values must be JSON-encodable
"""
try:
e = re.compile(r"^\s*define\(\s*(" + PHP_STR + ")")
out = []
patched = set()
with open(file_path, "r", encoding="utf-8") as f:
for line in (x.rstrip("\r\n") for x in f.readlines()):
m = e.match(line)
if e.match(line):
key = parse_php_string(m.group(1))
else:
key = None
if key in values:
out.append(
f"define("
f"{encode_php_string(key)}, "
f"{encode_php_value(values[key])}"
f");"
)
patched.add(key)
else:
out.append(line)
missing = set(values.keys()) - patched
extra = []
if missing:
extra = ["/** Extra values created by LUH3417 */"]
for k in missing:
v = values[k]
extra.append(
f"define("
f"{encode_php_string(k)}, "
f"{encode_php_value(v)}"
f");"
)
extra.append("")
with open(file_path, "w", encoding="utf-8") as f:
f.write("\n".join(chain(out[0:1], extra, out[1:])) + "\n")
except OSError as e:
raise LuhError(f"Could not open config file at {file_path}: {e}")
def create_root_from_source(wp_config, mysql_root, source: Location) -> "LuhSql":
"""
Based on the regular DB accessor, create a root version which will be able
to run DB manipulation queries
"""
db = create_from_source(wp_config, source)
method = mysql_root.get("method")
options = mysql_root.get("options", {})
try:
if method == "socket":
db.db_name = None
db.user = options.get("mysql_user", "root")
db.sudo_user = options.get("sudo_user")
return db
else:
raise LuhError(f"Wrong MySQL root method: {method}")
except KeyError as e:
raise LuhError(f"Missing key for mysql_root: {e}")
def restore_db(db: LuhSql, dump_path: Text):
"""
Restores the specified file into DB, using the wp config and remote
location to connect the DB.
"""
try:
with open(dump_path, "r", encoding="utf-8") as f:
db.restore_dump(f)
except OSError as e:
raise LuhError(f"Could not read SQL dump: {e}")
def copy_files(remote: Location, local: Location):
"""
Copies files from the remote location to the local locations. Files are
serialized and pipelined through tar, maybe locally, maybe through SSH
depending on the locations.
"""
remote_args = _build_args(
remote, ["tar", "--warning=no-file-changed", "-C", remote.path, "-c", "."]
)
local_args_1 = _build_args(local, ["mkdir", "-p", local.path])
local_args_2 = _build_args(local, ["tar", "-C", local.path, "-x"])
cp = subprocess.run(local_args_1, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
if cp.returncode:
raise LuhError(f'Error while creating target dir "{local}": {cp.stderr}')
remote_p = subprocess.Popen(
remote_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
local_p = subprocess.Popen(
local_args_2,
stdin=remote_p.stdout,
stderr=subprocess.PIPE,
stdout=subprocess.DEVNULL,
)
remote_p.wait()
local_p.wait()
if remote_p.returncode:
err = remote_p.stderr.read(1000)
if err == b"":
return
raise LuhError(f'Error while reading files from "{remote}": {err}')
if local_p.returncode:
raise LuhError(f'Error writing files to "{local}": {local_p.stderr.read(1000)}')
def set_content(self, content) -> None:
"""
Creates or overrides the file so it receives the provided content.
The parent directory must exist and the location must not be a
directory.
"""
out, err, ret = self.run_script(
f"echo -n {quote(content)} > {quote(self.path)}")
if ret:
raise LuhError(f"Cannot set content: {err}")
def chown(self, owner: Text) -> None:
"""
A simple local chown
"""
cp = subprocess.run(
["chown", "-R", owner, self.path],
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
)
if cp.returncode:
raise LuhError(f"Failed to chown: {cp.stderr[:1000]}")
def parse_php_string(string: Text) -> Text:
"""
Parses a PHP string literal (including quotes) and returns the equivalent
Python string (as a string, not a literal)
"""
parser = f"<?php echo json_encode({string});"
try:
data = run_php(parser)
return json.loads(data)
except (ValueError, TypeError):
raise LuhError("Invalid PHP string")
def encode_php_value(val):
"""
Transforms a JSON-encodable value into a PHP literal
"""
data = encode_php_string(json.dumps(val))
parser = f"<?php var_export(json_decode({data}, true));"
try:
data = run_php(parser)
assert data is not None
return data
except AssertionError:
raise LuhError(f"Could not encode to PHP value: {repr(val)}")
def ensure_exists_as_dir(self) -> None:
"""
Runs mkdir -p remotely
"""
cp = self.ssh_run(
["mkdir", "-p", self.path],
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
)
if cp.returncode:
raise LuhError(
f"Could not create {self} as a directory: {cp.stderr}")
def extract_archive_to_dir(self, target_dir: Text) -> None:
"""
Plain old local archive extraction
"""
parse_location(target_dir).ensure_exists_as_dir()
tar = subprocess.run(
["tar", "-C", target_dir, "-x", "-z", "-f", self.path],
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
)
if tar.returncode:
raise LuhError(f"Error while extracting the archive: {tar.stderr}")
def restore_dump(self, fp: TextIO):
"""
Restores a dump into the DB, reading the dump from an input TextIO
(which can be the stdout of another process or simply an open file, by
example).
"""
p = subprocess.Popen(
self.args("mysql"), stderr=PIPE, stdout=DEVNULL, stdin=fp, encoding="utf-8"
)
_, err = p.communicate()
if p.returncode:
raise LuhError(f"Could not import MySQL DB: {err}")
def ssh_popen(self, args, *p_args, **kwargs) -> Popen:
"""
Opens a process through SSH. It's the same arguments as Popen() except
that the SSH command will be prepended to the args.
"""
new_args = SshManager.instance(self.user, self.host).get_args(args)
cp = subprocess.Popen(new_args, *p_args, **kwargs)
if cp.returncode == 255:
raise LuhError(
f"SSH connection to {self.ssh_target} could not be established"
)
return cp
def patch_sql_dump(source_path: Text, dest_path: Text, replace: ReplaceMap):
"""
Patches the SQL dump found at source_path into a new SQL dump found in
dest_path. It will use the replace map to replace values.
Values are replaced in a holistic way so that PHP serialized values are
not broken and escaped character are detected as such. This is by far not
perfect but seems sufficient for most use cases.
"""
try:
with open(source_path, "rb") as i, open(dest_path, "wb") as o:
for line in i:
o.write(walk(line, replace))
except OSError as e:
raise LuhError(f"Could not open SQL dump: {e}")
def run_query(self, query: Text):
"""
Runs a single SQL query
"""
p = subprocess.Popen(
self.args("mysql"),
stderr=PIPE,
stdout=DEVNULL,
stdin=PIPE,
encoding="utf-8",
)
_, err = p.communicate(query)
if p.returncode:
raise LuhError(f"Could not run MySQL query: {err}")
def dump_to_file(self, file_path: Text):
"""
Dumps the database into the specified file
"""
with open(file_path, "w", encoding="utf-8") as f:
p = subprocess.Popen(
self.args("mysqldump", ["--hex-blob"]),
stderr=PIPE,
stdout=f,
stdin=DEVNULL,
encoding="utf-8",
)
_, err = p.communicate()
if p.returncode:
raise LuhError(f"Could not dump MySQL DB: {err}")
def parse_wp_config(location: "Location", config_file_name: Text = "wp-config.php"):
"""
Parses the WordPress configuration to get the DB configuration
"""
config_location = location.child(config_file_name)
config = config_location.get_content()
const = extract_php_constants(config)
try:
return {
"db_host": const["DB_HOST"],
"db_user": const["DB_USER"],
"db_password": const["DB_PASSWORD"],
"db_name": const["DB_NAME"],
}
except KeyError as e:
LuhError(f"Missing config value: {e}")
def ssh_run(self, args, *p_args, **kwargs) -> CompletedProcess:
"""
Runs a process remotely using subprocess.run(). This will enforce
an UTF-8 encoding for stdin/out. Otherwise it's the same argument
as run() and the SSH command is automatically appended to the args.
"""
kwargs = dict(kwargs, encoding="utf-8")
new_args = SshManager.instance(self.user, self.host).get_args(args)
cp = subprocess.run(new_args, *p_args, **kwargs)
if cp.returncode == 255:
raise LuhError(
f"SSH connection to {self.ssh_target} could not be established"
)
return cp
def set_git_repo(self, repo: Text, version: Text):
"""
Sets the current location to be a git repo at the given version. Any
pre-existing file or directory at this location will be overridden.
"""
location = self.path
if location and location[-1] == "/":
location = location[0:-1]
out, err, ret = self.run_script("""
git clone -b {version} {repo} {location}__ \\
&& mv {location} {location}___ \\
&& mv {location}__ {location} \\
&& rm -fr {location}___
""".format(repo=quote(repo),
version=quote(version),
location=quote(location)))
if ret:
raise LuhError(f"Could not clone repo: {err}")
