def run_app(self):
"""Runs desired binary and returns the time elapsed during execution."""
cmd = self.gen_command()
start_time = time.time()
run_command(cmd)
end_time = time.time()
return end_time - start_time
def s3_cp(self,
src,
dst,
exclude=None,
include=None,
recursive=True,
run_silently=False):
"""Copies files in src to dst according to given exclude and include filters
Args:
src (str): source path
dst (str): destination path
exclude (str): exclude filter
include (str): include filter
"""
try:
filters = ""
if exclude:
filters += f" --exclude '{exclude}'"
if include:
filters += f" --include '{include}'"
if recursive:
filters += f" --recursive"
run_command(f"aws s3 cp {src} {dst} {filters}", run_silently)
except subprocess.CalledProcessError:
raise Exception(f"Failed to cp {src} to {dst}!")
def upload(src, dst, filters=None, run_silently=False):
"""Recursively uploads the objects from source that adhere to the filters. No
operation is performed for local paths.
Args:
src (str): Path to the source directory.
dst (str): Path to the destination directory where uploaded files will be saved.
filters (list[str], optional): List of filters to choose files to upload.
run_silently (bool, optional): Whether or not to display results to stdout.
Returns:
bool: Success of upload.
"""
filters = filters if filters is not None else []
remote = Address(dst)
if remote.protocol == "s3":
print("Uploading to S3...")
# Slight difference between downloading a single file and a folder
if "." in os.path.basename(remote.path):
upload_cmd = f"aws s3 cp {src} {dst}"
else:
filter_cmd = "--exclude '*' "
filter_cmd += " ".join(
[f"--include '{filter}'" for filter in filters])
upload_cmd = f"aws s3 sync {src} {dst} {filter_cmd}"
run_command(upload_cmd, run_silently)
return True
return False
def main(argv):
"""Creates a kops cluster, deploys a kubernetes cluster atop it, and runs render
with the kubernetes nodes as workers. The cluster remain upon completion and must
be externally terminated (re: clean.py).
Args:
argv (list[str]): List of arguments (used interally by abseil).
"""
aws_util = AWSUtil(FLAGS.csv_path, region_name=FLAGS.region)
key_fn = create_key(aws_util)
instance_id, ip_staging = create_instance(aws_util, key_fn)
ec2_file = os.path.expanduser(FLAGS.ec2_file)
with open(ec2_file, "w") as f:
f.write(instance_id)
configure_remote_shell(aws_util, key_fn, ip_staging)
master_ip = run_ssh_command(
key_fn,
ip_staging,
"""aws ec2 describe-instances \
--instance-ids $(ec2metadata --instance-id) \
--query 'Reservations[*].Instances[*].PublicIpAddress' \
--output text""",
hide=True,
)
ecr_registry_name = f"fb360-{aws_util.username}"
repo_uri = get_repo_uri(key_fn, ip_staging, ecr_registry_name)
if not repo_uri.strip():
run_ssh_command(
key_fn,
ip_staging,
f"aws ecr create-repository --repository-name {ecr_registry_name}",
)
repo_uri = get_repo_uri(key_fn, ip_staging, ecr_registry_name)
render_pid = get_render_pid(key_fn, ip_staging)
if render_pid is None:
setup_instance(aws_util, key_fn, ip_staging, master_ip, repo_uri)
run_render(key_fn, ip_staging, master_ip)
render_pid = get_render_pid(key_fn, ip_staging)
sync_logs = f"""while true; do \
rsync -avz -e 'ssh -i {key_fn}' \
ubuntu@{ip_staging}:/home/ubuntu/logs/ {config.DOCKER_INPUT_ROOT}/logs/; \
sleep 10; \
done &"""
run_command(sync_logs, run_async=True)
display_render_progress = f"""tail --pid {render_pid} -n +1 -f render.out"""
run_ssh_command(key_fn, ip_staging, display_render_progress)
def on_process_finished(self, exitCode, exitStatus, p_id):
"""Callback event handler for a process completing.
Args:
exitCode (int): Return code from running the process.
exitStatus (str): Description message of process exit.
p_id (str): PID of completed process.
"""
if self.parent.is_aws:
remote_rigs = os.path.join(self.parent.ui_flags.project_root,
"rigs/")
run_command(f"aws s3 sync {self.path_rigs} {remote_rigs}")
common.on_process_finished(self, p_id)
def setup_local_gpu():
# Check if we are using Linux and we have an NVIDIA card, and we are not rendering in AWS
if not FLAGS.project_root.startswith("s3://"):
host_os = get_os_type(config.LOCALHOST)
if host_os == OSType.LINUX and pyvidia.get_nvidia_device() is not None:
gpu_script = os.path.join(dir_scripts, "render", "setup_gpu.sh")
print(glog.green("Setting up GPU environment..."))
run_command(f"/bin/bash {gpu_script}",
run_silently=not FLAGS.verbose)
else:
print(
glog.yellow(
"We can only access an Nvidia GPU from a Linux host. Skipping Docker GPU setup"
))
def get_sample_file(src, run_silently=False):
"""Gets the name of a single objects in the source directory. Local paths are
also supported.
Args:
src (str): Path to the source directory.
run_silently (bool, optional): Whether or not to display results to stdout.
Returns:
str: Single filename from the directory. If no such file or directory exists,
None is returned instead.
"""
remote = Address(src)
if remote.protocol == "s3":
if not src.endswith("/"):
src = src + "/"
raw_output = run_command(f"aws s3 ls {src}", run_silently)
raw_lines = raw_output.split("\n")
if len(raw_lines) > 0:
return raw_lines[0].split(" ")[-1]
return None
else:
for root, _, files in os.walk(src):
for file in files:
if not file.startswith("."):
return os.path.join(root, file)
return None
def run_app(self, binary_name, args=None, log_file=None):
"""Runs desired binary with arguments pulled from the corresponding flagfile if
None are passed in.
Args:
binary_name (str): Name of the binary.
args (str, optional): CLI arguments. If None is passed in, this is pulled from
the corresponding flagfile from res/test/.
stream (bool, optional): Whether or not to stream output to the screen.
"""
if args is None:
cmd = self.gen_command_flagfile(binary_name)
else:
cmd = self.gen_command(binary_name, args, self.io_args.output_root,
self.io_args.log_dir)
run_command(cmd, file_fn=log_file)
def s3_sync(self, src, dst, exclude=None, include=None, run_silently=False):
"""Syncs files in src to dst according to given exclude and include filters
Args:
src (str): source path
dst (str): destination path
exclude (str): exclude filter
include (list[str]): include filters
"""
try:
filters = ""
if exclude:
filters += f" --exclude '{exclude}'"
if include:
for incl in include:
filters += f" --include '{incl}'"
run_command(f"aws s3 sync {src} {dst} {filters}", run_silently)
except subprocess.CalledProcessError:
raise Exception(f"Failed to sync {src} to {dst}!")
def _run_bin(msg):
"""Runs the binary associated with the message. The execution assumes the worker is
running in a configured Docker container.
Args:
msg (dict[str, str]): Message received from RabbitMQ publisher.
"""
msg_cp = copy(msg)
# The binary flag convention includes the "last" frame
msg_cp["last"] = get_frame_name(int(msg["last"]))
app_name = msg_cp["app"].split(":")[0]
relevant_flags = [flag["name"] for flag in bin_to_flags[app_name]]
cmd_flags = " ".join([
f"--{flag}={msg_cp[flag]}" for flag in relevant_flags
if flag in msg_cp and msg_cp[flag] != ""
])
# Order is determined to prevent substrings from being accidentally replaced
input_root = msg_cp["input_root"].rstrip("/")
output_root = msg_cp["output_root"].rstrip("/")
root_order = ([output_root, input_root]
if input_root in output_root else [input_root, output_root])
root_to_docker = {
input_root: config.DOCKER_INPUT_ROOT,
output_root: config.DOCKER_OUTPUT_ROOT,
}
for root in root_order:
if not os.path.exists(root):
cmd_flags = cmd_flags.replace(root, root_to_docker[root])
bin_path = os.path.join(config.DOCKER_BUILD_ROOT, "bin", app_name + ".exe")
cmd = f"set GLOG_alsologtostderr=1 && set GLOG_stderrthreshold=0 && {bin_path} {cmd_flags}"
run_command(cmd)
def s3_ls(self, s3_url, run_silently=False):
"""Lists the contents of the given S3 path
Args:
s3_url (str): S3 URL
"""
if not s3_url.endswith("/"):
s3_url = f"{s3_url}/"
try:
raw_output = run_command(f"aws s3 ls {s3_url}", run_silently)
except subprocess.CalledProcessError:
if not run_silently:
print(f"Failed to list: {s3_url}!")
return []
raw_lines = raw_output.split("\n")
return [raw_line.split(" ")[-1].strip().rstrip("/") for raw_line in raw_lines]
def configure_shell(self, run_silently=False):
"""Configures local shell with the AWS credentials and region."""
cmd = "aws configure set"
run_command(
f"{cmd} aws_access_key_id {self.aws_access_key_id}",
run_silently=run_silently,
)
run_command(
f"{cmd} aws_secret_access_key {self.aws_secret_access_key}",
run_silently=run_silently,
)
if self.region_name:
run_command(f"{cmd} default.region {self.region_name}",
run_silently=run_silently)
def copy_frame(src,
dst,
frame,
cameras,
run_silently=False,
uncompressed=False):
"""Copies a single frame to a new directory. A standard cp operation is
performed for local paths.
Args:
src (str): Path to the source directory.
dst (str): Path to the destination directory.
frames (list[str]): Frame names being copied.
run_silently (bool, optional): Whether or not to display results to stdout.
uncompressed (bool, optional): Whether or not the src should be uncompressed before
copying to the destination. Only relevant for archived intermediates.
"""
remote = Address(src)
if remote.protocol == "s3":
frame_tar_fn = f"{frame}.tar"
src_frame = os.path.join(src, frame_tar_fn)
if uncompressed:
tmp_extract_dir = "tmp"
run_command(f"aws s3 cp {src_frame} .", run_silently)
os.makedirs(tmp_extract_dir, exist_ok=True)
tar_ref = tarfile.open(frame_tar_fn)
tar_ref.extractall(tmp_extract_dir)
tar_ref.close()
run_command(f"aws s3 cp {tmp_extract_dir} {dst} --recursive",
run_silently)
rmtree(tmp_extract_dir)
os.remove(frame_tar_fn)
else:
dst_frame = os.path.join(dst, frame_tar_fn)
run_command(f"aws s3 cp {src_frame} {dst_frame}", run_silently)
else:
for camera in cameras:
src_cam = os.path.join(src, camera)
dst_cam = os.path.join(dst, camera)
os.makedirs(dst_cam, exist_ok=True)
# Copy all extensions for current frame
for f in glob.iglob(f"{src_cam}/{frame}.*", recursive=False):
f = os.path.basename(f)
copyfile(os.path.join(src_cam, f), os.path.join(dst_cam, f))
def setup_master(base_params):
"""Sets up the master node for rendering.
Args:
base_params (dict[str, _]): Map of all the FLAGS defined in render.py.
"""
protocol = Address(base_params["input_root"]).protocol
try:
if protocol == "s3":
run_command("sudo service rabbitmq-server start")
else:
run_command("service rabbitmq-server start")
except Exception:
runtime = "nvidia" if which("nvidia-docker") else ""
cmd = f"""docker run --runtime={runtime} -p 5672:5672 -p 15672:15672 \
-d {config.DOCKER_IMAGE}:latest rabbitmq-server start"""
run_command(cmd)
def listdir(src, run_silently=False, recursive=True):
"""Lists objects in a source directory. A standard ls is performed for local paths.
Args:
src (str): Path to the source directory.
run_silently (bool, optional): Whether or not to display results to stdout.
Returns:
set[str]: Set of contained filenames.
"""
remote = Address(src)
if remote.protocol == "s3":
if not src.endswith("/"):
src = src + "/"
try:
cmd = f"aws s3 ls {src}"
if recursive:
cmd = cmd + "--recursive"
raw_output = run_command(cmd, run_silently)
except Exception:
return set(
) # An exception is raised if no such src directory is found
raw_lines = raw_output.split("\n")
files = {raw_line.split(" ")[-1] for raw_line in raw_lines}
return set(filter(None, files)) # Remove empty results
else:
result = set()
if recursive:
for root, _, files in os.walk(src):
for file in files:
if not file.startswith("."):
result.add(
os.path.join(os.path.relpath(root, src), file))
else:
result = set(os.listdir(src))
return result
def run_ui(client, docker_img):
"""Starts the UI.
Args:
client (DockerClient): Docker client configured to the host environment.
docker_img (str): Name of the Docker image.
"""
if not FLAGS.verbose:
print(glog.green("Initializing container"), end="")
loading_context = RepeatedTimer(1,
lambda: print(glog.green("."), end=""))
host_os = get_os_type(config.LOCALHOST)
# Setup steps for X11 forwarding vary slightly per the host operating system
volumes = {
"/var/run/docker.sock": {
"bind": "/var/run/docker.sock",
"mode": "ro"
}
}
if host_os == OSType.MAC or host_os == OSType.LINUX:
volumes.update(
{"/tmp/.X11-unix": {
"bind": "/tmp/.X11-unix",
"mode": "ro"
}})
if host_os == OSType.MAC or host_os == OSType.LINUX:
run_command(f"xhost + {config.LOCALHOST}",
run_silently=not FLAGS.verbose)
if host_os == OSType.LINUX:
run_command(f"xhost + {config.DOCKER_LOCAL_HOSTNAME}",
run_silently=not FLAGS.verbose)
host_to_docker_path = {FLAGS.project_root: config.DOCKER_INPUT_ROOT}
project = Project(
FLAGS.project_root,
FLAGS.cache,
FLAGS.csv_path,
FLAGS.s3_sample_frame,
FLAGS.s3_ignore_fullsize_color,
FLAGS.verbose,
)
project.verify()
cmds = [
"cd scripts/ui",
f"""python3 -u dep.py \
--host_os={get_os_type(config.LOCALHOST)} \
--local_bin={FLAGS.local_bin} \
--master={FLAGS.master} \
--password={FLAGS.password} \
--project_root={FLAGS.project_root} \
--s3_ignore_fullsize_color={FLAGS.s3_ignore_fullsize_color} \
--s3_sample_frame={FLAGS.s3_sample_frame} \
--username={FLAGS.username} \
--verbose={FLAGS.verbose}""",
]
docker_networks = client.networks.list()
network_names = [docker_network.name for docker_network in docker_networks]
if config.DOCKER_NETWORK not in network_names:
client.networks.create(config.DOCKER_NETWORK, driver="bridge")
project_address = Address(FLAGS.project_root)
project_protocol = project_address.protocol
if project_protocol == "smb":
mounts = docker_mounts(FLAGS.project_root, host_to_docker_path,
FLAGS.username, FLAGS.password)
cmds = [f"mkdir {config.DOCKER_INPUT_ROOT}"] + mounts + cmds
local_project_root = None
elif project_protocol == "s3":
glog.check_ne(FLAGS.csv_path, "",
"csv_path cannot be empty if rendering on AWS")
aws_util = AWSUtil(FLAGS.csv_path, s3_url=FLAGS.project_root)
glog.check(
aws_util.s3_bucket_is_valid(FLAGS.project_root),
f"Invalid S3 project path: {FLAGS.project_root}",
)
volumes.update({
FLAGS.csv_path: {
"bind": config.DOCKER_AWS_CREDENTIALS,
"mode": "rw"
}
})
project_name = project_address.path
cache_path = os.path.join(os.path.expanduser(FLAGS.cache),
project_name)
os.makedirs(cache_path, exist_ok=True)
volumes.update(
{cache_path: {
"bind": config.DOCKER_INPUT_ROOT,
"mode": "rw"
}})
local_project_root = cache_path
else:
glog.check(
os.path.isdir(FLAGS.project_root),
f"Invalid project path: {FLAGS.project_root}",
)
volumes.update({
host_path: {
"bind": docker_path,
"mode": "rw"
}
for host_path, docker_path in host_to_docker_path.items()
})
local_project_root = FLAGS.project_root
ipc_dir = os.path.join(local_project_root, "ipc")
os.makedirs(ipc_dir, exist_ok=True)
volumes.update({ipc_dir: {"bind": config.DOCKER_IPC_ROOT, "mode": "rw"}})
cmd = f'/bin/bash -c "{" && ".join(cmds)}"'
global container_name
display = ":0" if host_os == OSType.LINUX else "host.docker.internal:0"
runtime = "nvidia" if which("nvidia-docker") else ""
if host_os != OSType.LINUX:
display = "host.docker.internal:0"
if not FLAGS.verbose:
loading_context.stop()
print("")
try:
container = client.containers.run(
docker_img,
command=cmd,
detach=True,
environment={"DISPLAY": display},
runtime=runtime,
network=config.DOCKER_NETWORK,
ports={
config.RABBITMQ_PORT: config.RABBITMQ_PORT,
config.RABBITMQ_MANAGE_PORT: config.RABBITMQ_MANAGE_PORT,
},
privileged=True,
volumes=volumes,
stderr=True,
)
except docker.errors.APIError as e:
if "port is already allocated" in str(e):
raise Exception(
"Failed to launch UI! Ensure: \n"
"(1) No other instance of UI is running (check: docker ps) and\n"
"(2) RabbitMQ is not running on your machine (check: ps aux | grep 'rabbitmq')"
) from None
raise e
container_name = container.name
create_viewer_watchdog(client, ipc_dir, local_project_root)
def on_modified(self, event):
"""When a viewer file is modified from the UI, the appropriate viewer runs on the host.
Args:
event (watchdog.FileSystemEvent): Watchdog event for when viewer file has been modified.
"""
if isinstance(event, DirModifiedEvent):
return
ipc_name = os.path.basename(event.src_path)
host_os = get_os_type(config.LOCALHOST)
if ipc_name == config.DOCKER_RIFT_VIEWER_IPC and host_os != OSType.WINDOWS:
print(glog.yellow("RiftViewer is only supported on Windows!"))
return
app_name = config.get_app_name(ipc_name)
if not app_name:
print(glog.red(f"Invalid IPC name: {ipc_name}"))
return
try:
output_dir = posixpath.join(self.local_project_root,
config.OUTPUT_ROOT_NAME)
if ipc_name == config.DOCKER_RIFT_VIEWER_IPC:
fused_dir = posixpath.join(output_dir,
image_type_paths["fused"])
fused_json = self.get_fused_json(fused_dir)
if not fused_json:
print(
glog.red(f"Cannot find fused rig json in {fused_dir}"))
return
cmd_flags = f"""--rig={posixpath.join(fused_dir, fused_json)} \
--catalog={posixpath.join(fused_dir, "fused.json")} \
--strip_files={posixpath.join(fused_dir, "fused_0.bin")}"""
elif ipc_name in [
config.DOCKER_SMR_IPC, config.DOCKER_SMR_ONSCREEN_IPC
]:
flags_render = self.get_render_flags("export")
if ipc_name == config.DOCKER_SMR_IPC:
flags_render["output"] = posixpath.join(
output_dir, image_type_paths["exports"])
flags_smr = [flag["name"] for flag in bin_to_flags[app_name]]
cmd_flags = ""
ignore_onscreen = ["format", "output"]
for flag in flags_render:
if flag in flags_smr:
if (flag in ignore_onscreen and ipc_name
== config.DOCKER_SMR_ONSCREEN_IPC):
continue
cmd_flags += f" --{flag}={flags_render[flag]}"
cmd_flags = cmd_flags.replace(config.DOCKER_INPUT_ROOT,
self.local_project_root)
cmd = f"{posixpath.join(FLAGS.local_bin, app_name)} {cmd_flags}"
if os.name != "nt": # GLOG initiatives don't work in Powershell/cmd
cmd = f"GLOG_alsologtostderr=1 GLOG_stderrthreshold=0 {cmd}"
run_command(cmd)
except Exception as e:
print(glog.red(e))
