def rsync_project():
conf = Configs.get()
exclude_path = Path('.') / '.remote' / 'exclude.txt'
exclude_path = exclude_path.absolute()
rsync_cmd = ['rsync', '-zravuKLt', '--perms', '--executability']
if conf.private_key_file is not None:
rsync_cmd += ['-e']
rsync_cmd += [
f'"ssh -o StrictHostKeyChecking=no -i {conf.private_key_file}"'
]
if exclude_path.exists():
rsync_cmd += [f"--exclude-from='{str(exclude_path)}'"]
rsync_cmd += ['./'] # source
rsync_cmd += [f'{conf.username}@{conf.hostname}:~/{conf.name}/'
] # destination
try:
process = subprocess.run(' '.join(rsync_cmd), shell=True)
except FileNotFoundError as e:
logger.log('rsync not found', Text.danger)
print(e)
return 1
return process.returncode
def rsync(self, *, ui_mode: UIMode = UIMode.dots):
with monit.section(f"RSync {self.conf.name}"):
logger.log()
exclude_path = Configs.get().exclude_file
exclude_path = exclude_path.absolute()
# z = compress
# r = recursive
# a = equivalent to (-rlptgoD) archive (recursive/preserve everything)
# v = verbose
# u = update (skip whats newer on receiver)
# K = keep symlinks
# L = transform links to dir
# t = preserve modification times
# l = copy links
# p = preserve permissions
# g = preserve group
# o = preserve owner
# D = preserve device files
rsync_cmd = ['rsync', '-zravuKLt', '--executability']
if self.conf.private_key_file is not None:
rsync_cmd += ['-e', f'"ssh -o StrictHostKeyChecking=no -i {self.conf.private_key_file}"']
else:
rsync_cmd += ['-e', f'"ssh -o StrictHostKeyChecking=no"']
if exclude_path.exists():
rsync_cmd += [f"--exclude-from='{str(exclude_path)}'"]
rsync_cmd += ['./'] # source
rsync_cmd += [f'{self.conf.username}@{self.conf.hostname}:~/{self.project_name}/'] # destination
log_dir = self._get_log_folder(f'rsync_{self.conf.name}')
exit_code = self.local_exec.stream(' '.join(rsync_cmd),
log_dir=log_dir,
ui_mode=ui_mode)
if exit_code != 0:
raise RemoteError("Failed to run rsync")
def rsync_jobs(self, *, ui_mode: UIMode = UIMode.dots, is_silent=False):
with monit.section(f"RSync {self.conf.name} jobs",
is_silent=is_silent):
if not is_silent:
logger.log()
rsync_cmd = ['rsync', '-zravuKLt', '--executability']
if self.conf.private_key_file is not None:
rsync_cmd += [
'-e',
f'"ssh -o StrictHostKeyChecking=no -i {self.conf.private_key_file}"'
]
else:
rsync_cmd += ['-e', '"ssh -o StrictHostKeyChecking=no"']
rsync_cmd += [
f'{self.conf.username}@{self.conf.hostname}:'
f'~/{self.project_name}/{Configs.get().remote_jobs_folder_name}/'
]
rsync_cmd += [str(Configs.get().project_jobs_folder)]
log_dir = self._get_log_folder(f'rsync_jobs_{self.conf.name}')
exit_code = self.local_exec.stream(' '.join(rsync_cmd),
log_dir=log_dir,
ui_mode=ui_mode)
if exit_code != 0:
raise RemoteError("Failed to run rsync")
def __init__(
self,
*,
job_id: str,
job_key: str,
server: Server,
command: str,
env_vars: Dict[str, str],
tags: List[str],
pid: Optional[int] = None,
exit_code: Optional[int] = None,
started: bool = False,
stopped: bool = False,
):
self.tags = set(tags)
self.job_key = job_key
self.env_vars = env_vars
self.command = command
self.server = server
self.job_id = job_id
self.path = Configs.get().project_jobs_folder / self.job_id
self.pid = pid
self.exit_code = exit_code
self.started = started
self.stopped = stopped
self.out_tail_offset = -1
self.err_tail_offset = -1
if not self.path.exists():
self.path.mkdir(parents=True)
self.save()
self.update_stopped()
def __init__(self, server_id: str):
conf = Configs.get()
self.project_name = conf.name
self.scripts_folder = conf.template_scripts_folder
self.conf = conf.servers[server_id]
self.__client = None
self.__home_path = None
self.__remote_executor = None
self.__local_executor = None
def __init__(self, server_id: str):
conf = Configs.get()
self.project_name = conf.name
self.scripts_folder = conf.template_scripts_folder
self.conf = conf.servers[server_id]
self.__client: SSHClient
self.__home_path: str
self.__remote_executor: Optional[RemoteExecutor] = None
self.__local_executor: Optional[LocalExecutor] = None
def _get_log_folder(name: str):
counter = 1
while True:
folder_name = f'{name}_{counter :04d}'
log_dir = Configs.get().project_logs_folder / folder_name
if not log_dir.exists():
log_dir.mkdir(parents=True)
return log_dir
counter += 1
def connect() -> paramiko.SSHClient:
conf = Configs.get()
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
with monit.section(f'Connecting to {conf.hostname}'):
c.connect(hostname=conf.hostname,
username=conf.username,
pkey=conf.private_key,
password=conf.password)
return c
def run_command(client: SSHClient, home_path: str, command: List[str]):
conf = Configs.get()
pipfile = Path('.') / 'Pipfile'
requirements = Path('.') / 'requirements.txt'
script = template(
conf.scripts_folder / 'run.sh', {
'name': conf.name,
'home': home_path,
'use_pipenv': str(pipfile.exists()),
'run_command': ' '.join(command)
})
return run_script(client, script, 'run', home_path)
def update_packages(client: SSHClient, home_path: str):
conf = Configs.get()
pipfile = Path('.') / 'Pipfile'
requirements = Path('.') / 'requirements.txt'
script = template(
conf.scripts_folder / 'update.sh', {
'name': conf.name,
'home': home_path,
'has_pipfile': str(pipfile.exists()),
'has_requirements': str(requirements.exists())
})
return run_script(client, script, 'update', home_path)
def copy_script(self, script: str, script_name: str):
scripts_path = Configs.get().project_scripts_folder
if not scripts_path.exists():
scripts_path.mkdir()
script_file = scripts_path / script_name
with open(str(script_file), 'w') as f:
f.write(script)
os.chmod(str(script_file), stat.S_IRWXU | stat.S_IRWXG)
scp = SCPClient(self.client.get_transport())
remote_path = f'{self.remote_scripts_path}/{script_name}'
scp.put(str(script_file), remote_path)
return remote_path
def run_script(client: SSHClient, script: str, script_name: str,
home_path: str):
conf = Configs.get()
scripts_path = Path('./.remote/scripts')
if not scripts_path.exists():
scripts_path.mkdir()
script_file = scripts_path / f'{script_name}.sh'
with open(str(script_file), 'w') as f:
f.write(script)
os.chmod(str(script_file), stat.S_IRWXU | stat.S_IRWXG)
scp = SCPClient(client.get_transport())
scp.put(str(script_file),
f'{home_path}/{conf.name}/.remote-scripts/{script_name}.sh')
return execute_stream(
client, f'{home_path}/{conf.name}/.remote-scripts/{script_name}.sh')
def setup_server(client: SSHClient, home_path: str):
conf = Configs.get()
has_folder, _ = execute(client, f'test -d {conf.name}')
if has_folder != 0:
print(has_folder)
execute(client, f'mkdir {conf.name}')
has_folder, _ = execute(client, f'test -d {conf.name}/.remote-scripts')
if has_folder != 0:
execute(client, f'mkdir {conf.name}/.remote-scripts')
python_version = f'{sys.version_info.major}.{sys.version_info.minor}'
script = template(conf.scripts_folder / 'setup.sh', {
'name': conf.name,
'python_version': python_version,
'home': home_path
})
return run_script(client, script, 'setup', home_path)
def load_all(self):
if not Configs.get().project_jobs_folder.exists():
return
for p in Configs.get().project_jobs_folder.iterdir():
if p.name not in self._jobs:
self.load(p.name)
def load(self, job_id: str):
path = Configs.get().project_jobs_folder / job_id / 'job.yaml'
with open(str(path), 'r') as f:
self._jobs[job_id] = Job.from_dict(
yaml.load(f.read(), Loader=yaml.FullLoader))
self._keys[self._jobs[job_id].job_key] = self._jobs[job_id]
def load_all(self):
for server in Configs.get().servers:
if server not in self._servers:
self._servers[server] = Server(server)
def load_all(self):
for s in Configs.get().servers:
if s not in self._servers:
self._servers[s] = Server(s)