def _wait_for_remote_mount( max_attempts: int, ctx: click.core.Context, remote_encrypted: Path, path_on_cloud_drive: str, ) -> None: """ Wait for the rclone mount or error if it does not start within some time close to 25 seconds. """ relative_path_on_cloud_drive = Path(path_on_cloud_drive).relative_to('/') remote_mount = remote_encrypted / relative_path_on_cloud_drive attempts = 0 sleep_seconds = 5 while not remote_mount.exists(): attempts += 1 if attempts > max_attempts: message = ( f'Remote mount not found after {max_attempts} attempts, ' 'exiting.') ctx.fail(message) message = f'Remote mount {remote_mount} does not exist yet, waiting.' LOGGER.info(message) time.sleep(sleep_seconds)
def download_artifact( ctx: click.core.Context, dcos_version: str, download_path: str, ) -> None: """ Download a DC/OS Open Source artifact. For DC/OS Enterprise release artifacts, contact your sales representative. """ path = Path(download_path) label = 'Downloading to ' + str(path) base_url = 'https://downloads.dcos.io/dcos/' url = base_url + dcos_version + '/dcos_generate_config.sh' head_resp = requests.head(url) if not head_resp.ok: message = 'Cannot download artifact from {url}.'.format(url=url) ctx.fail(message=message) if path.is_dir(): path = path / 'dcos_generate_config.sh' if not path.exists(): path.parent.mkdir(parents=True, exist_ok=True) # See # https://stackoverflow.com/questions/16694907/how-to-download-large-file-in-python-with-requests-py stream = requests.get(url, stream=True) assert stream.ok content_length = int(stream.headers['Content-Length']) total_written = 0 chunk_size = 1024 # See http://click.pocoo.org/6/arguments/#file-args for parameter # information with click.open_file( filename=str(path), mode='wb', atomic=True, lazy=True, ) as file_descriptor: content_iter = stream.iter_content(chunk_size=chunk_size) with click.progressbar( # type: ignore content_iter, length=content_length / chunk_size, label=label, ) as progress_bar: for chunk in progress_bar: # Filter out keep-alive new chunks. if chunk: total_written += len(chunk) file_descriptor.write(chunk) # type: ignore message = ('Downloaded {total_written} bytes. ' 'Expected {content_length} bytes.').format( total_written=total_written, content_length=content_length, ) assert total_written == content_length, message
def init(ctx: click.core.Context, site_path: str) -> None: """Create an new mackerel site""" output_path = Path(site_path) sample_site_path = Path(os.path.dirname( os.path.realpath(mackerel.__file__))) / 'site' try: shutil.copytree(src=sample_site_path, dst=output_path) except FileExistsError as e: ctx.fail(f'Initialize failed, file {e.filename} already exists') click.echo(f'Initialized empty mackerel site in {output_path}')
def init(ctx: click.core.Context, site_path: str) -> None: """Create an new mackerel site""" output_path = Path(site_path) sample_site_path = Path( os.path.dirname(os.path.realpath(mackerel.__file__))) / 'site' try: shutil.copytree(src=sample_site_path, dst=output_path) except FileExistsError as e: ctx.fail(f'Initialize failed, file {e.filename} already exists') click.echo(f'Initialized empty mackerel site in {output_path}')
def get_command(self, ctx: click.core.Context, cmd_name: str) -> Optional[click.core.Command]: rv = click.core.Group.get_command(self, ctx, cmd_name) if rv is not None: return rv matches = difflib.get_close_matches(cmd_name, self.list_commands(ctx), n=2) if not matches: return None elif len(matches) == 1: return click.core.Group.get_command(self, ctx, str(matches[0])) else: ctx.fail('Too many matches: {0}'.format(matches)) return None
def upload(ctx: click.core.Context, config: _Config) -> None: """ Upload local data to the cloud. """ upload_pid_file = Path(__file__).parent / 'upload.pid' if upload_pid_file.exists(): running_pid = upload_pid_file.read_text() if running_pid: proc_file = Path('/proc') / running_pid if proc_file.exists(): message = 'Upload script already running' LOGGER.error(msg=message) ctx.fail(message=message) current_pid = os.getpid() upload_pid_file.write_text(str(current_pid)) _sync_deletes( local_decrypted=config.local_decrypted, encfs_pass=config.encfs_pass, remote_encrypted=config.remote_encrypted, rclone_remote=config.rclone_remote, path_on_cloud_drive=config.path_on_cloud_drive, rclone_config_path=config.rclone_config_path, rclone=config.rclone, rclone_verbose=config.rclone_verbose, ) # Determine the .unionfs-fuse directory name as to not upload it exclude_name = _encode_with_encfs( path_or_file_name='.unionfs-fuse', encfs_pass=config.encfs_pass, root_dir=config.remote_encrypted, ) upload_args = [ str(config.rclone), '--config', str(config.rclone_config_path), _rclone_verbosity_flag(verbose=config.rclone_verbose), 'copy', # Try to avoid Google 403: User Rate Limit Exceeded. # This happens when 10 total transfers happen in a second. '--tpslimit', '1', # Try to avoid limit of uploading files > 100 GB. '--max-size', '100G', # Avoid retrying when the 750 GB / day upload limit is hit. '--drive-stop-on-upload-limit', # Make fewer API requests. '--fast-list', # Exclude the ``.unionfs-fuse`` directory as this is where files to be # deleted go. '--exclude', f'/{exclude_name}/*', str(config.local_encrypted), _rclone_path( rclone_remote=config.rclone_remote, rclone_root=config.path_on_cloud_drive, rclone_relative_path=None, ), ] children = config.local_encrypted.glob('*') if children: subprocess.run(args=upload_args, check=True) else: message = f'{config.local_encrypted} is empty - nothing to upload' LOGGER.info(msg=message) message = 'Upload Complete - Syncing changes' LOGGER.info(message) _local_cleanup( days_to_keep_local=config.days_to_keep_local, local_decrypted=config.local_decrypted, ) upload_pid_file.unlink()
def download_installer( ctx: click.core.Context, dcos_version: str, download_path: str, ) -> None: """ Download a DC/OS Open Source installer. For DC/OS Enterprise installers, contact your sales representative. """ path = Path(download_path) path.parent.mkdir(exist_ok=True, parents=True) path = path.parent.resolve() / path.name click.echo('Downloading to {path}.'.format(path=path)) if dcos_version.startswith('http'): url = dcos_version else: base_url = 'https://downloads.dcos.io/dcos/' url = base_url + dcos_version + '/dcos_generate_config.sh' head_resp = requests.head(url) if not head_resp.ok: message = 'Cannot download installer from {url}.'.format(url=url) ctx.fail(message=message) if path.is_dir(): path = path / 'dcos_generate_config.sh' if not path.exists(): path.parent.mkdir(parents=True, exist_ok=True) # See # https://stackoverflow.com/questions/16694907/how-to-download-large-file-in-python-with-requests-py stream = requests.get(url, stream=True) assert stream.ok content_length = int(stream.headers['Content-Length']) total_written = 0 chunk_size = 1024 # See http://click.pocoo.org/7/arguments/#file-args for parameter # information. content_iter = stream.iter_content(chunk_size=chunk_size) progress_bar = tqdm( iterable=content_iter, total=content_length / chunk_size, dynamic_ncols=True, bar_format='{l_bar}{bar}', unit_scale=None, ) with click.open_file( filename=str(path), mode='wb', atomic=True, lazy=True, ) as file_descriptor: for chunk in progress_bar: # Enable at the start of each chunk, disable at the end, to avoid # showing statistics at the end. progress_bar.disable = False # Filter out keep-alive new chunks. if chunk: total_written += len(chunk) file_descriptor.write(chunk) # type: ignore progress_bar.disable = True message = ( 'Downloaded {total_written} bytes. ' 'Expected {content_length} bytes.' ).format( total_written=total_written, content_length=content_length, ) assert total_written == content_length, message