def build_cmd(target, version, skip_tests): client = utils.get_docker_client() project_dir = utils.get_project_dir() config = load_shub_config() image = config.get_image(target) _create_setup_py_if_not_exists() image_name = utils.format_image_name(image, version) if not os.path.exists(os.path.join(project_dir, 'Dockerfile')): raise shub_exceptions.BadParameterException( "Dockerfile is not found, please use shub image 'init' command") is_built = False for data in client.build(path=project_dir, tag=image_name, decode=True): if 'stream' in data: utils.debug_log("{}".format(data['stream'][:-1])) is_built = re.search(r'Successfully built ([0-9a-f]+)', data['stream']) elif 'error' in data: click.echo("Error {}:\n{}".format(data['error'], data['errorDetail'])) if not is_built: raise shub_exceptions.RemoteErrorException( "Build image operation failed") click.echo("The image {} build is completed.".format(image_name)) # Test the image content after building it if not skip_tests: test_cmd(target, version)
def push_cmd(target, version, username, password, email, apikey, insecure, skip_tests): # Test the image content after building it if not skip_tests: test_cmd(target, version) client = utils.get_docker_client() config = load_shub_config() image = config.get_image(target) username, password = utils.get_credentials( username=username, password=password, insecure=insecure, apikey=apikey, target_apikey=config.get_apikey(target)) if username: _execute_push_login(client, image, username, password, email) image_name = utils.format_image_name(image, version) click.echo("Pushing {} to the registry.".format(image_name)) events = client.push(image_name, stream=True, decode=True, insecure_registry=not bool(username)) if utils.is_verbose(): push_progress_cls = _LoggedPushProgress else: push_progress_cls = _PushProgress push_progress = push_progress_cls(events) push_progress.show() click.echo("The image {} pushed successfully.".format(image_name))
def build_cmd(target, version, skip_tests): config = load_shub_config() create_scrapinghub_yml_wizard(config, target=target, image=True) client = utils.get_docker_client() project_dir = utils.get_project_dir() image = config.get_image(target) image_name = utils.format_image_name(image, version) if not os.path.exists(os.path.join(project_dir, 'Dockerfile')): raise shub_exceptions.NotFoundException( "Dockerfile is not found and it is required because project '{}' is configured " "to deploy Docker images. Please add a Dockerfile that will be used to build " "the image and retry this command. If you want to migrate an existing Scrapy project " "you can use `shub image init` command to create a Dockerfile.". format(target)) if utils.is_verbose(): build_progress_cls = _LoggedBuildProgress else: build_progress_cls = _BuildProgress click.echo("Building {}.".format(image_name)) events = client.build(path=project_dir, tag=image_name, decode=True) build_progress = build_progress_cls(events) build_progress.show() click.echo("The image {} build is completed.".format(image_name)) # Test the image content after building it if not skip_tests: test_cmd(target, version)
def build_cmd(target, version, skip_tests, no_cache, filename='Dockerfile'): config = load_shub_config() create_scrapinghub_yml_wizard(config, target=target, image=True) client = utils.get_docker_client() project_dir = utils.get_project_dir() image = config.get_image(target) image_name = utils.format_image_name(image, version) if not os.path.exists(os.path.join(project_dir, filename)): raise shub_exceptions.NotFoundException( "Dockerfile is not found and it is required because project '{}' is configured " "to deploy Docker images. Please add a Dockerfile that will be used to build " "the image and retry this command. If you want to migrate an existing Scrapy project " "you can use `shub image init` command to create a Dockerfile.".format(target)) if utils.is_verbose(): build_progress_cls = _LoggedBuildProgress else: build_progress_cls = _BuildProgress click.echo("Building {}.".format(image_name)) events = client.build( path=project_dir, tag=image_name, decode=True, dockerfile=filename, nocache=no_cache ) build_progress = build_progress_cls(events) build_progress.show() click.echo("The image {} build is completed.".format(image_name)) # Test the image content after building it if not skip_tests: test_cmd(target, version)
def push_cmd(target, version, username, password, email, apikey, insecure, skip_tests): # Test the image content after building it if not skip_tests: test_cmd(target, version) client = utils.get_docker_client() config = load_shub_config() image = config.get_image(target) username, password = utils.get_credentials( username=username, password=password, insecure=insecure, apikey=apikey, target_apikey=config.get_apikey(target)) if username: _execute_push_login(client, image, username, password, email) image_name = utils.format_image_name(image, version) click.echo("Pushing {} to the registry.".format(image_name)) events = client.push(image_name, stream=True, decode=True, insecure_registry=not bool(username)) if utils.is_verbose(): push_progress_cls = _LoggedPushProgress else: push_progress_cls = _PushProgress push_progress = push_progress_cls(events) push_progress.show() click.echo("The image {} pushed successfully.".format(image_name))
def test_cmd(target, version): config = load_shub_config() image = config.get_image(target) version = version or config.get_version() image_name = utils.format_image_name(image, version) docker_client = utils.get_docker_client() for check in [_check_image_size, _check_start_crawl_entry, _check_shub_image_info_entry]: check(image_name, docker_client)
def list_cmd_full(target, silent, version): config = load_shub_config() image = config.get_image(target) version = version or config.get_version() image_name = utils.format_image_name(image, version) target_conf = config.get_target_conf(target) metadata = list_cmd(image_name, target_conf.project_id, target_conf.endpoint, target_conf.apikey) for spider in metadata.get('spiders', []): click.echo(spider)
def test_cmd(target, version): config = load_shub_config() image = config.get_image(target) version = version or config.get_version() image_name = utils.format_image_name(image, version) docker_client = utils.get_docker_client() for check in [ _check_image_size, _check_start_crawl_entry, _check_shub_image_info_entry ]: check(image_name, docker_client)
def list_cmd_full(target, silent, version): config = load_shub_config() image = config.get_image(target) version = version or config.get_version() image_name = utils.format_image_name(image, version) target_conf = config.get_target_conf(target) metadata = list_cmd(image_name, target_conf.project_id, target_conf.endpoint, target_conf.apikey) for spider in metadata.get('spiders', []): click.echo(spider)
def run_cmd(spider, args, settings, environment, version, keep_volume): try: target, spider = spider.rsplit('/', 1) except ValueError: target = 'default' config = load_shub_config() image = config.get_image(target) version = version or config.get_version() image_name = utils.format_image_name(image, version) docker_client = utils.get_docker_client() env = _format_environment(spider, args, settings, environment) _run_with_docker(docker_client, image_name, env, keep_volume)
def deploy_cmd(target, version, username, password, email, apikey, insecure, async): config = load_shub_config() target_conf = config.get_target_conf(target) endpoint, target_apikey = target_conf.endpoint, target_conf.apikey image = config.get_image(target) version = version or config.get_version() image_name = utils.format_image_name(image, version) username, password = utils.get_credentials(username=username, password=password, insecure=insecure, apikey=apikey, target_apikey=target_apikey) apikey = apikey or target_apikey params = _prepare_deploy_params(target_conf.project_id, version, image_name, endpoint, apikey, username, password, email) utils.debug_log('Deploy with params: {}'.format(params)) req = requests.post(urljoin(endpoint, '/api/releases/deploy.json'), data=params, auth=(apikey, ''), timeout=300, allow_redirects=False) try: req.raise_for_status() except requests.exceptions.HTTPError: _handle_deploy_errors(req) click.echo("Deploy task results: {}".format(req)) status_url = req.headers['location'] status_id = utils.store_status_url(status_url, limit=STORE_N_LAST_STATUS_URLS) click.echo("You can check deploy results later with " "'shub image check --id {}'.".format(status_id)) click.echo("Deploy results:") actual_state = _check_status_url(status_url) click.echo(" {}".format(actual_state)) if not async: status = actual_state['status'] while status in SYNC_DEPLOY_WAIT_STATUSES: time.sleep(SYNC_DEPLOY_REFRESH_TIMEOUT) actual_state = _check_status_url(status_url) if actual_state['status'] != status: click.echo(" {}".format(actual_state)) status = actual_state['status']
def run_cmd(spider, args, settings, environment, version, keep_volume): try: target, spider = spider.rsplit('/', 1) except ValueError: target = 'default' config = load_shub_config() image = config.get_image(target) version = version or config.get_version() image_name = utils.format_image_name(image, version) docker_client = utils.get_docker_client() env = _format_environment(spider, args, settings, environment) _run_with_docker(docker_client, image_name, env, keep_volume)
def deploy_cmd(target, version, username, password, email, apikey, insecure, async_): config = load_shub_config() target_conf = config.get_target_conf(target) endpoint, target_apikey = target_conf.endpoint, target_conf.apikey image = config.get_image(target) version = version or config.get_version() image_name = utils.format_image_name(image, version) username, password = utils.get_credentials( username=username, password=password, insecure=insecure, apikey=apikey, target_apikey=target_apikey) apikey = apikey or target_apikey params = _prepare_deploy_params( target_conf.project_id, version, image_name, endpoint, apikey, username, password, email) click.echo("Deploying {}".format(image_name)) utils.debug_log('Deploy parameters: {}'.format(params)) req = requests.post( urljoin(endpoint, '/api/releases/deploy.json'), data=params, auth=(apikey, ''), timeout=300, allow_redirects=False ) if req.status_code == 400: reason = req.json().get('non_field_errors') raise ShubException('\n'.join(reason) if reason else req.text) req.raise_for_status() status_url = req.headers['location'] status_id = utils.store_status_url( status_url, limit=STORE_N_LAST_STATUS_URLS) click.echo( "You can check deploy results later with " "'shub image check --id {}'.".format(status_id)) if async_: return if utils.is_verbose(): deploy_progress_cls = _LoggedDeployProgress else: deploy_progress_cls = _DeployProgress events = _convert_status_requests_to_events(status_url) deploy_progress = deploy_progress_cls(events) deploy_progress.show()
def deploy_cmd(target, version, username, password, email, apikey, insecure, async_): config = load_shub_config() target_conf = config.get_target_conf(target) endpoint, target_apikey = target_conf.endpoint, target_conf.apikey image = config.get_image(target) version = version or config.get_version() image_name = utils.format_image_name(image, version) username, password = utils.get_credentials( username=username, password=password, insecure=insecure, apikey=apikey, target_apikey=target_apikey) apikey = apikey or target_apikey params = _prepare_deploy_params( target_conf.project_id, version, image_name, endpoint, apikey, username, password, email) click.echo("Deploying {}".format(image_name)) utils.debug_log('Deploy parameters: {}'.format(params)) req = requests.post( urljoin(endpoint, '/api/releases/deploy.json'), data=params, auth=(apikey, ''), timeout=300, allow_redirects=False ) if req.status_code == 400: reason = req.json().get('non_field_errors') raise ShubException('\n'.join(reason) if reason else req.text) req.raise_for_status() status_url = req.headers['location'] status_id = utils.store_status_url( status_url, limit=STORE_N_LAST_STATUS_URLS) click.echo( "You can check deploy results later with " "'shub image check --id {}'.".format(status_id)) if async_: return if utils.is_verbose(): deploy_progress_cls = _LoggedDeployProgress else: deploy_progress_cls = _DeployProgress events = _convert_status_requests_to_events(status_url) deploy_progress = deploy_progress_cls(events) deploy_progress.show()
def test_format_image_name(self): assert format_image_name('simple', 'tag') == 'simple:tag' assert format_image_name('user/simple', 'tag') == 'user/simple:tag' assert format_image_name('registry/user/simple', 'tag') == \ 'registry/user/simple:tag' assert format_image_name('registry:port/user/simple', 'tag') == \ 'registry:port/user/simple:tag' assert format_image_name('registry:port/user/simple:test', 'tag') == \ 'registry:port/user/simple:tag' with mock.patch('shub.config.load_shub_config') as mocked: config = mock.Mock() config.get_version.return_value = 'test-version' mocked.return_value = config assert format_image_name('test', None) == 'test:test-version'