def push_cmd(target, version, username, password, email, apikey, insecure):
client = utils.get_docker_client()
config = utils.load_release_config()
image = config.get_image(target)
username, password = utils.get_credentials(
username=username,
password=password,
insecure=insecure,
apikey=apikey,
target_apikey=config.get_apikey(target))
if username:
_execute_push_login(client, image, username, password, email)
image_name = utils.format_image_name(image, version)
click.echo("Pushing {} to the registry.".format(image_name))
for data in client.push(image_name,
stream=True,
decode=True,
insecure_registry=not bool(username)):
if 'status' in data:
utils.debug_log("Logs:{} {}".format(data['status'],
data.get('progress')))
if 'error' in data:
click.echo("Error {}: {}".format(data['error'],
data['errorDetail']))
raise shub_exceptions.RemoteErrorException(
"Docker push operation failed")
click.echo("The image {} pushed successfully.".format(image_name))
def build_cmd(target, version, skip_tests):
client = utils.get_docker_client()
project_dir = utils.get_project_dir()
config = utils.load_release_config()
image = config.get_image(target)
_create_setup_py_if_not_exists()
image_name = utils.format_image_name(image, version)
if not os.path.exists(os.path.join(project_dir, 'Dockerfile')):
raise shub_exceptions.BadParameterException(
'Dockerfile is not found, please use shub-image init cmd')
is_built = False
for data in client.build(path=project_dir, tag=image_name, decode=True):
if 'stream' in data:
utils.debug_log("{}".format(data['stream'][:-1]))
is_built = re.search(
r'Successfully built ([0-9a-f]+)', data['stream'])
elif 'error' in data:
click.echo("Error {}:\n{}".format(
data['error'], data['errorDetail']))
if not is_built:
raise shub_exceptions.RemoteErrorException(
"Build image operation failed")
click.echo("The image {} build is completed.".format(image_name))
# Test the image content after building it
if not skip_tests:
test_mod.test_cmd(target, version)
def test_cmd(target, version):
config = utils.load_release_config()
image = config.get_image(target)
version = version or config.get_version()
image_name = utils.format_image_name(image, version)
docker_client = utils.get_docker_client()
for check in [
_check_image_exists, _check_start_crawl_entry,
_check_list_spiders_entry, _check_sh_entrypoint
]:
check(image_name, docker_client)
def deploy_cmd(target, version, username, password, email,
apikey, insecure, async):
config = utils.load_release_config()
project, endpoint, target_apikey = config.get_target(target)
image = config.get_image(target)
version = version or config.get_version()
image_name = utils.format_image_name(image, version)
username, password = utils.get_credentials(
username=username, password=password, insecure=insecure,
apikey=apikey, target_apikey=target_apikey)
apikey = apikey or target_apikey
params = _prepare_deploy_params(
project, version, image_name, endpoint, apikey,
username, password, email)
utils.debug_log('Deploy with params: {}'.format(params))
req = requests.post(
urljoin(endpoint, '/api/releases/deploy.json'),
data=params,
auth=(apikey, ''),
timeout=300,
allow_redirects=False
)
try:
req.raise_for_status()
except requests.exceptions.HTTPError:
_handle_deploy_errors(req)
click.echo("Deploy task results: {}".format(req))
status_url = req.headers['location']
status_id = utils.store_status_url(
status_url, limit=STORE_N_LAST_STATUS_URLS)
click.echo(
"You can check deploy results later with "
"'shub-image check --id {}'.".format(status_id))
click.echo("Deploy results:")
actual_state = _check_status_url(status_url)
click.echo(" {}".format(actual_state))
if not async:
status = actual_state['status']
while status in SYNC_DEPLOY_WAIT_STATUSES:
time.sleep(SYNC_DEPLOY_REFRESH_TIMEOUT)
actual_state = _check_status_url(status_url)
if actual_state['status'] != status:
click.echo(" {}".format(actual_state))
status = actual_state['status']
def deploy_cmd(target, version, username, password, email, apikey, insecure,
async):
config = utils.load_release_config()
project, endpoint, target_apikey = config.get_target(target)
image = config.get_image(target)
version = version or config.get_version()
image_name = utils.format_image_name(image, version)
username, password = utils.get_credentials(username=username,
password=password,
insecure=insecure,
apikey=apikey,
target_apikey=target_apikey)
apikey = apikey or target_apikey
params = _prepare_deploy_params(project, version, image_name, endpoint,
apikey, username, password, email)
utils.debug_log('Deploy with params: {}'.format(params))
req = requests.post(urljoin(endpoint, '/api/releases/deploy.json'),
data=params,
auth=(apikey, ''),
timeout=300,
allow_redirects=False)
try:
req.raise_for_status()
except requests.exceptions.HTTPError:
_handle_deploy_errors(req)
click.echo("Deploy task results: {}".format(req))
status_url = req.headers['location']
status_id = utils.store_status_url(status_url,
limit=STORE_N_LAST_STATUS_URLS)
click.echo("You can check deploy results later with "
"'shub-image check --id {}'.".format(status_id))
click.echo("Deploy results:")
actual_state = _check_status_url(status_url)
click.echo(" {}".format(actual_state))
if not async:
status = actual_state['status']
while status in SYNC_DEPLOY_WAIT_STATUSES:
time.sleep(SYNC_DEPLOY_REFRESH_TIMEOUT)
actual_state = _check_status_url(status_url)
if actual_state['status'] != status:
click.echo(" {}".format(actual_state))
status = actual_state['status']
def list_cmd_full(target, silent, version):
config = utils.load_release_config()
image = config.get_image(target)
version = version or config.get_version()
image_name = utils.format_image_name(image, version)
project, endpoint, apikey = None, None, None
try:
project, endpoint, apikey = config.get_target(target)
except shub_exceptions.BadParameterException as exc:
if 'Could not find target' not in exc.message:
raise
if not silent:
click.echo(
"Not found project for target {}, "
"not getting project settings from Dash.".format(target))
spiders = list_cmd(image_name, project, endpoint, apikey)
for spider in spiders:
click.echo(spider)
def list_cmd_full(target, silent, version):
config = utils.load_release_config()
image = config.get_image(target)
version = version or config.get_version()
image_name = utils.format_image_name(image, version)
project, endpoint, apikey = None, None, None
try:
project, endpoint, apikey = config.get_target(target)
except shub_exceptions.BadParameterException as exc:
if 'Could not find target' not in exc.message:
raise
if not silent:
click.echo(
"Not found project for target {}, "
"not getting project settings from Dash.".format(target))
spiders = list_cmd(image_name, project, endpoint, apikey)
for spider in spiders:
click.echo(spider)
def push_cmd(target, version, username, password, email, apikey, insecure):
client = utils.get_docker_client()
config = utils.load_release_config()
image = config.get_image(target)
username, password = utils.get_credentials(
username=username, password=password, insecure=insecure,
apikey=apikey, target_apikey=config.get_apikey(target))
if username:
_execute_push_login(client, image, username, password, email)
image_name = utils.format_image_name(image, version)
click.echo("Pushing {} to the registry.".format(image_name))
for data in client.push(image_name, stream=True, decode=True,
insecure_registry=not bool(username)):
if 'status' in data:
utils.debug_log("Logs:{} {}".format(data['status'],
data.get('progress')))
if 'error' in data:
click.echo("Error {}: {}".format(data['error'],
data['errorDetail']))
raise shub_exceptions.RemoteErrorException(
"Docker push operation failed")
click.echo("The image {} pushed successfully.".format(image_name))
def test_load_release_config(self):
assert isinstance(load_release_config(), ReleaseConfig)
def test_load_release_config(self):
assert isinstance(load_release_config(), ReleaseConfig)