def _get_parallels_endpoints(self):
"""
:rtype: tuple[str]
"""
client = HttpClient(self.args, always=True)
display.info('Getting available endpoints...', verbosity=1)
sleep = 3
for _ in range(1, 10):
response = client.get(
'https://s3.amazonaws.com/ansible-ci-files/ansible-test/parallels-endpoints.txt'
)
if response.status_code == 200:
endpoints = tuple(response.response.splitlines())
display.info(
'Available endpoints (%d):\n%s' %
(len(endpoints), '\n'.join(' - %s' % endpoint
for endpoint in endpoints)),
verbosity=1)
return endpoints
display.warning(
'HTTP %d error getting endpoints, trying again in %d seconds.'
% (response.status_code, sleep))
time.sleep(sleep)
raise ApplicationError('Unable to get available endpoints.')
def main():
"""Main program function."""
try:
args = parse_args()
display.verbosity = args.verbosity
display.color = args.color
try:
run_id = os.environ['SHIPPABLE_BUILD_ID']
except KeyError as ex:
raise MissingEnvironmentVariable(ex.args[0])
client = HttpClient(args)
response = client.get('https://api.shippable.com/jobs?runIds=%s' %
run_id)
jobs = response.json()
if len(jobs) == 1:
raise ApplicationError(
'Shippable run %s has only one job. Did you use the "Rebuild with SSH" option?'
% run_id)
except ApplicationWarning as ex:
display.warning(str(ex))
exit(0)
except ApplicationError as ex:
display.error(str(ex))
exit(1)
except KeyboardInterrupt:
exit(2)
except IOError as ex:
if ex.errno == errno.EPIPE:
exit(3)
raise
def _setup_dynamic(self):
"""Request Azure credentials through Sherlock."""
display.info('Provisioning %s cloud environment.' % self.platform,
verbosity=1)
config = self._read_config_template()
response = {}
if os.path.isfile(self.SHERLOCK_CONFIG_PATH):
with open(self.SHERLOCK_CONFIG_PATH, 'r') as sherlock_fd:
sherlock_uri = sherlock_fd.readline().strip() + '&rgcount=2'
parts = urlparse(sherlock_uri)
query_string = parse_qs(parts.query)
base_uri = urlunparse(parts[:4] + ('', ''))
if 'code' not in query_string:
example_uri = 'https://example.azurewebsites.net/api/sandbox-provisioning'
raise ApplicationError(
'The Sherlock URI must include the API key in the query string. Example: %s?code=xxx'
% example_uri)
display.info('Initializing azure/sherlock from: %s' % base_uri,
verbosity=1)
http = HttpClient(self.args)
result = http.get(sherlock_uri)
display.info('Started azure/sherlock from: %s' % base_uri,
verbosity=1)
if not self.args.explain:
response = result.json()
else:
aci = self._create_ansible_core_ci()
aci_result = aci.start()
if not self.args.explain:
response = aci_result['azure']
self.aci = aci
if not self.args.explain:
values = dict(
AZURE_CLIENT_ID=response['clientId'],
AZURE_SECRET=response['clientSecret'],
AZURE_SUBSCRIPTION_ID=response['subscriptionId'],
AZURE_TENANT=response['tenantId'],
RESOURCE_GROUP=response['resourceGroupNames'][0],
RESOURCE_GROUP_SECONDARY=response['resourceGroupNames'][1],
)
display.sensitive.add(values['AZURE_SECRET'])
config = '\n'.join('%s: %s' % (key, values[key])
for key in sorted(values))
config = '[default]\n' + config
self._write_config(config)
def ping_tower_api(self):
"""Wait for Tower API to become available."""
display.info('Waiting for the Tower API to become reachable')
config = TowerConfig.parse(self.config_path)
http = HttpClient(self.args, insecure=True)
http.username = config.username
http.password = config.password
uri = 'https://%s/api/v1/ping/' % config.host
attempts = 60
while attempts:
attempts -= 1
response = http.get(uri)
if response.status_code == 200:
return
time.sleep(5)
raise ApplicationError(
'Timed out waiting for Tower API to become reachable.')
def get_merge_runs(self, project_id, branch):
"""
:type project_id: str
:type branch: str
:rtype: list[dict]
"""
params = dict(
isPullRequest='false',
projectIds=project_id,
branch=branch,
)
client = HttpClient(self.args, always=True)
response = client.get('https://api.shippable.com/runs?%s' % urlencode(params))
return response.json()
def _get_credentials(self):
"""Wait for the CloudStack simulator to return credentials.
:rtype: dict[str, str]
"""
client = HttpClient(self.args, always=True)
endpoint = '%s/admin.json' % self.endpoint
for _ in range(1, 30):
display.info('Waiting for CloudStack credentials: %s' % endpoint, verbosity=1)
response = client.get(endpoint)
if response.status_code == 200:
return response.json()
time.sleep(30)
raise ApplicationError('Timeout waiting for CloudStack credentials.')
def _wait_for_service(self):
"""Wait for the CloudStack service endpoint to accept connections."""
if self.args.explain:
return
client = HttpClient(self.args, always=True)
endpoint = self.endpoint
for _ in range(1, 30):
display.info('Waiting for CloudStack service: %s' % endpoint, verbosity=1)
try:
client.get(endpoint)
return
except SubprocessError:
pass
time.sleep(30)
raise ApplicationError('Timeout waiting for CloudStack service.')
def _wait_for_service(self):
"""Wait for the VCenter service endpoint to accept connections."""
if self.args.explain:
return
client = HttpClient(self.args, always=True, insecure=self.insecure)
endpoint = 'https://%s:%s' % (self.endpoint, self.port)
for i in range(1, 30):
display.info('Waiting for VCenter service: %s' % endpoint, verbosity=1)
try:
client.get(endpoint)
return
except SubprocessError:
pass
time.sleep(10)
raise ApplicationError('Timeout waiting for VCenter service.')
def _wait_for_service(self, protocol, acme_host, port, local_part, name):
"""Wait for an endpoint to accept connections."""
if self.args.explain:
return
client = HttpClient(self.args, always=True, insecure=True)
endpoint = '%s://%s:%d/%s' % (protocol, acme_host, port, local_part)
for dummy in range(1, 30):
display.info('Waiting for %s: %s' % (name, endpoint), verbosity=1)
try:
client.get(endpoint)
return
except SubprocessError:
pass
time.sleep(1)
raise ApplicationError('Timeout waiting for %s.' % name)
def _wait_for_service(self, endpoint):
"""Wait for the OpenShift service endpoint to accept connections.
:type endpoint: str
"""
if self.args.explain:
return
client = HttpClient(self.args, always=True, insecure=True)
for dummy in range(1, 30):
display.info('Waiting for OpenShift service: %s' % endpoint, verbosity=1)
try:
client.get(endpoint)
return
except SubprocessError:
pass
time.sleep(10)
raise ApplicationError('Timeout waiting for OpenShift service.')
def __init__(self,
args,
platform,
version,
stage='prod',
persist=True,
name=None):
"""
:type args: EnvironmentConfig
:type platform: str
:type version: str
:type stage: str
:type persist: bool
:type name: str
"""
self.args = args
self.platform = platform
self.version = version
self.stage = stage
self.client = HttpClient(args)
self.connection = None
self.instance_id = None
self.endpoint = None
self.max_threshold = 1
self.name = name if name else '%s-%s' % (self.platform, self.version)
self.ci_key = os.path.expanduser('~/.ansible-core-ci.key')
aws_platforms = (
'aws',
'azure',
'windows',
'freebsd',
'rhel',
'vyos',
'junos',
'ios',
)
osx_platforms = ('osx', )
if self.platform in aws_platforms:
if args.remote_aws_region:
# permit command-line override of region selection
region = args.remote_aws_region
# use a dedicated CI key when overriding the region selection
self.ci_key += '.%s' % args.remote_aws_region
elif is_shippable():
# split Shippable jobs across multiple regions to maximize use of launch credits
if self.platform == 'windows':
region = 'us-east-2'
else:
region = 'us-east-1'
else:
# send all non-Shippable jobs to us-east-1 to reduce api key maintenance
region = 'us-east-1'
self.endpoints = AWS_ENDPOINTS[region],
if self.platform == 'windows':
self.ssh_key = None
self.port = 5986
else:
self.ssh_key = SshKey(args)
self.port = 22
elif self.platform in osx_platforms:
self.endpoints = self._get_parallels_endpoints()
self.max_threshold = 6
self.ssh_key = SshKey(args)
self.port = None
else:
raise ApplicationError('Unsupported platform: %s' % platform)
self.path = os.path.expanduser('~/.ansible/test/instances/%s-%s' %
(self.name, self.stage))
if persist and self._load():
try:
display.info('Checking existing %s/%s instance %s.' %
(self.platform, self.version, self.instance_id),
verbosity=1)
self.connection = self.get(always_raise_on=[404])
display.info('Loaded existing %s/%s from: %s' %
(self.platform, self.version, self._uri),
verbosity=1)
except HttpError as ex:
if ex.status != 404:
raise
self._clear()
display.info('Cleared stale %s/%s instance %s.' %
(self.platform, self.version, self.instance_id),
verbosity=1)
self.instance_id = None
self.endpoint = None
else:
self.instance_id = None
self.endpoint = None
self._clear()
if self.instance_id:
self.started = True
else:
self.started = False
self.instance_id = str(uuid.uuid4())
self.endpoint = None
def __init__(self,
args,
platform,
version,
stage='prod',
persist=True,
load=True,
name=None,
provider=None):
"""
:type args: EnvironmentConfig
:type platform: str
:type version: str
:type stage: str
:type persist: bool
:type load: bool
:type name: str
"""
self.args = args
self.platform = platform
self.version = version
self.stage = stage
self.client = HttpClient(args)
self.connection = None
self.instance_id = None
self.endpoint = None
self.max_threshold = 1
self.name = name if name else '%s-%s' % (self.platform, self.version)
self.ci_key = os.path.expanduser('~/.ansible-core-ci.key')
self.resource = 'jobs'
# Assign each supported platform to one provider.
# This is used to determine the provider from the platform when no provider is specified.
providers = dict(
aws=(
'aws',
'windows',
'freebsd',
'vyos',
'junos',
'ios',
'tower',
'rhel',
),
azure=('azure', ),
parallels=('osx', ),
)
if provider:
# override default provider selection (not all combinations are valid)
self.provider = provider
else:
for candidate in providers:
if platform in providers[candidate]:
# assign default provider based on platform
self.provider = candidate
break
for candidate in providers:
if '%s/%s' % (platform, version) in providers[candidate]:
# assign default provider based on platform and version
self.provider = candidate
break
self.path = os.path.expanduser('~/.ansible/test/instances/%s-%s-%s' %
(self.name, self.provider, self.stage))
if self.provider in ('aws', 'azure'):
if self.provider != 'aws':
self.resource = self.provider
if args.remote_aws_region:
# permit command-line override of region selection
region = args.remote_aws_region
# use a dedicated CI key when overriding the region selection
self.ci_key += '.%s' % args.remote_aws_region
elif is_shippable():
# split Shippable jobs across multiple regions to maximize use of launch credits
if self.platform == 'windows':
region = 'us-east-2'
else:
region = 'us-east-1'
else:
# send all non-Shippable jobs to us-east-1 to reduce api key maintenance
region = 'us-east-1'
self.path = "%s-%s" % (self.path, region)
self.endpoints = (AWS_ENDPOINTS[region], )
self.ssh_key = SshKey(args)
if self.platform == 'windows':
self.port = 5986
else:
self.port = 22
elif self.provider == 'parallels':
self.endpoints = self._get_parallels_endpoints()
self.max_threshold = 6
self.ssh_key = SshKey(args)
self.port = None
else:
raise ApplicationError('Unsupported platform: %s' % platform)
if persist and load and self._load():
try:
display.info('Checking existing %s/%s instance %s.' %
(self.platform, self.version, self.instance_id),
verbosity=1)
self.connection = self.get(always_raise_on=[404])
display.info('Loaded existing %s/%s from: %s' %
(self.platform, self.version, self._uri),
verbosity=1)
except HttpError as ex:
if ex.status != 404:
raise
self._clear()
display.info('Cleared stale %s/%s instance %s.' %
(self.platform, self.version, self.instance_id),
verbosity=1)
self.instance_id = None
self.endpoint = None
elif not persist:
self.instance_id = None
self.endpoint = None
self._clear()
if self.instance_id:
self.started = True
else:
self.started = False
self.instance_id = str(uuid.uuid4())
self.endpoint = None
display.sensitive.add(self.instance_id)
def __init__(self, args, platform, version, stage='prod', persist=True, name=None):
"""
:type args: CommonConfig
:type platform: str
:type version: str
:type stage: str
:type persist: bool
:type name: str
"""
self.args = args
self.platform = platform
self.version = version
self.stage = stage
self.client = HttpClient(args)
self.connection = None
self.instance_id = None
self.name = name if name else '%s-%s' % (self.platform, self.version)
if self.platform == 'windows':
self.ssh_key = None
self.endpoint = 'https://14blg63h2i.execute-api.us-east-1.amazonaws.com'
self.port = 5986
elif self.platform == 'freebsd':
self.ssh_key = SshKey(args)
self.endpoint = 'https://14blg63h2i.execute-api.us-east-1.amazonaws.com'
self.port = 22
elif self.platform == 'osx':
self.ssh_key = SshKey(args)
self.endpoint = 'https://osx.testing.ansible.com'
self.port = None
else:
raise ApplicationError('Unsupported platform: %s' % platform)
self.path = os.path.expanduser('~/.ansible/test/instances/%s-%s' % (self.name, self.stage))
if persist and self._load():
try:
display.info('Checking existing %s/%s instance %s.' % (self.platform, self.version, self.instance_id),
verbosity=1)
self.connection = self.get()
display.info('Loaded existing %s/%s instance %s.' % (self.platform, self.version, self.instance_id),
verbosity=1)
except HttpError as ex:
if ex.status != 404:
raise
self._clear()
display.info('Cleared stale %s/%s instance %s.' % (self.platform, self.version, self.instance_id),
verbosity=1)
self.instance_id = None
else:
self.instance_id = None
self._clear()
if self.instance_id:
self.started = True
else:
self.started = False
self.instance_id = str(uuid.uuid4())
display.info('Initializing new %s/%s instance %s.' % (self.platform, self.version, self.instance_id),
verbosity=1)