def get_provider_details(self, project_id, service_id):
if service_id not in self.created_service_ids:
raise ValueError("service: % does not exist")
else:
return {
'MaxCDN':
provider_details.ProviderDetail(
provider_service_id=11942,
name='my_service_name',
access_urls=[
'my_service_name'
'.mycompanyalias.netdna-cdn.com'
]),
'Fastly':
provider_details.ProviderDetail(
provider_service_id=3488,
name="my_service_name",
access_urls=['my_service_name'
'.global.prod.fastly.net']),
'CloudFront':
provider_details.ProviderDetail(
provider_service_id=5892,
access_urls=['my_service_name'
'.gibberish.amzcf.com']),
'Mock':
provider_details.ProviderDetail(
provider_service_id="73242",
access_urls=['my_service_name.mock.com'])
}
def execute(self, responders, dns_responder, log_responders):
provider_details_dict = {}
for responder in responders:
for provider_name in responder:
error_class = None
domains_certificate_status = responder[provider_name].get(
'domains_certificate_status', {})
if 'error' in responder[provider_name]:
error_msg = responder[provider_name]['error']
error_info = responder[provider_name]['error_detail']
if 'error_class' in responder[provider_name]:
error_class = \
responder[provider_name]['error_class']
provider_details_dict[provider_name] = (
provider_details.ProviderDetail(
error_info=error_info,
status='failed',
domains_certificate_status=(
domains_certificate_status),
error_message=error_msg,
error_class=error_class))
elif 'error' in dns_responder[provider_name]:
error_msg = dns_responder[provider_name]['error']
error_info = dns_responder[provider_name]['error_detail']
if 'error_class' in dns_responder[provider_name]:
error_class = \
dns_responder[provider_name]['error_class']
provider_details_dict[provider_name] = (
provider_details.ProviderDetail(
error_info=error_info,
status='failed',
domains_certificate_status=(
domains_certificate_status),
error_message=error_msg,
error_class=error_class))
else:
access_urls = dns_responder[provider_name]['access_urls']
if log_responders:
access_urls.append({'log_delivery': log_responders})
provider_details_dict[provider_name] = (
provider_details.ProviderDetail(
provider_service_id=responder[provider_name]['id'],
domains_certificate_status=(
domains_certificate_status),
access_urls=access_urls))
if 'status' in responder[provider_name]:
provider_details_dict[provider_name].status = (
responder[provider_name]['status'])
else:
provider_details_dict[provider_name].status = (
'deployed')
# serialize provider_details_dict
for provider_name in provider_details_dict:
provider_details_dict[provider_name] = (
provider_details_dict[provider_name].to_dict())
return provider_details_dict
def test_purge(self, provider_details_json):
self.provider_details = {}
for provider_name in provider_details_json:
provider_detail_dict = json.loads(
provider_details_json[provider_name])
provider_service_id = provider_detail_dict.get('id', None)
access_urls = provider_detail_dict.get('access_urls', [])
status = provider_detail_dict.get('status', u'deployed')
provider_detail_obj = provider_details.ProviderDetail(
provider_service_id=provider_service_id,
access_urls=access_urls,
status=status)
self.provider_details[provider_name] = provider_detail_obj
self.sc.storage_controller.get_provider_details.return_value = (
self.provider_details)
self.service_obj.provider_details = self.provider_details
self.sc.storage_controller.get.return_value = (self.service_obj)
self.sc.purge(self.project_id, self.service_id)
# ensure the manager calls the storage driver with the appropriate data
sc = self.sc.storage_controller
sc.get_provider_details.assert_called_once_with(
self.project_id,
self.service_id,
)
def format_result(result):
service_id = result.get('service_id')
name = str(result.get('service_name'))
origins = [o for o in result.get('origins', [])]
domains = [d for d in result.get('domains', [])]
origins = [origin.Origin(o['origin'],
hostheadertype='domain',
hostheadervalue='blog.mywebsite.com',
port=o.get('port', 80),
ssl=o.get('ssl', False))
for o in origins]
domains = [domain.Domain(d['domain'], d['protocol'], d['certificate'])
for d in domains]
flavor_id = result.get('flavor_id')
s = service.Service(service_id, name, domains, origins, flavor_id)
provider_detail_results = result.get('provider_details') or {}
provider_details_dict = {}
for provider_name in provider_detail_results:
provider_detail_dict = json.loads(
provider_detail_results[provider_name])
provider_service_id = provider_detail_dict.get('id', None)
access_urls = provider_detail_dict.get('access_urls', [])
status = provider_detail_dict.get('status', u'unknown')
provider_detail_obj = provider_details.ProviderDetail(
provider_service_id=provider_service_id,
access_urls=access_urls,
status=status)
provider_details_dict[provider_name] = provider_detail_obj
s.provider_details = provider_details_dict
return s
def revert(self, provider_details, retry_sleep_time, responders,
project_id, service_id, **kwargs):
"""If failed to delete association dns mapping
failed revert the reestablish dns mapping task.
CNAME created at rackspace dns which used by customer for
their by vanity domain failed to delete. Will reestablish
the dns mapping.
:param dict provider_details: dict of providers
:param int retry_sleep_time: sleep time
:param list[dict] responders: list of responder
:param unicode project_id: project id of the user
:param unicode service_id: uuid of the service
"""
if self.name in kwargs['flow_failures'].keys():
retries = conf[DNS_GROUP].retries
current_progress = (1.0 / retries)
if hasattr(self, 'retry_progress') \
and hasattr(self, 'retry_index'):
self.retry_index = self.retry_index + 1
self.retry_progress = current_progress * self.retry_index
if not hasattr(self, 'retry_progress') \
and not hasattr(self, 'retry_index'):
self.retry_progress = current_progress
self.retry_index = 1
if self.retry_progress == 1.0:
LOG.warning('Maximum retry attempts of '
'{0} reached for Task {1}'.format(
retries, self.name))
LOG.warning('Setting of state of service_id: '
'{0} and project_id: {1} '
'to failed'.format(service_id, project_id))
provider_details_dict = {}
result = kwargs['result']
for responder in responders:
for provider_name in responder:
provider_details_dict[provider_name] = (
pd.ProviderDetail(error_info=result.traceback_str,
status='failed',
error_message='Failed after '
'{0} DNS '
'retries'.format(retries),
error_class=str(
result.exc_info[0])))
# serialize provider_details_dict
for provider_name in provider_details_dict:
provider_details_dict[provider_name] = (
provider_details_dict[provider_name].to_dict())
update_provider_details = common.UpdateProviderDetailTask()
update_provider_details.execute(provider_details_dict,
project_id, service_id)
else:
LOG.warning('Sleeping for {0} seconds and '
'retrying'.format(retry_sleep_time))
if retry_sleep_time is not None:
time.sleep(retry_sleep_time)
def revert(self, responders, retry_sleep_time, service_old, service_obj,
project_id, service_id, **kwargs):
if self.name in kwargs['flow_failures'].keys():
retries = conf[DNS_GROUP].retries
current_progress = (1.0 / retries)
if hasattr(self, 'retry_progress') \
and hasattr(self, 'retry_index'):
self.retry_index = self.retry_index + 1
self.retry_progress = current_progress * self.retry_index
if not hasattr(self, 'retry_progress') \
and not hasattr(self, 'retry_index'):
self.retry_progress = current_progress
self.retry_index = 1
if self.retry_progress == 1.0:
LOG.warning('Maximum retry attempts of '
'{0} reached for Task {1}'.format(
retries, self.name))
LOG.warning('Setting of state of service_id: '
'{0} and project_id: {1} '
'to failed'.format(service_id, project_id))
provider_details_dict = {}
result = kwargs['result']
service_controller, self.storage_controller = \
memoized_controllers.task_controllers('poppy', 'storage')
service_obj_json = json.loads(service_obj)
service_obj = service.load_from_json(service_obj_json)
for responder in responders:
for provider_name in responder:
provider_service_id = (
service_controller._driver.providers[
provider_name.lower()].obj.service_controller.
get_provider_service_id(service_obj))
provider_details_dict[provider_name] = (
provider_details.ProviderDetail(
provider_service_id=provider_service_id,
error_info=result.traceback_str,
status='failed',
error_message='Failed after '
'{0} DNS '
'retries'.format(retries),
error_class=str(result.exc_info[0])))
# serialize provider_details_dict
for provider_name in provider_details_dict:
provider_details_dict[provider_name] = (
provider_details_dict[provider_name].to_dict())
update_provider_details = common.UpdateProviderDetailTask()
update_provider_details.execute(provider_details_dict,
project_id, service_id)
else:
LOG.warning('Sleeping for {0} seconds and '
'retrying'.format(retry_sleep_time))
if retry_sleep_time is not None:
time.sleep(retry_sleep_time)
def test_delete_service_worker_with_error(self, provider_details_json):
self.provider_details = {}
for provider_name in provider_details_json:
provider_detail_dict = json.loads(
provider_details_json[provider_name])
provider_service_id = provider_detail_dict.get("id", None)
access_urls = provider_detail_dict.get("access_urls", None)
status = provider_detail_dict.get("status", u'deployed')
provider_detail_obj = provider_details.ProviderDetail(
provider_service_id=provider_service_id,
access_urls=access_urls,
status=status)
self.provider_details[provider_name] = provider_detail_obj
providers = self.sc._driver.providers
def get_provider_extension_by_name(name):
if name == 'cloudfront':
return mock.Mock(obj=mock.Mock(
provider_name='CloudFront',
service_controller=mock.Mock(delete=mock.Mock(
return_value={
'CloudFront': {
'id': '08d2e326-377e-11e4-b531-3c15c2b8d2d6',
}
}), )))
elif name == 'maxcdn':
return mock.Mock(
obj=mock.Mock(provider_name='MaxCDN',
service_controller=mock.Mock(
delete=mock.Mock(
return_value={
'MaxCDN': {
'error':
"fail to create servcice",
'error_detail':
'MaxCDN delete service'
' failed because of XYZ'
}
}))))
else:
return mock.Mock(obj=mock.Mock(
provider_name=name.title(),
service_controller=mock.Mock(delete=mock.Mock(
return_value={
name.title(): {
'id': '08d2e326-377e-11e4-b531-3c15c2b8d2d6',
}
}), )))
providers.__getitem__.side_effect = get_provider_extension_by_name
with MonkeyPatchControllers(self.sc, self.sc.dns_controller,
self.sc.storage_controller,
memoized_controllers.task_controllers):
self.mock_delete_service()
def load_from_json(json_data):
access_urls = json_data.get("access_urls")
error_info = json_data.get("error_info", )
provider_service_id = json_data.get("id")
status = json_data.get("status")
return provider_details.ProviderDetail(
provider_service_id=provider_service_id,
access_urls=access_urls,
status=status,
error_info=error_info)
def setUp(self):
super(TestProviderWrapper, self).setUp()
self.provider_wrapper_obj = providers.ProviderWrapper()
# fake a provider details to work with unittest
self.fake_provider_details = {
"Fastly":
provider_details.ProviderDetail(
provider_service_id=uuid.uuid1(),
access_urls=['mydummywebsite.prod.fastly.com'])
}
def test_update_provider_details(self, provider_details_json,
mock_session, mock_execute):
provider_details_dict = {}
for k, v in provider_details_json.items():
provider_detail_dict = json.loads(v)
provider_details_dict[k] = provider_details.ProviderDetail(
provider_service_id=(
provider_detail_dict["id"]),
access_urls=provider_detail_dict["access_urls"],
domains_certificate_status=provider_detail_dict.get(
"domains_certificate_status", {}))
# mock the response from cassandra
mock_execute.execute.return_value = None
# this is for update_provider_details unittest code coverage
arg_provider_details_dict = {}
for provider_name in provider_details_dict:
the_provider_detail_dict = collections.OrderedDict()
the_provider_detail_dict["id"] = (
provider_details_dict[provider_name].provider_service_id)
the_provider_detail_dict["access_urls"] = (
provider_details_dict[provider_name].access_urls)
the_provider_detail_dict["status"] = (
provider_details_dict[provider_name].status)
the_provider_detail_dict["name"] = (
provider_details_dict[provider_name].name)
the_provider_detail_dict["domains_certificate_status"] = (
provider_details_dict[provider_name].
domains_certificate_status.to_dict())
the_provider_detail_dict["error_info"] = (
provider_details_dict[provider_name].error_info)
the_provider_detail_dict["error_message"] = (
provider_details_dict[provider_name].error_message)
arg_provider_details_dict[provider_name] = json.dumps(
the_provider_detail_dict)
call_args = {
'project_id': self.project_id,
'service_id': self.service_id,
'provider_details': arg_provider_details_dict
}
# This is to verify mock has been called with the correct arguments
def assert_mock_execute_args(*args):
self.assertEqual(args[0].query_string,
services.CQL_UPDATE_PROVIDER_DETAILS)
self.assertEqual(args[1], call_args)
mock_execute.execute.side_effect = assert_mock_execute_args
self.sc.update_provider_details(
self.project_id,
self.service_id,
provider_details_dict)
def test_update(self, update_json):
provider_details_dict = {
"MaxCDN": {
"id": 11942,
"access_urls": ["mypullzone.netdata.com"]
},
"Mock": {
"id": 73242,
"access_urls": ["mycdn.mock.com"]
},
"CloudFront": {
"id": "5ABC892",
"access_urls": ["cf123.cloudcf.com"]
},
"Fastly": {
"id": 3488,
"access_urls": ["mockcf123.fastly.prod.com"]
}
}
providers_details = {}
for name in provider_details_dict:
details = provider_details_dict[name]
provider_detail_obj = provider_details.ProviderDetail(
provider_service_id=details['id'],
access_urls=details['access_urls'],
status=details.get('status', u'unknown'))
providers_details[name] = provider_detail_obj
providers = self.sc._driver.providers
self.sc.storage_controller.get_provider_details.return_value = (
providers_details)
service_obj = service.load_from_json(self.service_json)
service_obj.status = u'deployed'
self.sc.storage_controller.get.return_value = service_obj
service_updates = json.dumps([{
"op": "replace",
"path": "/domains/0",
"value": {
"domain": "added.mocksite4.com"
}
}])
self.sc.update(self.project_id, self.service_id, self.auth_token,
service_updates)
# ensure the manager calls the storage driver with the appropriate data
self.sc.storage_controller.update.assert_called_once()
# and that the providers are notified.
providers.map.assert_called_once()
def format_result(result):
service_id = result.get('service_id')
name = str(result.get('service_name'))
origins = [o for o in result.get('origins', [])]
domains = [d for d in result.get('domains', [])]
origins = [
origin.Origin(o['origin'],
hostheadertype='domain',
hostheadervalue='blog.mywebsite.com',
port=o.get('port', 80),
ssl=o.get('ssl', False)) for o in origins
]
domains = [
domain.Domain(d['domain'], d['protocol'], d['certificate'])
for d in domains
]
restrictions = [
restriction.Restriction(r.get('name'), r.get(
'access', 'whitelist'), [
rule.Rule(r_rule.get('name'),
referrer=r_rule.get('referrer'),
client_ip=r_rule.get('client_ip'),
geography=r_rule.get('geography'),
request_url=r_rule.get('request_url', "/*")
or "/*") for r_rule in r['rules']
]) for r in result.get('restrictions', [])
]
flavor_id = result.get('flavor_id')
s = service.Service(service_id,
name,
domains,
origins,
flavor_id,
restrictions=restrictions)
provider_detail_results = result.get('provider_details') or {}
provider_details_dict = {}
for provider_name in provider_detail_results:
provider_detail_dict = json.loads(
provider_detail_results[provider_name])
provider_service_id = provider_detail_dict.get('id', None)
access_urls = provider_detail_dict.get('access_urls', [])
status = provider_detail_dict.get('status', u'unknown')
provider_detail_obj = provider_details.ProviderDetail(
provider_service_id=provider_service_id,
access_urls=access_urls,
status=status)
provider_details_dict[provider_name] = provider_detail_obj
s.provider_details = provider_details_dict
return s
def revert(self, provider_details, retry_sleep_time,
responders, project_id, service_id, **kwargs):
if self.name in kwargs['flow_failures'].keys():
retries = conf[DNS_GROUP].retries
current_progress = (1.0 / retries)
if hasattr(self, 'retry_progress') \
and hasattr(self, 'retry_index'):
self.retry_index = self.retry_index + 1
self.retry_progress = current_progress * self.retry_index
if not hasattr(self, 'retry_progress') \
and not hasattr(self, 'retry_index'):
self.retry_progress = current_progress
self.retry_index = 1
if self.retry_progress == 1.0:
LOG.warning(
'Maximum retry attempts of '
'{0} reached for Task {1}'.format(retries, self.name))
LOG.warning(
'Setting of state of service_id: '
'{0} and project_id: {1} '
'to failed'.format(service_id, project_id))
provider_details_dict = {}
result = kwargs['result']
for responder in responders:
for provider_name in responder:
provider_details_dict[provider_name] = (
pd.ProviderDetail(
error_info=result.traceback_str,
status='failed',
error_message='Failed after '
'{0} DNS '
'retries'.format(retries),
error_class=str(result.exc_info[0])))
# serialize provider_details_dict
for provider_name in provider_details_dict:
provider_details_dict[provider_name] = (
provider_details_dict[provider_name].to_dict())
update_provider_details = common.UpdateProviderDetailTask()
update_provider_details.execute(provider_details_dict,
project_id,
service_id)
else:
LOG.warning('Sleeping for {0} seconds and '
'retrying'.format(retry_sleep_time))
if retry_sleep_time is not None:
time.sleep(retry_sleep_time)
def get_provider_details(self, project_id, service_id):
"""get_provider_details.
:param project_id
:param service_id
:returns results Provider details
"""
args = {
'project_id': project_id,
'service_id': uuid.UUID(str(service_id))
}
# TODO(tonytan4ever): Not sure this returns a list or a single
# dictionary.
# Needs to verify after cassandra unittest framework has been added in
# if a list, the return the first item of a list. if it is a dictionary
# returns the dictionary
stmt = query.SimpleStatement(
CQL_GET_PROVIDER_DETAILS,
consistency_level=self._driver.consistency_level)
exec_results_set = self.session.execute(stmt, args)
complete_results = list(exec_results_set)
if len(complete_results) != 1:
raise ValueError('No service found: %s'
% service_id)
provider_details_result = complete_results[0]['provider_details'] or {}
results = {}
for provider_name in provider_details_result:
provider_detail_dict = json.loads(
provider_details_result[provider_name])
provider_service_id = provider_detail_dict.get('id', None)
access_urls = provider_detail_dict.get("access_urls", [])
status = provider_detail_dict.get("status", u'creating')
domains_certificate_status = (
provider_detail_dict.get('domains_certificate_status', {}))
error_info = provider_detail_dict.get("error_info", None)
error_message = provider_detail_dict.get("error_message", None)
provider_detail_obj = provider_details.ProviderDetail(
provider_service_id=provider_service_id,
access_urls=access_urls,
status=status,
domains_certificate_status=domains_certificate_status,
error_info=error_info,
error_message=error_message)
results[provider_name] = provider_detail_obj
return results
def test_update_provider_details_new_provider_details_empty(self):
provider_details_dict = {}
# mock the response from cassandra
self.mock_session.execute.return_value = None
# this is for update_provider_details unittest code coverage
arg_provider_details_dict = {}
status = None
provider_details_args = {
'project_id': self.project_id,
'service_id': self.service_id,
'provider_details': arg_provider_details_dict
}
status_args = {
'status': status,
'project_id': self.project_id,
'service_id': self.service_id
}
# This is to verify mock has been called with the correct arguments
def assert_mock_execute_args(*args):
if args[0].query_string == services.CQL_UPDATE_PROVIDER_DETAILS:
self.assertEqual(args[1], provider_details_args)
elif args[0].query_string == services.CQL_SET_SERVICE_STATUS:
self.assertEqual(args[1], status_args)
self.mock_session.execute.side_effect = assert_mock_execute_args
with mock.patch.object(
services.ServicesController,
'get_provider_details') as mock_provider_det:
mock_provider_det.return_value = {
"MaxCDN": provider_details.ProviderDetail(
provider_service_id=(
"{\"id\": 11942, \"access_urls\": "
"[{\"provider_url\": \"maxcdn.provider.com\", "
"\"domain\": \"xk2.cd\"}], "
"\"domains_certificate_status\":"
"{\"mypullzone.com\": "
"\"failed\"} }"
),
access_urls=[
{
"provider_url": "fastly.provider.com",
"domain": "xk2.cd"
}
]
)
}
self.sc.update_provider_details(
self.project_id,
self.service_id,
provider_details_dict
)
delete_queries = []
deleted_domains = []
for query_mock_call in self.sc.session.execute.mock_calls:
name, args, kwargs = query_mock_call
for arg in args:
if hasattr(arg, 'query_string'):
if (
arg.query_string ==
services.CQL_DELETE_PROVIDER_URL
):
delete_queries.append(query_mock_call)
_, delete_query_args = args
deleted_domains.append(
delete_query_args["domain_name"])
self.assertEqual(1, len(delete_queries))
self.assertEqual(['xk2.cd'], deleted_domains)
self.assertTrue(self.sc.session.execute.called)
def test_update_provider_details(self, provider_details_json):
provider_details_dict = {}
for k, v in provider_details_json.items():
provider_detail_dict = json.loads(v)
provider_details_dict[k] = provider_details.ProviderDetail(
provider_service_id=(
provider_detail_dict["id"]),
access_urls=provider_detail_dict["access_urls"],
domains_certificate_status=provider_detail_dict.get(
"domains_certificate_status", {}))
# mock the response from cassandra
self.mock_session.execute.return_value = None
# this is for update_provider_details unittest code coverage
arg_provider_details_dict = {}
status = None
for provider_name in provider_details_dict:
the_provider_detail_dict = collections.OrderedDict()
the_provider_detail_dict["id"] = (
provider_details_dict[provider_name].provider_service_id)
the_provider_detail_dict["access_urls"] = (
provider_details_dict[provider_name].access_urls)
the_provider_detail_dict["status"] = (
provider_details_dict[provider_name].status)
status = the_provider_detail_dict["status"]
the_provider_detail_dict["name"] = (
provider_details_dict[provider_name].name)
the_provider_detail_dict["domains_certificate_status"] = (
provider_details_dict[provider_name].
domains_certificate_status.to_dict())
the_provider_detail_dict["error_info"] = (
provider_details_dict[provider_name].error_info)
the_provider_detail_dict["error_message"] = (
provider_details_dict[provider_name].error_message)
arg_provider_details_dict[provider_name] = json.dumps(
the_provider_detail_dict)
provider_details_args = {
'project_id': self.project_id,
'service_id': self.service_id,
'provider_details': arg_provider_details_dict
}
status_args = {
'status': status,
'project_id': self.project_id,
'service_id': self.service_id
}
# This is to verify mock has been called with the correct arguments
def assert_mock_execute_args(*args):
if args[0].query_string == services.CQL_UPDATE_PROVIDER_DETAILS:
self.assertEqual(args[1], provider_details_args)
elif args[0].query_string == services.CQL_SET_SERVICE_STATUS:
self.assertEqual(args[1], status_args)
self.mock_session.execute.side_effect = assert_mock_execute_args
with mock.patch.object(
services.ServicesController,
'get_provider_details') as mock_provider_det:
mock_provider_det.return_value = {
"MaxCDN": # "{\"id\": 11942, \"access_urls\": "
# "[{\"provider_url\": \"maxcdn.provider.com\", "
# "\"domain\": \"xk.cd\"}], "
# "\"domains_certificate_status\":"
# "{\"mypullzone.com\": "
# "\"failed\"} }",
provider_details.ProviderDetail(
provider_service_id='{}',
access_urls=[]
)
}
self.sc.update_provider_details(
self.project_id,
self.service_id,
provider_details_dict
)
def execute(self, responders, dns_responder, log_responders):
"""Gathers the status of create service, create dns and log delivery.
:param list[dict] responders:
:param dict dns_responder:
:param list[dict] log_responders: list of log_responder
:return: dict of provider_details_dict
:rtype: dict[str, collections.OrderedDict]
"""
provider_details_dict = {}
for responder in responders:
for provider_name in responder:
error_class = None
domains_certificate_status = responder[provider_name].get(
'domains_certificate_status', {})
if 'error' in responder[provider_name]:
error_msg = responder[provider_name]['error']
error_info = responder[provider_name]['error_detail']
if 'error_class' in responder[provider_name]:
error_class = \
responder[provider_name]['error_class']
provider_details_dict[provider_name] = (
provider_details.ProviderDetail(
error_info=error_info,
status='failed',
domains_certificate_status=(
domains_certificate_status),
error_message=error_msg,
error_class=error_class))
elif 'error' in dns_responder[provider_name]:
error_msg = dns_responder[provider_name]['error']
error_info = dns_responder[provider_name]['error_detail']
if 'error_class' in dns_responder[provider_name]:
error_class = \
dns_responder[provider_name]['error_class']
provider_details_dict[provider_name] = (
provider_details.ProviderDetail(
error_info=error_info,
status='failed',
domains_certificate_status=(
domains_certificate_status),
error_message=error_msg,
error_class=error_class))
else:
access_urls = dns_responder[provider_name]['access_urls']
if log_responders:
access_urls.append({'log_delivery': log_responders})
provider_details_dict[provider_name] = (
provider_details.ProviderDetail(
provider_service_id=responder[provider_name]['id'],
domains_certificate_status=(
domains_certificate_status),
access_urls=access_urls))
if 'status' in responder[provider_name]:
provider_details_dict[provider_name].status = (
responder[provider_name]['status'])
else:
provider_details_dict[provider_name].status = (
'deployed')
# serialize provider_details_dict
for provider_name in provider_details_dict:
provider_details_dict[provider_name] = (
provider_details_dict[provider_name].to_dict())
return provider_details_dict
def revert(self, responders, retry_sleep_time,
project_id, service_id, **kwargs):
"""Reverts the create dnsmapping task is failed.
:param list responders: list of responder
:param int retry_sleep_time: sleep time
:param unicode project_id: project id of user
:param unicode service_id: service id of the service(generated uuid)
"""
if self.name in kwargs['flow_failures'].keys():
retries = conf[DNS_GROUP].retries
current_progress = (1.0 / retries)
if hasattr(self, 'retry_progress') \
and hasattr(self, 'retry_index'):
self.retry_index = self.retry_index + 1
self.retry_progress = current_progress * self.retry_index
if not hasattr(self, 'retry_progress') \
and not hasattr(self, 'retry_index'):
self.retry_progress = current_progress
self.retry_index = 1
if self.retry_progress == 1.0:
LOG.warning(
'Maximum retry attempts of '
'{0} reached for Task {1}'.format(retries, self.name))
LOG.warning(
'Setting of state of service_id: '
'{0} and project_id: {1} '
'to failed'.format(service_id, project_id))
provider_details_dict = {}
result = kwargs['result']
service_controller, self.storage_controller = \
memoized_controllers.task_controllers('poppy', 'storage')
try:
service_obj = self.storage_controller.get_service(
project_id,
service_id
)
except ValueError:
msg = 'Creating service {0} from Poppy failed. ' \
'No such service exists'.format(service_id)
LOG.info(msg)
raise Exception(msg)
for responder in responders:
for provider_name in responder:
provider_service_id = (
service_controller._driver.
providers[provider_name.lower()].obj.
service_controller.
get_provider_service_id(service_obj))
provider_details_dict[provider_name] = (
provider_details.ProviderDetail(
provider_service_id=provider_service_id,
error_info=result.traceback_str,
status='failed',
error_message='Failed after '
'{0} DNS '
'retries'.format(retries),
error_class=str(result.exc_info[0])))
# serialize provider_details_dict
for provider_name in provider_details_dict:
provider_details_dict[provider_name] = (
provider_details_dict[provider_name].to_dict())
update_provider_details = common.UpdateProviderDetailTask()
update_provider_details.execute(provider_details_dict,
project_id,
service_id)
else:
LOG.warning('Sleeping for {0} seconds and '
'retrying'.format(retry_sleep_time))
if retry_sleep_time is not None:
time.sleep(retry_sleep_time)
def execute(self, responders, dns_responder, log_responders, project_id,
service_id, service_obj):
service_controller, self.storage_controller = \
memoized_controllers.task_controllers('poppy', 'storage')
service_obj_json = json.loads(service_obj)
service_obj = service.load_from_json(service_obj_json)
# gather links and status for service from providers
error_flag = False
error_class = None
provider_details_dict = {}
for responder in responders:
for provider_name in responder:
domains_certificate_status = responder[provider_name].get(
'domains_certificate_status', {})
if 'error' in responder[provider_name]:
error_flag = True
provider_details_dict[provider_name] = (
provider_details.ProviderDetail(
status='failed',
domains_certificate_status=(
domains_certificate_status),
error_message=responder[provider_name]['error'],
error_info=responder[provider_name]
['error_detail']))
elif 'error' in dns_responder[provider_name]:
error_flag = True
error_msg = dns_responder[provider_name]['error']
error_info = dns_responder[provider_name]['error_detail']
if 'error_class' in dns_responder[provider_name]:
# stores the error class for debugging purposes.
error_class = dns_responder[provider_name].get(
'error_class')
provider_details_dict[provider_name] = (
provider_details.ProviderDetail(
error_info=error_info,
status='failed',
domains_certificate_status=(
domains_certificate_status),
error_message=error_msg,
error_class=error_class))
else:
access_urls = dns_responder[provider_name]['access_urls']
if log_responders:
if not any('log_delivery' in access_url
for access_url in access_urls):
access_urls.append(
{'log_delivery': log_responders})
provider_details_dict[provider_name] = (
provider_details.ProviderDetail(
provider_service_id=responder[provider_name]['id'],
domains_certificate_status=(
domains_certificate_status),
access_urls=access_urls))
if 'status' in responder[provider_name]:
provider_details_dict[provider_name].status = (
responder[provider_name]['status'])
else:
provider_details_dict[provider_name].status = (
'deployed')
# serialize provider_details_dict
for provider_name in provider_details_dict:
provider_details_dict[provider_name] = (
provider_details_dict[provider_name].to_dict())
self.storage_controller.update_service(project_id, service_id,
service_obj)
provider_details_dict_error_tuple = (provider_details_dict, error_flag)
return provider_details_dict_error_tuple
def format_result(result):
"""format_result.
:param result
:returns formatted result
"""
service_id = result.get('service_id')
project_id = result.get('project_id')
name = result.get('service_name')
flavor_id = result.get('flavor_id')
origins = [json.loads(o) for o in result.get('origins', []) or []]
domains = [json.loads(d) for d in result.get('domains', []) or []]
restrictions = [
json.loads(r) for r in result.get('restrictions', []) or []
]
caching_rules = [
json.loads(c) for c in result.get('caching_rules', []) or []
]
log_delivery = json.loads(result.get('log_delivery', '{}') or '{}')
operator_status = result.get('operator_status', 'enabled')
# create models for each item
origins = [
origin.Origin(o['origin'], o.get('hostheadertype', 'domain'),
o.get('hostheadervalue', None), o.get('port', 80),
o.get('ssl', False), [
rule.Rule(rule_i.get('name'),
request_url=rule_i.get('request_url'))
for rule_i in o.get('rules', [])
]) for o in origins
]
domains = [
domain.Domain(d['domain'], d.get('protocol', 'http'),
d.get('certificate', None)) for d in domains
]
restrictions = [
restriction.Restriction(r.get('name'), r.get(
'access', 'whitelist'), [
rule.Rule(r_rule.get('name'),
referrer=r_rule.get('referrer'),
client_ip=r_rule.get('client_ip'),
geography=r_rule.get('geography'),
request_url=r_rule.get('request_url', "/*")
or "/*") for r_rule in r['rules']
]) for r in restrictions
]
caching_rules = [
cachingrule.CachingRule(
caching_rule.get('name'), caching_rule.get('ttl'), [
rule.Rule(rule_i.get('name'),
request_url=rule_i.get('request_url', '/*')
or '/*') for rule_i in caching_rule['rules']
]) for caching_rule in caching_rules
]
log_delivery = ld.LogDelivery(log_delivery.get('enabled', False))
# create the service object
s = service.Service(service_id=service_id,
name=name,
domains=domains,
origins=origins,
flavor_id=flavor_id,
caching=caching_rules,
restrictions=restrictions,
log_delivery=log_delivery,
operator_status=operator_status,
project_id=project_id)
# format the provider details
provider_detail_results = result.get('provider_details') or {}
provider_details_dict = {}
for provider_name in provider_detail_results:
provider_detail_dict = json.loads(
provider_detail_results[provider_name])
provider_service_id = provider_detail_dict.get('id', None)
access_urls = provider_detail_dict.get('access_urls', [])
status = provider_detail_dict.get('status', u'unknown')
domains_certificate_status = (provider_detail_dict.get(
'domains_certificate_status', {}))
error_message = provider_detail_dict.get('error_message', None)
provider_detail_obj = provider_details.ProviderDetail(
provider_service_id=provider_service_id,
access_urls=access_urls,
domains_certificate_status=domains_certificate_status,
status=status,
error_message=error_message)
provider_details_dict[provider_name] = provider_detail_obj
s.provider_details = provider_details_dict
# return the service
return s
def test_update_provider_details_domain_deleted(
self,
provider_details_json,
):
provider_details_dict = {}
for k, v in provider_details_json.items():
provider_detail_dict = json.loads(v)
provider_details_dict[k] = provider_details.ProviderDetail(
provider_service_id=(
provider_detail_dict["id"]),
access_urls=provider_detail_dict["access_urls"],
domains_certificate_status=provider_detail_dict.get(
"domains_certificate_status", {}))
# mock the response from cassandra
self.mock_session.execute.return_value = None
# this is for update_provider_details unittest code coverage
arg_provider_details_dict = {}
status = None
for provider_name in provider_details_dict:
the_provider_detail_dict = collections.OrderedDict()
the_provider_detail_dict["id"] = (
provider_details_dict[provider_name].provider_service_id)
the_provider_detail_dict["access_urls"] = (
provider_details_dict[provider_name].access_urls)
the_provider_detail_dict["status"] = (
provider_details_dict[provider_name].status)
status = the_provider_detail_dict["status"]
the_provider_detail_dict["name"] = (
provider_details_dict[provider_name].name)
the_provider_detail_dict["domains_certificate_status"] = (
provider_details_dict[provider_name].
domains_certificate_status.to_dict())
the_provider_detail_dict["error_info"] = (
provider_details_dict[provider_name].error_info)
the_provider_detail_dict["error_message"] = (
provider_details_dict[provider_name].error_message)
arg_provider_details_dict[provider_name] = json.dumps(
the_provider_detail_dict)
provider_details_args = {
'project_id': self.project_id,
'service_id': self.service_id,
'provider_details': arg_provider_details_dict
}
status_args = {
'status': status,
'project_id': self.project_id,
'service_id': self.service_id
}
# This is to verify mock has been called with the correct arguments
def assert_mock_execute_args(*args):
if args[0].query_string == services.CQL_UPDATE_PROVIDER_DETAILS:
self.assertEqual(args[1], provider_details_args)
elif args[0].query_string == services.CQL_SET_SERVICE_STATUS:
self.assertEqual(args[1], status_args)
self.mock_session.execute.side_effect = assert_mock_execute_args
with mock.patch.object(
services.ServicesController,
'get_provider_details') as mock_provider_det:
mock_provider_det.return_value = {
"MaxCDN": provider_details.ProviderDetail(
provider_service_id=(
"{\"id\": 11942, \"access_urls\": "
"[{\"provider_url\": \"maxcdn.provider.com\", "
"\"domain\": \"xk2.cd\"}], "
"\"domains_certificate_status\":"
"{\"mypullzone.com\": "
"\"failed\"} }"
),
access_urls=[
{
"provider_url": "fastly.provider.com",
"domain": "xk2.cd"
}
]
)
}
self.sc.update_provider_details(
self.project_id,
self.service_id,
provider_details_dict
)
delete_queries = []
deleted_domains = []
for query_mock_call in self.sc.session.execute.mock_calls:
name, args, kwargs = query_mock_call
for arg in args:
if hasattr(arg, 'query_string'):
if (
arg.query_string ==
services.CQL_DELETE_PROVIDER_URL
):
delete_queries.append(query_mock_call)
_, delete_query_args = args
deleted_domains.append(
delete_query_args["domain_name"])
self.assertEqual(1, len(delete_queries))
self.assertEqual(['xk2.cd'], deleted_domains)
self.assertTrue(self.sc.session.execute.called)
def test_create_service_worker(self, provider_details_json):
self.provider_details = {}
for provider_name in provider_details_json:
provider_detail_dict = json.loads(
provider_details_json[provider_name])
provider_service_id = provider_detail_dict.get('id', None)
access_urls = provider_detail_dict.get('access_urls', None)
status = provider_detail_dict.get('status', u'deployed')
provider_detail_obj = provider_details.ProviderDetail(
provider_service_id=provider_service_id,
access_urls=access_urls,
status=status)
self.provider_details[provider_name] = provider_detail_obj
providers = self.sc._driver.providers
def get_provider_extension_by_name(name):
if name == 'cloudfront':
return_mock = {
'CloudFront': {
'id': '08d2e326-377e-11e4-b531-3c15c2b8d2d6',
'links': [{
'href': 'www.mysite.com',
'rel': 'access_url'
}],
'status': 'deploy_in_progress'
}
}
service_controller = mock.Mock(create=mock.Mock(
return_value=return_mock))
return mock.Mock(
obj=mock.Mock(provider_name='CloudFront',
service_controller=service_controller))
elif name == 'fastly':
return_mock = {
'Fastly': {
'error':
"fail to create servcice",
'error_detail':
'Fastly Create failed'
' because of XYZ'
}
}
service_controller = mock.Mock(create=mock.Mock(
return_value=return_mock))
return mock.Mock(
obj=mock.Mock(provider_name='MaxCDN',
service_controller=service_controller))
else:
return_mock = {
name.title(): {
'id': '08d2e326-377e-11e4-b531-3c15c2b8d2d6',
'links': [{
'href': 'www.mysite.com',
'rel': 'access_url'
}]
}
}
service_controller = mock.Mock(create=mock.Mock(
return_value=return_mock))
return mock.Mock(
obj=mock.Mock(provider_name=name.title(),
service_controller=service_controller))
providers.__getitem__.side_effect = get_provider_extension_by_name
with MonkeyPatchControllers(self.sc, self.sc.dns_controller,
self.sc.storage_controller,
memoized_controllers.task_controllers):
self.mock_create_service(provider_details_json)
def test_update_service_worker_success_and_failure(self,
provider_details_json):
self.provider_details = {}
for provider_name in provider_details_json:
provider_detail_dict = json.loads(
provider_details_json[provider_name])
provider_service_id = provider_detail_dict.get('id', None)
access_urls = provider_detail_dict.get('access_urls', None)
status = provider_detail_dict.get('status', u'deployed')
provider_detail_obj = provider_details.ProviderDetail(
provider_service_id=provider_service_id,
access_urls=access_urls,
status=status)
self.provider_details[provider_name] = provider_detail_obj
providers = self.sc._driver.providers
def get_provider_extension_by_name(name):
if name == 'cloudfront':
return_mock = {
'CloudFront': {
'id': '08d2e326-377e-11e4-b531-3c15c2b8d2d6',
'links': [{
'href': 'www.mysite.com',
'rel': 'access_url'
}],
'status': 'deploy_in_progress'
}
}
service_controller = mock.Mock(create=mock.Mock(
return_value=return_mock))
return mock.Mock(
obj=mock.Mock(provider_name='CloudFront',
service_controller=service_controller))
elif name == 'fastly':
return_mock = {
'Fastly': {
'error':
"fail to create servcice",
'error_detail':
'Fastly Create failed'
' because of XYZ'
}
}
service_controller = mock.Mock(create=mock.Mock(
return_value=return_mock))
return mock.Mock(
obj=mock.Mock(provider_name='MaxCDN',
service_controller=service_controller))
else:
return_mock = {
name.title(): {
'id': '08d2e326-377e-11e4-b531-3c15c2b8d2d6',
'links': [{
'href': 'www.mysite.com',
'rel': 'access_url'
}]
}
}
service_controller = mock.Mock(create=mock.Mock(
return_value=return_mock))
return mock.Mock(
obj=mock.Mock(provider_name=name.title(),
service_controller=service_controller))
providers.__getitem__.side_effect = get_provider_extension_by_name
conf = cfg.CONF
conf(project='poppy', prog='poppy', args=[])
LOG_DELIVERY_OPTIONS = [
cfg.ListOpt('preferred_dcs',
default=['DC1', 'DC2'],
help='Preferred DCs to create container'),
]
LOG_DELIVERY_GROUP = 'log_delivery'
conf.register_opts(LOG_DELIVERY_OPTIONS, group=LOG_DELIVERY_GROUP)
region = random.choice(conf['log_delivery']['preferred_dcs'])
catalog_json = {
'access': {
'serviceCatalog': [{
'type':
'object-store',
'endpoints': [{
'region': region,
'publicURL': 'public',
'internalURL': 'private'
}]
}]
}
}
with MonkeyPatchControllers(self.sc, self.sc.dns_controller,
self.sc.storage_controller,
memoized_controllers.task_controllers):
# NOTE(TheSriram): Successful update
with mock.patch.object(requests,
'post',
return_value=Response(True, catalog_json)):
with mock.patch.object(requests,
'put',
return_value=Response(True)):
self.mock_update_service(provider_details_json)
# NOTE(TheSriram): Unsuccessful update due to keystone
with mock.patch.object(requests,
'post',
return_value=Response(False, catalog_json)):
with mock.patch.object(requests,
'put',
return_value=Response(True)):
self.mock_update_service(provider_details_json)
# NOTE(TheSriram): Unsuccessful update due to swift
with mock.patch.object(requests,
'post',
return_value=Response(True, catalog_json)):
with mock.patch.object(requests,
'put',
return_value=Response(False)):
self.mock_update_service(provider_details_json)
def setUp(self):
super(TestProviderDetails, self).setUp()
self.my_provider_detail = provider_details.ProviderDetail()