def delete_steps(self, steps):
"""
Delete one or more steps
:param steps: A list of dicts. Each dict must have the following keys:
stepId (:class:`str`):
Step Identifier.
resultId (:class:`str`):
Identifier for the result that this step is associated with.
:type steps: list(dict)
"""
delete_steps = []
for step in steps:
delete_step = testmon_messages.StepDeleteRequest.from_dict(step)
delete_steps.append(delete_step)
request = testmon_messages.TestMonitorDeleteStepsRequest(delete_steps)
generic_message = self._message_service.publish_synchronous_message(request)
if generic_message is None:
raise SystemLinkException.from_name('Skyline.RequestTimedOut')
if generic_message.has_error():
raise SystemLinkException(error=generic_message.error)
LOGGER.debug('generic_message = %s', generic_message)
res = testmon_messages.TestMonitorDeleteStepsResponse.from_message(generic_message)
LOGGER.debug('message = %s', res)
return res
def query_steps(self, query=None, skip=0, take=-1):
"""
Return steps that match query
:param query: Object indicating query parameters.
:type query: systemlink.testmonclient.messages.StepQuery
:param skip: Number of steps to skip before searching.
:type skip: int
:param take: Maximum number of steps to return.
:type take: int
:return: Results that matched the query.
:rtype: tuple(list(systemlink.testmonclient.messages.StepResponse), int)
"""
request = testmon_messages.TestMonitorQueryStepsRequest(query, skip, take)
generic_message = self._message_service.publish_synchronous_message(request)
if generic_message is None:
raise SystemLinkException.from_name('Skyline.RequestTimedOut')
if generic_message.has_error():
raise SystemLinkException(error=generic_message.error)
LOGGER.debug('generic_message = %s', generic_message)
res = testmon_messages.TestMonitorQueryStepsResponse.from_message(generic_message)
LOGGER.debug('TotalCount: %d', res.total_count)
return res.steps, res.total_count
def update_results(self, updates, replace=False, determine_status_from_steps=False):
"""
Update one or more test results
:param updates: A list of dicts. Each dict must have the following keys:
status (:class:`systemlink.testmonclient.messages.Status` or `str`):
Test status.
startedAt (:class:`datetime`):
Test start time.
programName (:class:`str`):
Test program name.
systemId (:class:`str`):
Identifier for the system that ran this test.
operator (:class:`str`):
Operator name for this result.
serialNumber (:class:`str`):
Serial number for the Unit Under Test.
totalTimeInSeconds (:class:`float` or :class:`int`):
Test duration time in seconds.
keywords (``list(str)``):
A list of keywords associated with the result.
properties (``dict(str)``):
Key/value pairs of properties associated with the result.
fileIds (``list(str)``):
List of fileIds associated with the result.
:type updates: list(dict)
:param replace: Indicates if keywords, properties, and file ids should replace the existing
collection, or be merged with the existing collection.
:type replace: bool
:param determine_status_from_steps: Indicates if the status should be set based on the
status of the related steps.
:type determine_status_from_steps: bool
"""
result_update_requests = []
for update in updates:
test_status = update['status']
if isinstance(test_status, str):
test_status = testmon_messages.Status(
testmon_messages.StatusType.from_string(test_status.upper()),
test_status)
update['status'] = test_status.to_dict()
result_update_request = testmon_messages.ResultUpdateRequest.from_dict(update)
result_update_requests.append(result_update_request)
request = testmon_messages.TestMonitorUpdateTestResultsRequest(
result_update_requests,
replace,
determine_status_from_steps)
generic_message = self._message_service.publish_synchronous_message(request)
if generic_message is None:
raise SystemLinkException.from_name('Skyline.RequestTimedOut')
if generic_message.has_error():
raise SystemLinkException(error=generic_message.error)
LOGGER.debug('generic_message = %s', generic_message)
res = testmon_messages.TestMonitorUpdateTestResultsResponse.from_message(generic_message)
LOGGER.debug('message = %s', res)
return res
def _der_to_datetime(der_time):
"""
Change a DER binary encoded UTCTime to a :class:`datetime.datetime` object.
DER stands for Distinguished Encoding Rules.
PyCrypto doesn't have logic that parses DER binary encoded times, so
we do it ourselves here.
:param der_time: The DER binary encoded UTCTime.
:type der_time: bytes
:return: A :class:`datetime.datetime` object that represents the DER
binary encoded UTCTime.
:rtype: datetime.datetime
:raises systemlink.messagebus.exceptions.SystemLinkException: If the conversion
fails.
"""
# We expect the DER to start with \x17 which is for UTCTime and then
# with \x0D (13) which is the length. The length will always be the
# same because it must be in the format 'YYMMDDHHMMSSZ'.
if not der_time.startswith(b'\x17\x0D'):
error_info = 'Unable to decode certificate UTCTime. Unexpected header.'
raise SystemLinkException.from_name('Skyline.Exception',
info=error_info)
if sys.version_info[0] < 3:
datetime_string = der_time[2:]
else:
datetime_string = der_time[2:].decode('utf-8')
# From the spec [https://www.ietf.org/rfc/rfc3280.txt]:
# Where YY is greater than or equal to 50, the year SHALL be
# interpreted as 19YY; and
#
# Where YY is less than 50, the year SHALL be interpreted as 20YY.
try:
year = int(datetime_string[0:2])
except ValueError:
error_info = ('Unable to decode certificate UTCTime. '
'Unexpected value: "{0}".'.format(datetime_string))
raise SystemLinkException.from_name('Skyline.Exception',
info=error_info)
if year >= 50:
year_prefix = '19'
else:
year_prefix = '20'
adj_datetime_string = year_prefix + datetime_string
format_str = '%Y%m%d%H%M%SZ'
try:
datetime_obj = datetime.datetime.strptime(adj_datetime_string,
format_str)
except ValueError:
error_info = ('Unable to decode certificate UTCTime. '
'Unexpected value: "{0}".'.format(datetime_string))
raise SystemLinkException.from_name('Skyline.Exception',
info=error_info)
return datetime_obj
def update_steps(self, steps):
"""
Update one or more steps
:param updates: A list of dicts. Each dict must at least have step_id and result_id. The
other dict members are used to update the step, if present:
name (:class:`str`):
Step name.
stepType (:class:`str`):
Step type
stepId (:class:`str`):
Step Identifier.
parentId (:class:`str`):
Identifier for the step parent.
resultId (:class:`str`):
Identifier for the result that this step is associated with.
status (:class:`systemlink.testmonclient.messages.Status` or `str`):
Step status.
totalTimeInSeconds (:class:`float` or :class:`int`):
Step duration time in seconds.
startedAt (:class:`datetime`):
Step start time.
dataModel (:class:`str`):
The name of the data structure in the stepData list. This value identifies
the key names that exist in the stepData parameters object. It is generally
used to provide context for custom UIs to know what values are expected.
stepData (``list(stepData)``):
A list of data objects for the step. Each element contains a text string and
a parameters dictionary of string:string key-value pairs.
children (``list(str)``):
A list of step ids that define other steps in the request that are children of
this step. The ids in this list must exist as objects in the in the request.
"""
step_update_requests = []
for step in steps:
if 'status' in step:
step_status = step['status']
if isinstance(step_status, str):
step_status = testmon_messages.Status(
testmon_messages.StatusType.from_string(step_status.upper()),
step_status)
step['status'] = step_status.to_dict()
step_update_request = testmon_messages.StepUpdateRequest.from_dict(step)
step_update_requests.append(step_update_request)
request = testmon_messages.TestMonitorUpdateTestStepsRequest(step_update_requests)
generic_message = self._message_service.publish_synchronous_message(request)
if generic_message is None:
raise SystemLinkException.from_name('Skyline.RequestTimedOut')
if generic_message.has_error():
raise SystemLinkException(error=generic_message.error)
LOGGER.debug('generic_message = %s', generic_message)
res = testmon_messages.TestMonitorUpdateTestStepsResponse.from_message(generic_message)
LOGGER.debug('message = %s', res)
return res
def create_results(self, results):
"""
Create one or more test results
:param results: A list of dicts. Each dict must have the following keys:
status (:class:`systemlink.testmonclient.messages.Status` or `str`):
Test status.
startedAt (:class:`datetime`):
Test start time.
programName (:class:`str`):
Test program name.
systemId (:class:`str`):
Identifier for the system that ran this test.
hostName (:class:`str`):
Host machine name.
operator (:class:`str`):
Operator name for this result.
serialNumber (:class:`str`):
Serial number for the Unit Under Test.
totalTimeInSeconds (:class:`float` or :class:`int`):
Test duration time in seconds.
keywords (``list(str)``):
A list of keywords associated with the result.
properties (``dict(str)``):
Key/value pairs of properties associated with the result.
fileIds (``list(str)``):
List of fileIds associated with the result.
:type results: list(dict)
"""
result_create_requests = []
for test in results:
test_status = test['status']
if isinstance(test_status, str):
test_status = testmon_messages.Status(
testmon_messages.StatusType.from_string(test_status.upper()), test_status)
test['status'] = test_status.to_dict()
result_create_request = testmon_messages.ResultCreateRequest.from_dict(test)
result_create_requests.append(result_create_request)
request = testmon_messages.TestMonitorCreateTestResultsRequest(result_create_requests)
generic_message = self._message_service.publish_synchronous_message(request)
if generic_message is None:
raise SystemLinkException.from_name('Skyline.RequestTimedOut')
if generic_message.has_error():
raise SystemLinkException(error=generic_message.error)
LOGGER.debug('generic_message = %s', generic_message)
res = testmon_messages.TestMonitorCreateTestResultsResponse.from_message(generic_message)
LOGGER.debug('message = %s', res)
return res
def _check_certificate(cert_abs_path):
"""
Check the cerficate. Will raise
:class:`systemlink.messagebus.exceptions.SystemLinkException` with details if
the certicate check fails.
:param cert_abs_path: Absolute path to the cerficate file.
:type cert_abs_path: str
:raises systemlink.messagebus.exceptions.SystemLinkException: If the certificate
check fails.
"""
if not HAS_ASN1CRYPTO and not HAS_PYCRYPTO:
error_info = 'Cannot import asn1crypto nor PyCrypto[dome] for certificate check.'
raise SystemLinkException.from_name('Skyline.InternalServiceError',
info=error_info)
with open(cert_abs_path) as fp_:
pem = fp_.read()
lines = pem.replace(' ', '').split()
# Skip over begin and end lines
der = a2b_base64(''.join(lines[1:-1]))
# Extract validity field from X.509 certificate (see RFC3280)
if HAS_ASN1CRYPTO:
cert = Certificate.load(der)
tbs_certificate = cert['tbs_certificate']
validity = tbs_certificate['validity']
not_before = validity['not_before'].native
not_before = not_before.replace(tzinfo=None)
not_after = validity['not_after'].native
not_after = not_after.replace(tzinfo=None)
else:
cert = DerSequence()
cert.decode(der)
tbs_certificate = DerSequence()
tbs_certificate.decode(cert[0])
validity = DerSequence()
validity.decode(tbs_certificate[4])
not_before = _der_to_datetime(validity[0])
not_after = _der_to_datetime(validity[1])
now = datetime.datetime.utcnow()
if now < not_before or now > not_after:
error_info = (
'Certificate check failed. Certificate is valid from {0} to {1} (UTC). '
'Current system UTC time is {2}, which is outside the valid range. '
'Please ensure that the current system time is properly set.'
''.format(not_before, not_after, now))
raise SystemLinkException.from_name(
'Skyline.AMQPErrorCertificateExpired', info=error_info)
def to_dict(self, convert_datetime: bool = True) -> Dict[str, Any]:
"""
Serializes the mixin client class to a dictionary.
:param convert_datetime: When ``True``, will convert a
:class:`datetime` object to a :class:`str`. When ``False``,
no such conversion will occur.
:type convert_datetime: bool
:return: The serialized dictionary.
:rtype: dict
"""
result = {}
for attr, (base_name, attr_type,
default) in self.ATTRIBUTE_MAP.items():
value = getattr(self, attr)
if value is None:
# If the value is None, omit it if it isn't a required
# attribute.
if default is _Required:
# Ensure that None is valid for this required attribute.
attr_type, optional = self._get_real_type(attr_type)
if optional is False and attr_type is not Any:
raise SystemLinkException.from_name(
'SkylineWebServices.InvalidJsonDataTypeForKey',
args=[base_name])
result[base_name] = None
else:
result[base_name] = self.serialize_value(
value, convert_datetime=convert_datetime)
return result
def _read_config_file(self, file_path):
"""
Read the configuration file.
:param file_path: Path to the configuration file.
:type file_path: str
:return: A tuple containging the configuration ID and a
:class:`systemlink.messagebus.amqp_configuration.AmqpConfiguration`
object containing the configuration information.
:rtype: tuple(str, systemlink.messagebus.amqp_configuration.AmqpConfiguration)
or tuple(None, None)
"""
last_modifed_time = os.path.getmtime(file_path)
with codecs.open(file_path, 'r', encoding='utf-8-sig') as fp_:
configuration = json.loads(fp_.read())
if 'Id' in configuration:
configuration_id = configuration['Id'].lower()
if configuration_id in self._configurations:
raise SystemLinkException.from_name('Skyline.DuplicateConfigurationId')
amqp_config = AmqpConfiguration(
file_path, configuration, last_modifed_time
)
return configuration_id, amqp_config
return None, None
def _initialize(self):
"""
Used to initialize this :class:`AmqpConsumer` object or re-initialize
it after a disconnect is detected.
"""
LOGGER.debug('AmqpConnection _initialize!')
credentials = pika.PlainCredentials(self._user_name, self._password)
cert_abs_path = None
if self._use_tls:
# Get RabbitMQ server certificate path
if os.path.isabs(self._cert_path):
cert_abs_path = self._cert_path
else:
cert_abs_path = os.path.join(
get_base_configuration_directory(), 'Certificates',
self._cert_path)
if pika.__version__[0] == '0':
# Backwards compatibility for pika < 1.0.0.
ssl_options = {
'ssl_version': ssl.PROTOCOL_TLSv1_2,
'cert_reqs': ssl.CERT_REQUIRED,
'ca_certs': cert_abs_path
}
kwargs = {'ssl': True}
else:
ssl_context = ssl.SSLContext(protocol=ssl.PROTOCOL_TLSv1_2)
ssl_context.verify_mode = ssl.CERT_REQUIRED
ssl_context.load_verify_locations(cert_abs_path)
ssl_options = pika.SSLOptions(ssl_context)
kwargs = {}
else:
ssl_options = None
kwargs = {}
conn_params = pika.ConnectionParameters(host=self._host_name,
port=self._port,
credentials=credentials,
ssl_options=ssl_options,
**kwargs)
try:
self._connection = pika.BlockingConnection(conn_params)
except Exception as exc: # pylint: disable=broad-except
# This will throw its own exception if it detects a problem.
if cert_abs_path:
_check_certificate(cert_abs_path)
# Throw a generic exception since we don't know what the specific
# problem is.
exc_name = exc.__class__.__name__
exc_str = str(exc)
if exc_str:
msg = 'Error connecting to AMQP. {0}: {1}'.format(
exc_name, exc_str)
else:
msg = 'Error connecting to AMQP. {0}'.format(exc_name)
raise SystemLinkException.from_name(
'Skyline.AMQPErrorOpeningTCPConnection', info=msg)
def message_body_as_bytes(self): # pylint: disable=no-self-use
"""
Generate the raw message body from message specific parameters.
:return: The raw message body.
:rtype: bytes
"""
error_info = 'MessageBase.message_body_as_bytes should be overriden by subclass.'
raise SystemLinkException.from_name('Skyline.UnexpectedException', info=error_info)
def _from_embedded_dict(
cls, dct: Dict[Any, Any], key_type: type,
val_type: type) -> Dict[Any, Any]: # pylint: disable=too-many-branches
"""
Performs deserialization of an embedded dictionary.
:param dct: The embedded dictionary.
:type dct: dict
:param key_type: The type of each dictionary key.
:type key_type: type
:param val_type: The type of each dictionary value.
:type val_type: type
:return: A dictionary of deserialized objects meant to be used as
constructor arguments for the parent class.
:rtype: dict
:raises SystemLinkException: if an error occurs.
"""
ret_dct = {}
val_type, optional = cls._get_real_type(val_type)
embedded_list = False
embedded_dict = False
if hasattr(val_type,
'__origin__') and val_type.__origin__ in (List, list):
item_type = val_type.__args__[0]
embedded_list = True
elif hasattr(val_type,
'__origin__') and val_type.__origin__ in (Dict, dict):
key_type, val_type = val_type.__args__
embedded_dict = True
for key, value in dct.items():
if value is None:
if optional or val_type is Any:
ret_dct[key] = None
else:
raise SystemLinkException.from_name(
'SkylineWebServices.InvalidJsonDataTypeForKey',
args=[key])
elif embedded_list:
ret_dct[key] = cls._from_embedded_list(value, item_type) # pylint: disable=protected-access
elif embedded_dict:
ret_dct[key] = cls._from_embedded_dict(value, key_type,
val_type) # pylint: disable=protected-access
elif val_type is Any:
ret_dct[key] = value
elif val_type is datetime:
ret_dct[key] = to_datetime(value)
elif hasattr(val_type, 'from_dict'):
ret_dct[key] = val_type.from_dict(value)
else:
ret_dct[key] = val_type(value)
return ret_dct
def delete_results(self, ids, delete_steps=True):
"""
Delete one or more test results
:param ids: A list of the IDs of the results to delete.
:type ids: list(str)
:param delete_steps: Whether or not to delete the results corresponding steps.
:type delete_steps: bool
"""
request = testmon_messages.TestMonitorDeleteResultsRequest(ids, delete_steps)
generic_message = self._message_service.publish_synchronous_message(request)
if generic_message is None:
raise SystemLinkException.from_name('Skyline.RequestTimedOut')
if generic_message.has_error():
raise SystemLinkException(error=generic_message.error)
LOGGER.debug('generic_message = %s', generic_message)
res = testmon_messages.TestMonitorDeleteResultsResponse.from_message(generic_message)
LOGGER.debug('message = %s', res)
return res
def _get_configuration_helper(self, id_, enable_fallbacks, refresh):
"""
Helper function to obtain an instance of
:class:`systemlink.messagebus.amqp_configuration.AmqpConfiguration`.
:param id_: The configuration ID to use or ``None`` to use
the default configuration.
:type id_: str or None
:param enable_fallbacks: If ``True`` and the desired configuration
ID is not found, will fall back to the default configuration.
If ``False`` and the desired configuration is not found, will
raise a :class:`systemlink.messagebus.exceptions.SystemLinkException`.
:type enable_fallbacks: bool
:param refresh: If ``True``, will refresh the selected configuration
if it has changed since it was last loaded. If ``False``, will not
refresh and could potentially use stale data.
:type refresh: bool
"""
if refresh:
# Check for new files
self._configurations.update(
self._read_configurations(only_new_files=True)
)
selected_amqp_configuration = None
if not id_:
selected_amqp_configuration = self._fallback()
if selected_amqp_configuration is None:
raise SystemLinkException.from_name('Skyline.NoSkylineConfigurations')
else:
selected_amqp_configuration = self._configurations.get(id_)
if selected_amqp_configuration is None:
if enable_fallbacks:
selected_amqp_configuration = self._fallback()
if selected_amqp_configuration is None:
raise SystemLinkException.from_name('Skyline.NoSkylineConfigurations')
else:
raise SystemLinkException.from_name('Skyline.NoSkylineConfigurations')
if refresh:
selected_amqp_configuration.refresh()
return selected_amqp_configuration
def get_services(message_service=None):
"""
Get information for all services.
:param message_service: An instance of
:class:`systemlink.messagebus.message_service.MessageService`.
May be ``None`` in which case a temporary instance will be
used.
:type message_service: systemlink.messagebus.message_service.MessageService
or None
:return: A list of service information.
:rtype:
list(systemlink.messagebus.service_manager_messages.ServiceInstanceBase)
"""
if message_service:
own_message_service = False
else:
own_message_service = True
connection_manager = AmqpConnectionManager()
builder = MessageServiceBuilder('ServiceManagerClient')
builder.connection_manager = connection_manager
message_service = MessageService(builder)
try:
request = service_manager_messages.SvcMgrGetServiceInfoSnapshotRequest(
)
generic_message = message_service.publish_synchronous_message(request)
if generic_message is None:
raise SystemLinkException.from_name('Skyline.RequestTimedOut')
if generic_message.has_error():
raise SystemLinkException(error=generic_message.error)
response = service_manager_messages.SvcMgrGetServiceInfoSnapshotResponse.from_message(
generic_message)
return response.services
finally:
if own_message_service:
message_service.close()
message_service = None
connection_manager.close()
connection_manager = None
def trace_logger(self):
"""
Get the Trace Logger.
:return: The Trace Logger.
:rtype: systemlink.messagebus.trace_logger.TraceLogger
"""
if self._trace_logger is None:
error_info = 'InvalidOperationException: No TraceLogger set.'
raise SystemLinkException.from_name('Skyline.Exception',
info=error_info)
return self._trace_logger
def _load_config(self):
"""
Load the File Subscriber Example Service configuration.
"""
self.message_service.subscriber.start_handling_messages()
request = configuration_messages.ConfigurationGetKeysRequest(self._service_name)
generic_response = self.message_service.publish_synchronous_message(request)
if generic_response is None:
raise SystemLinkException.from_name(
'Skyline.UnexpectedException',
info='Unable to find the configuration for the \"{0}\" service.'.format(
self._service_name
)
)
if generic_response.has_error():
raise SystemLinkException(error=generic_response.error)
response = configuration_messages.ConfigurationGetKeysResponse.from_message(
generic_response
)
self._config = response.keys
amqp_config = AmqpConfigurationManager.get_configuration(refresh=False)
self._config['Amqp.ExchangeName'] = amqp_config.exchange_name
self._config['Amqp.Host'] = amqp_config.host
self._config['Amqp.Port'] = str(amqp_config.port)
self._config['Amqp.User'] = amqp_config.user
self._config['Amqp.Password'] = amqp_config.password
if amqp_config.use_tls:
self._config['Amqp.UseTls'] = 'true'
else:
self._config['Amqp.UseTls'] = 'false'
self._config['Amqp.TlsServerName'] = amqp_config.tls_server_name
if os.path.isabs(amqp_config.cert_path):
self._config['Amqp.CertPath'] = amqp_config.cert_path
else:
self._config['Amqp.CertPath'] = os.path.join(
get_application_data_directory(),
'Certificates',
amqp_config.cert_path
)
def _from_embedded_list( # pylint: disable=too-many-branches
cls, lst: List[Any], item_type: type) -> List[Any]:
"""
Performs deserialization of an embedded list.
:param lst: The embedded list.
:type lst: list
:param item_type: The type of each list element.
:type item_type: type
:return: A list of deserialized objects meant to be used as
constructor arguments for the parent class.
:rtype: list
:raises SystemLinkException: if an error occurs.
"""
ret_lst = []
item_type, optional = cls._get_real_type(item_type)
embedded_list = False
embedded_dict = False
if hasattr(item_type,
'__origin__') and item_type.__origin__ in (List, list):
item_type = item_type.__args__[0]
embedded_list = True
elif hasattr(item_type,
'__origin__') and item_type.__origin__ in (Dict, dict):
key_type, val_type = item_type.__args__
embedded_dict = True
for value in lst:
if value is None:
if optional or item_type is Any:
ret_lst.append(None)
else:
raise SystemLinkException.from_name(
'SkylineWebServices.InvalidJsonDataTypeForKey',
args=[cls.__name__])
elif embedded_list:
ret_lst.append(cls._from_embedded_list(value, item_type)) # pylint: disable=protected-access
elif embedded_dict:
ret_lst.append(
cls._from_embedded_dict(value, key_type, val_type)) # pylint: disable=protected-access
elif item_type is Any:
ret_lst.append(value)
elif item_type is datetime:
ret_lst.append(to_datetime(value))
elif hasattr(item_type, 'from_dict'):
ret_lst.append(item_type.from_dict(value))
else:
ret_lst.append(item_type(value))
return ret_lst
def deserialize_value(cls, value: Any, type_: type, name=None) -> Any: # pylint: disable=too-many-return-statements
"""
Deserialize a single value.
:param value: The value to deserialize.
:type value: Any
:param type_: The type of the value to deserialize.
:type type_: Any
:param name: The name of the value. Used in exception messages.
:type name: str or None
:return: An instance of the class of the deserialized value.
:rtype: Any
:raises SystemLinkException: if an error occurs.
"""
if value is _Required:
raise SystemLinkException.from_name(
'SkylineWebServices.MissingRequiredJsonKey', args=[name])
real_type, optional = cls._get_real_type(type_)
if value is None:
if optional or real_type is Any:
return None
raise SystemLinkException.from_name(
'SkylineWebServices.InvalidJsonDataTypeForKey', args=[name])
if hasattr(real_type,
'__origin__') and real_type.__origin__ in (List, list):
item_type = real_type.__args__[0]
return cls._from_embedded_list(value, item_type) # pylint: disable=protected-access
if hasattr(real_type,
'__origin__') and real_type.__origin__ in (Dict, dict):
key_type, val_type = real_type.__args__
return cls._from_embedded_dict(value, key_type, val_type) # pylint: disable=protected-access
if real_type is Any:
return value
if real_type is datetime:
return to_datetime(value)
if hasattr(real_type, 'from_dict'):
return real_type.from_dict(value)
return real_type(value)
def __init__(self,
name,
parent,
process_logger,
log_to_trace_logger=False):
"""
:param name: The last part of the name to use for this Trace Logger.
Will not be the full name if ``parent`` is not ``None.
May be ``None``.
:type name: str or None
:param parent: The parent Trace Logger object used to create this one.
May be ``None``.
:type parent: TraceLogger
:param process_logger: The Process Logger associated with this Trace
Point.
:type process_logger: systemlink.messagebus.process_logger.ProcessLogger
:param log_to_trace_logger: ``True`` if this :class:`TraceLogger`
instance should automatically send Python logging to the Trace
Logger service. Only one :class:`TraceLogger` instance may do so
per :class:`systemlink.messagebus.process_logger.ProcessLogger`
instance. ``False`` otherwise.
:type log_to_trace_logger: bool
"""
self._name = ''
self._process_logger = None
self._log_handler = None
self._closing = False
builder = ''
if parent:
builder += parent.name + '.'
if name:
builder += name
self._name = builder
self._process_logger = process_logger
self._log_to_trace_logger = log_to_trace_logger
if self._log_to_trace_logger:
if self._process_logger.log_to_trace_logger:
# Only one TraceLogger instance can be set to automatically
# send Python logging to the TraceLogger service.
error_info = (
'Cannot direct Python logging to TraceLogger more than '
'once per ProcessLogger instance')
raise SystemLinkException.from_name('Skyline.Exception',
info=error_info)
self._setup_log_handler()
self._process_logger._log_to_trace_logger = True # pylint: disable=protected-access
def start_service(service_name, message_service=None):
"""
Start a specific service.
:param service_name: The name of the service.
:type service_name: str
:param message_service: An instance of
:class:`systemlink.messagebus.message_service.MessageService`.
May be ``None`` in which case a temporary instance will be
used.
:type message_service: systemlink.messagebus.message_service.MessageService
or None
:param silent: If ``False``, will print status to stdout/stderr.
If ``True``, will not print status.
:type silent: bool
"""
if message_service:
own_message_service = False
else:
own_message_service = True
connection_manager = AmqpConnectionManager()
builder = MessageServiceBuilder('ServiceManagerClient')
builder.connection_manager = connection_manager
message_service = MessageService(builder)
try:
node_name = None
services = get_services(message_service=message_service)
for service in services: # pylint: disable=not-an-iterable
if service.name == service_name:
node_name = service.node_name
break
if node_name is None:
error_info = 'Service name "{0}" not found.'.format(service_name)
raise SystemLinkException.from_name('Skyline.Exception',
info=error_info)
broadcast = service_manager_messages.SvcMgrStartServicesBroadcast(
False, [node_name], False, [service_name], 1)
message_service.publish_broadcast(broadcast)
finally:
if own_message_service:
message_service.close()
message_service = None
connection_manager.close()
connection_manager = None
def queue_declare(self, amqp_channel, queue_name, durable, exclusive,
autodelete): # pylint: disable=too-many-arguments
"""
Declare a queue.
:param amqp_channel: A :class:`systemlink.messagebus.amqp_channel.AmqpChannel` object.
:type amqp_channel: systemlink.messagebus.amqp_channel.AmqpChannel
:param queue_name: The queue name.
:type queue_name: str
:param durable: ``True`` if the queue is durable (backed up to
persistent storage).
:type durable: bool
:param exclusive: ``True`` if the queue is exclusive.
:type exclusive: bool
:param autodelete: ``True`` is the queue should automatically delete
itself when it is no longer referenced.
:type autodelete: bool
"""
LOGGER.debug('AmqpConnection queue_declare! '
'Queue name: %s.', queue_name)
passive = False
with self._connection_lock:
if not self._connection or not amqp_channel.channel:
raise_connection_closed()
with self._convert_error_contextmanager(
amqp_channel.channel,
default_error_name='Skyline.AMQPErrorDeclaringQueue'):
amqp_queue_declare_ok = amqp_channel.channel.queue_declare(
queue=queue_name,
passive=passive,
durable=durable,
exclusive=exclusive,
auto_delete=autodelete)
LOGGER.debug('amqp_queue_declare_ok = %s', amqp_queue_declare_ok)
if amqp_queue_declare_ok.method.queue != queue_name:
error_info = ('Error declaring queue. '
'Channel: {0}. '
'Queue name: {1}. '
'Durable: {2}. '
'Exclusive: {3}. '
'Auto delete: {4}. '.format(
amqp_channel.channel, queue_name, durable,
exclusive, autodelete))
raise SystemLinkException.from_name(
'Skyline.AMQPErrorDeclaringQueue', error_info)
def create_channel(self):
"""
Create a new channel on this connection.
:return: A :class:`systemlink.messagebus.amqp_channel.AmqpChannel`
object.
:rtype: systemlink.messagebus.amqp_channel.AmqpChannel
"""
cls = self.__class__
with self._connection_lock:
if not self._connection:
raise_connection_closed()
found_channel_id = 0
for channel_id in range(1, 32000):
if channel_id not in self._amqp_channels.keys():
LOGGER.debug('channel_id %d not in keys %s!', channel_id,
self._amqp_channels.keys())
with self._convert_error_contextmanager(
default_error_name='Skyline.AMQPErrorOpeningChannel'
):
channel = self._connection.channel(channel_id)
queue = Queue()
amqp_channel = AmqpChannel(self, channel, channel_id,
queue)
self._amqp_channels[channel_id] = amqp_channel
found_channel_id = channel_id
break
if found_channel_id == 0:
error_info = 'All channel IDs have been reserved'
raise SystemLinkException.from_name('Skyline.Exception',
info=error_info)
if self._heartbeat_only and len(self._amqp_channels) == 1:
self.start_consuming(True)
LOGGER.debug(
'AmqpConnection create_channel exchange_initialized = %s',
cls._exchange_initialized # pylint: disable=protected-access
)
if not cls._exchange_initialized: # pylint: disable=protected-access
self.declare_exchange(found_channel_id)
cls._exchange_initialized = True # pylint: disable=protected-access
LOGGER.debug('Setting exchange_initialized: %s',
cls._exchange_initialized) # pylint: disable=protected-access
return amqp_channel
def get_full_name(service_name, instance_name=None):
"""
Get the full name based on service name and instance name if
it exists.
:param service_name: The name of the message service.
:type service_name: str
:param instance_name: The name of the message service instance.
May be ``None`` if there is only one instance of the
message service.
:type instance_name: str or None
:return: The full name.
:rtype: str
"""
if service_name is None:
error_info = 'Service name cannot be none.'
raise SystemLinkException.from_name('Skyline.Exception',
info=error_info)
if instance_name is None:
return service_name
return service_name + '_' + instance_name
def _read_configurations(self, only_new_files=False):
"""
Read configurations by reading all '.json' files in the
configuration directory as well as the environment variables.
:param only_new_files: If ``True``, read only files not read since
last scan.
:type only_new_files: bool
:return: A dictionary with key of the configuration ID and value
an instance of
:class:`systemlink.messagebus.amqp_configuration.AmqpConfiguration`.
:rtype: dict(str, systemlink.messagebus.amqp_configuration.AmqpConfiguration)
"""
configurations = self._read_configurations_from_env()
path = get_skyline_configurations_directory()
if not os.path.exists(path):
if configurations:
return configurations
raise SystemLinkException.from_name('Skyline.NoSkylineConfigurations')
only_files = [file_name for file_name in os.listdir(path)
if os.path.isfile(os.path.join(path, file_name))]
for file_name in only_files:
file_path = os.path.join(
path,
file_name
)
if file_name.endswith('.json'):
if not only_new_files or file_path not in self._read_file_set:
# Read .json configuration file
configuration_id, amqp_config = self._read_config_file(file_path)
if configuration_id is not None:
configurations[configuration_id] = amqp_config
self._read_file_set.add(file_path)
return configurations
def unregister_binding(self, amqp_channel, queue_name, binding_parameters):
"""
Unregister a binding.
:param amqp_channel: A :class:`systemlink.messagebus.amqp_channel.AmqpChannel` object.
:type amqp_channel: systemlink.messagebus.amqp_channel.AmqpChannel
:param queue_name: The queue name.
:type queue_name: str
:param binding_parameters: A list of strings that represent a routing key.
:type binding_parameters: list(str)
"""
binding_key = ROUTING_KEY_PREFIX
for param in binding_parameters:
binding_key += '.' + param
with self._connection_lock:
if not self._connection or not amqp_channel.channel:
raise_connection_closed()
with self._convert_error_contextmanager(
amqp_channel.channel,
default_error_name='Skyline.AMQPErrorUnbindingQueue'):
amqp_rpc_reply = amqp_channel.channel.queue_unbind(
queue_name, self._exchange_name, binding_key)
if amqp_rpc_reply != 0:
error_info = ('Error unbinding queue.'
'Channel: {0}. '
'Queue name: {1}. '
'Exchange name: {2}. '
'Binding key: {3}. '
'AmqpRPCReply: {4}. '.format(
amqp_channel.channel, queue_name,
self._exchange_name, binding_key,
amqp_rpc_reply))
raise SystemLinkException.from_name(
'Skyline.AMQPErrorUnbindingQueue', info=error_info)
def _read_configurations_from_env(): # pylint: disable=too-many-locals
"""
Attempt to read a configuration by reading the environment variables.
:return: A dictionary with key of the configuration ID and value
an instance of
:class:`systemlink.messagebus.amqp_configuration.AmqpConfiguration`.
Will return an empty dictionary if there is no
configuration in the environment variables.
:rtype: dict(str, systemlink.messagebus.amqp_configuration.AmqpConfiguration)
"""
configurations = {}
host = os.environ.get(Environment.HOST)
if not host:
return configurations
port = os.environ.get(Environment.PORT)
exchange = os.environ.get(Environment.EXCHANGE)
user = os.environ.get(Environment.USER)
password = os.environ.get(Environment.PASSWORD)
cert = os.environ.get(Environment.CERTIFICATE)
cert_path = os.environ.get(Environment.CERTIFICATE_PATH)
# The certificate is optional.
if not port or not exchange or not user or password is None:
return configurations
actual_cert_path = None
if cert_path:
actual_cert_path = cert_path
elif cert:
if is_windows():
raise NotImplementedError(
'Passing a TLS certificate via the environment is not supported on Windows'
)
temp_cert_dir = os.path.dirname(_TEMP_CERT_PATH)
if not os.path.isdir(temp_cert_dir):
os.makedirs(temp_cert_dir)
with open(_TEMP_CERT_PATH, 'w') as fp_:
fp_.write(cert)
actual_cert_path = _TEMP_CERT_PATH
try:
port_num = int(port)
except (TypeError, ValueError):
raise SystemLinkException.from_name('Skyline.FailedToParse')
configuration = {
'Id': 'skyline_localhost',
'DisplayName': 'Local',
'ConnectionType': 'Local',
'ExchangeName': exchange,
'Host': host,
'Port': port_num,
'User': user,
'Password': password,
'UseTls' : actual_cert_path is not None,
'TlsServerName': host if actual_cert_path is not None else None,
'CertPath': actual_cert_path
}
amqp_config = AmqpConfiguration(None, configuration)
configurations[SKYLINE_LOCALHOST_CONFIGURATION_ID] = amqp_config
return configurations
def __init__(self,
service_name,
shutdown_event,
managed_service_builder=None): # pylint: disable=too-many-branches,too-many-statements
"""
:param service_name: The name of the message service. Ignored
if ``managed_service_builder`` is not ``None``. In this
case, ``service_name`` may be ``None``.
:type service_name: str or None
:param shutdown_event: A :class`threading.Semaphore` instance
that is released when the Skyline Service Manager requires
the service to shut down.
:type shutdown_event: threading.Semaphore
:param message_service_builder: A
:class:`systemlink.messagebus.managed_service_builder.ManagedServiceBuilder`
object used in the construction of this object. May be
``None``, in which case ``service_name`` must be set.
:type message_service_builder:
systemlink.messagebus.managed_service_builder.ManagedServiceBuilder
or None
"""
LOGGER.debug('ManagedServiceBase constructor!')
if managed_service_builder is None:
managed_service_builder = ManagedServiceBuilder(service_name)
if shutdown_event is None:
error_info = 'Shutdown event cannot be None'
raise SystemLinkException.from_name('Skyline.Exception',
info=error_info)
self._closing = False
self._work_subscriber = None
self._control_subscriber = None
self._node_name = ''
self._service_name = None
self._instance_name = None
self._shutdown_thread = None
self._old_stdout = None
self._old_stderr = None
self._devnull_fp = None
if (sys.stdout is None or sys.stderr is None
or sys.stdout.encoding != 'utf-8'
or sys.stderr.encoding != 'utf-8'):
# We will hit this case when the stdout/stderr pipes
# are closed before creating this process. This is the
# current behavior of Service Manager. In Python 3.5.x,
# this will result in `sys.stdout` and `sys.stderr` being
# None. In Python 3.6.x, this will result in `sys.stdout`
# and `sys.stderr` being created by Python, but imperfectly
# such that the encoding is the system encoding (typically
# 'cp1252') instead of 'utf-8' which will still cause
# problems. This should be fixed in Python 3.7.x in which
# case it should create pipes with an encoding of 'utf-8'.
# Once Python 3.7.x is the minimum supported version, this
# code may be removed.
self._old_stdout = sys.stdout
self._old_stderr = sys.stderr
self._devnull_fp = open(os.devnull, 'w', encoding='utf-8')
sys.stdout = self._devnull_fp
sys.stderr = self._devnull_fp
self._service_name = managed_service_builder.service_name
self._status = service_manager_messages.ServiceState(
service_manager_messages.ServiceState.STARTING)
self._standalone = managed_service_builder.standalone_property
if not managed_service_builder.instance_name:
self._instance_name = self.get_unique_instance_name()
else:
self._instance_name = managed_service_builder.instance_name
self._no_configuration_request = managed_service_builder.no_configuration_request
self._service_guid = os.environ.get(SERVICE_GUID)
if not self._service_guid:
self._service_guid = ''
self._service_group_name = os.environ.get(SERVICE_GROUP_NAME)
if not self._service_group_name:
self._service_group_name = DEFAULT_SERVICE_GROUP_NAME
if not self._standalone:
message_subscriber_builder = MessageSubscriberBuilder(
self.control_queue_name)
message_subscriber_builder.callback = self._control_message_callback
message_subscriber_builder.register_default_binding = True
message_subscriber_builder.auto_start_consumers = False
self._control_subscriber = MessageSubscriber(
message_subscriber_builder)
self._control_subscriber.register_callback(
service_manager_messages.
SvcMgrSendServiceStatusRequestBroadcast,
self._control_message_callback)
self._control_subscriber.register_callback(
configuration_messages.
ConfigurationGetSectionKeyValuesResponse,
self._control_message_callback)
if not managed_service_builder.no_work_subscriber:
if managed_service_builder.work_subscriber_builder:
work_builder = managed_service_builder.work_subscriber_builder
else:
work_builder = MessageSubscriberBuilder(self._service_name)
if work_builder.callback is None:
work_builder.callback = self._receive_message
work_builder.auto_start_consumers = False
self._work_subscriber = MessageSubscriber(work_builder)
if managed_service_builder.instance_subscriber_builder is not None:
instance_builder = managed_service_builder.instance_subscriber_builder
else:
instance_builder = MessageSubscriberBuilder(self.get_full_name())
instance_builder.register_default_binding = True
instance_builder.auto_start_consumers = False
message_service_builder = MessageServiceBuilder(self._service_name)
message_service_builder.instance_name = self._instance_name
message_service_builder.subscriber_builder = instance_builder
self._message_service = MessageService(
message_service_builder=message_service_builder)
self._message_service.instance_name = self._instance_name
self._process_logger = ProcessLogger(
self.get_full_name(), None,
self._message_service.publisher.publish_message_callback,
self._message_service.register_callback)
log_to_trace_logger = managed_service_builder.log_to_trace_logger
self._message_service.trace_logger = (
self._process_logger.make_trace_logger(
self._service_name, log_to_trace_logger=log_to_trace_logger))
if self._control_subscriber is not None:
self._control_subscriber.trace_logger = (
self._process_logger.make_trace_logger(
'Control', self._message_service.trace_logger))
if self._work_subscriber is not None:
self._work_subscriber.trace_logger = self._message_service.trace_logger
self._trace_unhandled_message = self.trace_logger.make_trace_point(
'UnhandledMessage')
self._configured = False
self._need_to_go_live = False
self._trace_unhandled_message = None
self._shutdown_event = shutdown_event
self._shutdown_thread = None
def _convert_error_contextmanager(self,
channel=None,
default_error_name='Skyline.Exception'):
"""
A context manager to convert Pika errors to Skyline errors.
:param channel: If a channel is being used, the channel. Otherwise
``None``.
:type channel: pika.channel.Channel or None
:param default_error_code: The error code to use if a more
specific error code cannot be determined.
:type default_error_code: int
"""
try:
yield
except SystemLinkException:
raise
except Exception as orig_exc: # pylint: disable=broad-except
# As for AttributeError:
# We see this sometimes when the socket is closed:
#
# File "/usr/lib/python2.7/site-packages/pika/adapters/base_connection.py", line 427,
# in _handle_write
# bw = self.socket.send(frame)
# AttributeError: 'NoneType' object has no attribute 'send'
#
# In this case, the socket is gone but it is still trying to use it.
# This could happen if the broker went down.
# As for ValueError:
# We see this sometimes when the socket is closed on the consumer:
#
# File "c:\...\lib\site-packages\pika\adapters\base_connection.py", line 415, in
# _handle_read
# data = self.socket.read(self._buffer_size)
# File "c:\...\lib\ssl.py", line 869, in read
# raise ValueError("Read on closed or unwrapped SSL socket.")
orig_exc_cls = orig_exc.__class__
orig_exc_name = orig_exc_cls.__name__
msg = '{0}: {1}'.format(orig_exc_name, orig_exc)
error_name = default_error_name
if (issubclass(orig_exc_cls,
pika.exceptions.ProbableAuthenticationError)
or issubclass(orig_exc_cls,
pika.exceptions.AuthenticationError)
or issubclass(orig_exc_cls,
pika.exceptions.ProbableAccessDeniedError)):
error_name = 'Skyline.AMQPErrorFailedToLogIn'
elif issubclass(orig_exc_cls, pika.exceptions.ConsumerCancelled):
error_name = 'Skyline.AMQPErrorPerformingBasicConsume'
elif issubclass(orig_exc_cls, pika.exceptions.UnroutableError):
error_name = 'Skyline.AMQPErrorPublishingMessage'
if ((channel and channel.is_closed) or # pylint: disable=too-many-boolean-expressions
not self._connection or self._connection.is_closed or
issubclass(orig_exc_cls, pika.exceptions.ChannelClosed) or
issubclass(orig_exc_cls, pika.exceptions.ConnectionClosed)
or issubclass(orig_exc_cls, AttributeError) or issubclass(
orig_exc_cls, ValueError)):
self._connected = False
if self._auto_reconnect:
# Kick off the auto-reconnect functionality.
self._connection_close_event.set()
raise SystemLinkException.from_name(error_name, info=msg)