def _transform(self, obj):
# Note: This check to detect an IonObject is a bit risky (only type_)
if isinstance(obj, dict) and "type_" in obj:
objc = obj
otype = objc['type_'].encode('ascii') # Correct?
# don't supply a dict - we want the object to initialize with all its defaults intact,
# which preserves things like IonEnumObject and invokes the setattr behavior we want there.
ion_obj = self._obj_registry.new(otype)
# get outdated attributes in data that are not defined in the current schema
extra_attributes = objc.viewkeys() - ion_obj._schema.viewkeys() - BUILT_IN_ATTRS
for extra in extra_attributes:
objc.pop(extra)
log.info('discard %s not in current schema' % extra)
for k, v in objc.iteritems():
# unicode translate to utf8
if isinstance(v, unicode):
v = str(v.encode('utf8'))
# CouchDB adds _attachments and puts metadata in it
# in pyon metadata is in the document
# so we discard _attachments while transforming between the two
if k not in ("type_", "_attachments", "_conflicts"):
setattr(ion_obj, k, v)
if k == "_conflicts":
log.warn("CouchDB conflict detected for ID=%S (ignored): %s", obj.get('_id', None), v)
return ion_obj
return obj
def assertNoParticleRegression(self, filepath, data_handler):
"""
Compares particles with previous run. If no YAML file exists, creates one.
:param filepath: fully qualified name of the input file (that was parsed)
:param data_handler: ParticleDataHandler returned from parse()
:return:
"""
yaml_file = os.path.splitext(filepath)[0] + '.yml'
particles = data_handler._samples
if os.path.isfile(yaml_file):
with open(yaml_file, 'r') as stream:
prev_particles = yaml.load(stream)
# particle key names should match
self.assertListEqual(sorted(prev_particles.keys()),
sorted(particles.keys()))
# compare number of samples across one of the particle keys
for p in prev_particles.keys():
log.debug('%s: %d %d', p, len(prev_particles[p]),
len(particles[p]))
self.assertEqual(len(prev_particles[p]), len(particles[p]))
else:
with open(yaml_file, 'w') as stream:
log.warn(
'creating yaml output file for regression testing - commit %s',
yaml_file)
yaml.dump(particles, stream, default_flow_style=False)
def getpid(self):
"""
Get the pid of the current running process and ensure that it is running.
@returns the pid of the driver process if it is running, otherwise None
"""
if self._driver_process:
if self.poll():
return self._driver_process.pid
else:
log.warn("Driver process found, but poll failed for pid %s" % self._driver_process.pid)
else:
return None
def getpid(self):
"""
Get the pid of the current running process and ensure that it is running.
@returns the pid of the driver process if it is running, otherwise None
"""
if self._driver_process:
if self.poll():
return self._driver_process.pid
else:
log.warn("Driver process found, but poll failed for pid %s" %
self._driver_process.pid)
else:
return None
def poll(self):
"""
Check to see if the port agent process is alive.
@return true if process is running, false otherwise
"""
if not self._pid:
return False
try:
os.kill(self._pid, 0)
except OSError, e:
log.warn("Could not send a signal to the driver, pid: %s" % self._pid)
return False
def __str__(self):
ds = str(self.__dict__)
try:
# Remove the type_ from the dict str - cheaper this way than copying the dict
typeidx = ds.find("'type_': '")
if typeidx:
endidx = ds.find("'", typeidx + 10)
if ds[typeidx - 2] == ",":
typeidx -= 2
ds = ds[:typeidx] + ds[endidx + 1:]
except Exception as ex:
log.warn("Could not create IonObject __str__ representation")
# This is a more eye pleasing variant but does not eval
return "%s(%s)" % (self.__class__.__name__, ds)
def __str__(self):
ds = str(self.__dict__)
try:
# Remove the type_ from the dict str - cheaper this way than copying the dict
typeidx = ds.find("'type_': '")
if typeidx:
endidx = ds.find("'", typeidx+10)
if ds[typeidx-2] == ",":
typeidx -= 2
ds = ds[:typeidx] + ds[endidx+1:]
except Exception as ex:
log.warn("Could not create IonObject __str__ representation")
# This is a more eye pleasing variant but does not eval
return "%s(%s)" % (self.__class__.__name__, ds)
def poll(self):
"""
Check to see if the port agent process is alive.
@return true if process is running, false otherwise
"""
if not self._pid:
return False
try:
os.kill(self._pid, 0)
except OSError, e:
log.warn("Could not send a signal to the driver, pid: %s" %
self._pid)
return False
def poll(self):
"""
Check to see if the driver process is alive.
@return true if driver process is running, false otherwise
"""
# The Popen.poll() doesn't seem to be returning reliable results. Sending a signal 0 to the process might be
# more reliable.
if not self._driver_process:
return False
try:
os.kill(self._driver_process.pid, 0)
except OSError, e:
log.warn("Could not send a signal to the driver, pid: %s" % self._driver_process.pid)
return False
def poll(self):
"""
Check to see if the driver process is alive.
@return true if driver process is running, false otherwise
"""
# The Popen.poll() doesn't seem to be returning reliable results. Sending a signal 0 to the process might be
# more reliable.
if not self._driver_process:
return False
try:
os.kill(self._driver_process.pid, 0)
except OSError, e:
log.warn("Could not send a signal to the driver, pid: %s" %
self._driver_process.pid)
return False
def _read_pid(self):
pid_file = PID_FILE % (PROCESS_BASE_DIR, self._command_port)
start_time = time.time()
boo = 0
log.debug("read pid file: " + pid_file)
while (start_time + DEFAULT_TIMEOUT > time.time()):
try:
file = open(pid_file)
pid = file.read().strip('\0\n\r')
if (pid):
int(pid)
log.info("port agent pid: [%s]" % (pid))
return int(pid)
except ValueError, e:
log.warn("Failed to convert %s to an int '%s" % (pid, e))
break
except:
def _read_pid(self):
pid_file = PID_FILE % (PROCESS_BASE_DIR, self._command_port)
start_time = time.time()
boo = 0;
log.debug("read pid file: " + pid_file)
while(start_time + DEFAULT_TIMEOUT > time.time()):
try:
file = open(pid_file)
pid = file.read().strip('\0\n\r')
if(pid):
int(pid)
log.info("port agent pid: [%s]" % (pid))
return int(pid)
except ValueError, e:
log.warn("Failed to convert %s to an int '%s" % (pid, e) )
break
except:
def get_method_arguments(module, method_name, **kwargs):
"""
Returns a dict of the allowable method parameters
@param module:
@param method_name:
@param kwargs:
@return:
"""
param_dict = {}
if hasattr(module,method_name):
try:
#This will fail running unit tests with mock objects - BOO!
method_args = inspect.getargspec(getattr(module,method_name))
for arg in method_args[0]:
if kwargs.has_key(arg):
param_dict[arg] = kwargs[arg]
except Exception, e:
#Log a warning and simply return an empty dict
log.warn('Cannot determine the arguments for method: %s in module: %s: %s',module, method_name, e.message )
def is_service_available(self, service_name, local_rr_only=False):
try:
service_resource = None
#from pyon.core.bootstrap import container_instance
from mi.core.bootstrap import container_instance
from interface.objects import ServiceStateEnum
# Use container direct RR connection if available, otherwise use messaging to the RR service
if hasattr(container_instance, 'has_capability') and container_instance.has_capability('RESOURCE_REGISTRY'):
service_resource, _ = container_instance.resource_registry.find_resources(restype='Service', name=service_name)
elif not local_rr_only:
from interface.services.coi.iresource_registry_service import ResourceRegistryServiceClient
rr_client = ResourceRegistryServiceClient(container_instance.node)
service_resource, _ = rr_client.find_resources(restype='Service', name=service_name)
else:
log.warn("is_service_available(%s) - No RR connection" % service_name)
# The service is available only of there is a single RR object for it and it is in one of these states:
if service_resource and len(service_resource) > 1:
log.warn("is_service_available(%s) - Found multiple service instances: %s", service_name, service_resource)
# MM 2013-08-17: Added PENDING, because this means service will be there shortly
if service_resource and service_resource[0].state in (ServiceStateEnum.READY, ServiceStateEnum.STEADY, ServiceStateEnum.PENDING):
return True
elif service_resource:
log.warn("is_service_available(%s) - Service resource in invalid state", service_resource)
return False
except Exception as ex:
return False
def memory_usage(self):
"""
Get the current memory usage for the current driver process.
@returns memory usage in KB of the current driver process
"""
driver_pid = self.getpid()
if not driver_pid:
log.warn("no process running")
return 0
ps_process = subprocess.Popen(["ps", "-o rss,pid", "-p %s" % self.getpid()], stdout=subprocess.PIPE)
ps_process.poll()
usage = 0
for line in ps_process.stdout:
if not line.strip().startswith('RSS'):
try:
fields = line.split()
pid = int(fields[1])
if pid == driver_pid:
usage = int(fields[0])
except:
log.warn("Failed to parse output for memory usage: %s" % line)
usage = 0
if usage:
log.info("process memory usage: %dk" % usage)
else:
log.warn("process not running")
return usage
def memory_usage(self):
"""
Get the current memory usage for the current driver process.
@returns memory usage in KB of the current driver process
"""
driver_pid = self.getpid()
if not driver_pid:
log.warn("no process running")
return 0
ps_process = subprocess.Popen(
["ps", "-o rss,pid", "-p %s" % self.getpid()],
stdout=subprocess.PIPE)
retcode = ps_process.poll()
usage = 0
for line in ps_process.stdout:
if not line.strip().startswith('RSS'):
try:
fields = line.split()
pid = int(fields[1])
if pid == driver_pid:
usage = int(fields[0])
except:
log.warn("Failed to parse output for memory usage: %s" %
line)
usage = 0
if usage:
log.info("process memory usage: %dk" % usage)
else:
log.warn("process not running")
return usage
def assertNoParticleRegression(self, filepath, data_handler):
"""
Compares particles with previous run. If no YAML file exists, creates one.
:param filepath: fully qualified name of the input file (that was parsed)
:param data_handler: ParticleDataHandler returned from parse()
:return:
"""
yaml_file = os.path.splitext(filepath)[0] + '.yml'
particles = data_handler._samples
if os.path.isfile(yaml_file):
with open(yaml_file, 'r') as stream:
prev_particles = yaml.load(stream)
# particle key names should match
self.assertListEqual(sorted(prev_particles.keys()), sorted(particles.keys()))
# compare number of samples across one of the particle keys
for p in prev_particles.keys():
log.debug('%s: %d %d', p, len(prev_particles[p]), len(particles[p]))
self.assertEqual(len(prev_particles[p]), len(particles[p]))
else:
with open(yaml_file, 'w') as stream:
log.warn('creating yaml output file for regression testing - commit %s', yaml_file)
yaml.dump(particles, stream, default_flow_style=False)
def get_method_arguments(module, method_name, **kwargs):
"""
Returns a dict of the allowable method parameters
@param module:
@param method_name:
@param kwargs:
@return:
"""
param_dict = {}
if hasattr(module, method_name):
try:
#This will fail running unit tests with mock objects - BOO!
method_args = inspect.getargspec(getattr(module, method_name))
for arg in method_args[0]:
if kwargs.has_key(arg):
param_dict[arg] = kwargs[arg]
except Exception, e:
#Log a warning and simply return an empty dict
log.warn(
'Cannot determine the arguments for method: %s in module: %s: %s',
module, method_name, e.message)
def stop(self):
log.info('Stop port agent')
# When calling stop, IMS grabs a new port agent process via PortAgentProcess.get_process
# So self._pid is None and needs to be initialized
pid_file = PID_FILE % (PROCESS_BASE_DIR, self._command_port)
try:
with open(pid_file, 'r') as f:
pid = f.read().strip('\0\n\4')
if pid:
try:
self._pid = int(pid)
except ValueError:
pass
except IOError:
log.exception('Port agent pid file not found!')
command_line = [self._binary_path]
command_line.append("-c")
command_line.append(self._tmp_config.name)
command_line.append("-k")
command_line.append("-p")
command_line.append("%s" % (self._command_port))
self.run_command(command_line)
timeout = Timeout(5)
timeout.start()
try:
while self.poll():
log.warn('WAITING HERE with pid %s' % self._pid)
gevent.sleep(1)
except Timeout, t:
log.error(
'Timed out waiting for pagent to die. Going in for kill.')
os.kill(self._pid, signal.SIGKILL)
def _transform(self, obj):
# Note: This check to detect an IonObject is a bit risky (only type_)
if isinstance(obj, dict) and "type_" in obj:
objc = obj
otype = objc['type_'].encode('ascii') # Correct?
# don't supply a dict - we want the object to initialize with all its defaults intact,
# which preserves things like IonEnumObject and invokes the setattr behavior we want there.
ion_obj = self._obj_registry.new(otype)
# get outdated attributes in data that are not defined in the current schema
extra_attributes = objc.viewkeys() - ion_obj._schema.viewkeys(
) - BUILT_IN_ATTRS
for extra in extra_attributes:
objc.pop(extra)
log.info('discard %s not in current schema' % extra)
for k, v in objc.iteritems():
# unicode translate to utf8
if isinstance(v, unicode):
v = str(v.encode('utf8'))
# CouchDB adds _attachments and puts metadata in it
# in pyon metadata is in the document
# so we discard _attachments while transforming between the two
if k not in ("type_", "_attachments", "_conflicts"):
setattr(ion_obj, k, v)
if k == "_conflicts":
log.warn(
"CouchDB conflict detected for ID=%S (ignored): %s",
obj.get('_id', None), v)
return ion_obj
return obj
def stop(self):
log.info('Stop port agent')
# When calling stop, IMS grabs a new port agent process via PortAgentProcess.get_process
# So self._pid is None and needs to be initialized
pid_file = PID_FILE % (PROCESS_BASE_DIR, self._command_port)
try:
with open(pid_file, 'r') as f:
pid = f.read().strip('\0\n\4')
if pid:
try:
self._pid = int(pid)
except ValueError:
pass
except IOError:
log.exception('Port agent pid file not found!')
command_line = [ self._binary_path ]
command_line.append("-c")
command_line.append(self._tmp_config.name);
command_line.append("-k")
command_line.append("-p")
command_line.append("%s" % (self._command_port));
self.run_command(command_line);
timeout = Timeout(5)
timeout.start()
try:
while self.poll():
log.warn('WAITING HERE with pid %s' % self._pid)
gevent.sleep(1)
except Timeout, t:
log.error('Timed out waiting for pagent to die. Going in for kill.')
os.kill(self._pid, signal.SIGKILL)
def is_service_available(self, service_name, local_rr_only=False):
try:
service_resource = None
#from pyon.core.bootstrap import container_instance
from mi.core.bootstrap import container_instance
from interface.objects import ServiceStateEnum
# Use container direct RR connection if available, otherwise use messaging to the RR service
if hasattr(container_instance,
'has_capability') and container_instance.has_capability(
'RESOURCE_REGISTRY'):
service_resource, _ = container_instance.resource_registry.find_resources(
restype='Service', name=service_name)
elif not local_rr_only:
from interface.services.coi.iresource_registry_service import ResourceRegistryServiceClient
rr_client = ResourceRegistryServiceClient(
container_instance.node)
service_resource, _ = rr_client.find_resources(
restype='Service', name=service_name)
else:
log.warn("is_service_available(%s) - No RR connection" %
service_name)
# The service is available only of there is a single RR object for it and it is in one of these states:
if service_resource and len(service_resource) > 1:
log.warn(
"is_service_available(%s) - Found multiple service instances: %s",
service_name, service_resource)
# MM 2013-08-17: Added PENDING, because this means service will be there shortly
if service_resource and service_resource[0].state in (
ServiceStateEnum.READY, ServiceStateEnum.STEADY,
ServiceStateEnum.PENDING):
return True
elif service_resource:
log.warn(
"is_service_available(%s) - Service resource in invalid state",
service_resource)
return False
except Exception as ex:
return False
def _validate(self):
"""
Compare fields to the schema and raise AttributeError if mismatched.
Named _validate instead of validate because the data may have a field named "validate".
"""
fields, schema = self.__dict__, self._schema
# Check for extra fields not defined in the schema
extra_fields = fields.viewkeys() - schema.viewkeys() - BUILT_IN_ATTRS
if len(extra_fields) > 0:
raise AttributeError('Fields found that are not in the schema: %r' % (list(extra_fields)))
required_decorator = 'Required'
content_type_decorator = 'ContentType'
content_count_decorator = 'ContentCount'
value_range_decorator = 'ValueRange'
value_pattern_decorator = 'ValuePattern'
# Check required field criteria met
for key in schema:
if 'decorators' in schema[key] and required_decorator in schema[key]['decorators']:
if not key in fields or fields[key] is None:
raise AttributeError('Required value "%s" not set' % key)
# Check each attribute
for key in fields.iterkeys():
if key in BUILT_IN_ATTRS:
continue
schema_val = schema[key]
# Correct any float or long types that got downgraded to int
if isinstance(fields[key], int):
if schema_val['type'] == 'float':
fields[key] = float(fields[key])
elif schema_val['type'] == 'long':
fields[key] = long(fields[key])
# argh, annoying work around for OrderedDict vs dict issue
if type(fields[key]) == dict and schema_val['type'] == 'OrderedDict':
fields[key] = OrderedDict(fields[key])
# Basic type checking
field_val = fields[key]
if type(field_val).__name__ != schema_val['type']:
# if the schema doesn't define a type, we can't very well validate it
if schema_val['type'] == 'NoneType':
continue
# Allow unicode instead of str. This may be too lenient.
if schema_val['type'] == 'str' and type(field_val).__name__ == 'unicode':
continue
# Already checked for required above. Assume optional and continue
if field_val is None:
continue
# Allow unicode instead of str. This may be too lenient.
if schema_val['type'] == 'str' and type(field_val).__name__ == 'unicode':
continue
# IonObjects are ok for dict fields too!
if isinstance(field_val, IonObjectBase) and schema_val['type'] == 'OrderedDict':
continue
if not key in fields or fields[key] is None:
raise AttributeError('Required value "%s" not set' % key)
# Check for inheritance
if self.check_inheritance_chain(type(field_val), schema_val['type']):
continue
# Check enum types
from pyon.core.registry import enum_classes
if isinstance(field_val, int) and schema_val['type'] in enum_classes:
if field_val not in enum_classes(schema_val['type'])._str_map:
raise AttributeError('Invalid enum value "%d" for field "%s.%s", should be between 1 and %d' %
(fields[key], type(self).__name__, key, len(enum_classes(schema_val['type'])._str_map)))
else:
continue
if type(field_val) == tuple and schema_val['type'] == 'list':
continue
if isinstance(field_val, IonObjectBase) and schema_val['type'] == 'dict':
log.warn('TODO: Please convert generic dict attribute type to abstract type for field "%s.%s"' % (type(self).__name__, key))
continue
# Special case check for ION object being passed where default type is dict or str
if 'decorators' in schema_val:
if content_type_decorator in schema_val['decorators']:
if isinstance(field_val, IonObjectBase) and schema_val['type'] == 'dict' or schema_val['type'] == 'str':
self.check_content(key, field_val, schema_val['decorators'][content_type_decorator])
continue
raise AttributeError('Invalid type "%s" for field "%s.%s", should be "%s"' %
(type(fields[key]), type(self).__name__, key, schema_val['type']))
if type(field_val).__name__ == 'str':
if value_pattern_decorator in schema_val['decorators']:
self.check_string_pattern_match(key, field_val, schema_val['decorators'][value_pattern_decorator])
if type(field_val).__name__ in ['int', 'float', 'long']:
if value_range_decorator in schema_val['decorators']:
self.check_numeric_value_range(key, field_val, schema_val['decorators'][value_range_decorator])
if 'decorators' in schema_val:
if content_type_decorator in schema_val['decorators']:
if schema_val['type'] == 'list':
self.check_collection_content(key, field_val, schema_val['decorators'][content_type_decorator])
elif schema_val['type'] == 'dict' or schema_val['type'] == 'OrderedDict':
self.check_collection_content(key, field_val.values(), schema_val['decorators'][content_type_decorator])
else:
self.check_content(key, field_val, schema_val['decorators'][content_type_decorator])
if content_count_decorator in schema_val['decorators']:
if schema_val['type'] == 'list':
self.check_collection_length(key, field_val, schema_val['decorators'][content_count_decorator])
if schema_val['type'] == 'dict' or schema_val['type'] == 'OrderedDict':
self.check_collection_length(key, field_val.values(), schema_val['decorators'][content_count_decorator])
if isinstance(field_val, IonObjectBase):
field_val._validate()
# Next validate only IonObjects found in child collections.
# Note that this is non-recursive; only for first-level collections.
elif isinstance(field_val, Mapping):
for subkey in field_val:
subval = field_val[subkey]
if isinstance(subval, IonObjectBase):
subval._validate()
elif isinstance(field_val, Iterable):
for subval in field_val:
if isinstance(subval, IonObjectBase):
subval._validate()
boo = 0
log.debug("read pid file: " + pid_file)
while (start_time + DEFAULT_TIMEOUT > time.time()):
try:
file = open(pid_file)
pid = file.read().strip('\0\n\r')
if (pid):
int(pid)
log.info("port agent pid: [%s]" % (pid))
return int(pid)
except ValueError, e:
log.warn("Failed to convert %s to an int '%s" % (pid, e))
break
except:
log.warn("Failed to open pid file: %s" % (pid_file))
gevent.sleep(1)
log.error("port agent startup failed")
return None
def _read_config(self):
self._tmp_config.seek(0)
return "".join(self._tmp_config.readlines())
def stop(self):
log.info('Stop port agent')
# When calling stop, IMS grabs a new port agent process via PortAgentProcess.get_process
# So self._pid is None and needs to be initialized
pid_file = PID_FILE % (PROCESS_BASE_DIR, self._command_port)
def _validate(self):
"""
Compare fields to the schema and raise AttributeError if mismatched.
Named _validate instead of validate because the data may have a field named "validate".
"""
fields, schema = self.__dict__, self._schema
# Check for extra fields not defined in the schema
extra_fields = fields.viewkeys() - schema.viewkeys() - BUILT_IN_ATTRS
if len(extra_fields) > 0:
raise AttributeError(
'Fields found that are not in the schema: %r' %
(list(extra_fields)))
required_decorator = 'Required'
content_type_decorator = 'ContentType'
content_count_decorator = 'ContentCount'
value_range_decorator = 'ValueRange'
value_pattern_decorator = 'ValuePattern'
# Check required field criteria met
for key in schema:
if 'decorators' in schema[key] and required_decorator in schema[
key]['decorators']:
if not key in fields or fields[key] is None:
raise AttributeError('Required value "%s" not set' % key)
# Check each attribute
for key in fields.iterkeys():
if key in BUILT_IN_ATTRS:
continue
schema_val = schema[key]
# Correct any float or long types that got downgraded to int
if isinstance(fields[key], int):
if schema_val['type'] == 'float':
fields[key] = float(fields[key])
elif schema_val['type'] == 'long':
fields[key] = long(fields[key])
# argh, annoying work around for OrderedDict vs dict issue
if type(fields[key]
) == dict and schema_val['type'] == 'OrderedDict':
fields[key] = OrderedDict(fields[key])
# Basic type checking
field_val = fields[key]
if type(field_val).__name__ != schema_val['type']:
# if the schema doesn't define a type, we can't very well validate it
if schema_val['type'] == 'NoneType':
continue
# Allow unicode instead of str. This may be too lenient.
if schema_val['type'] == 'str' and type(
field_val).__name__ == 'unicode':
continue
# Already checked for required above. Assume optional and continue
if field_val is None:
continue
# Allow unicode instead of str. This may be too lenient.
if schema_val['type'] == 'str' and type(
field_val).__name__ == 'unicode':
continue
# IonObjects are ok for dict fields too!
if isinstance(
field_val,
IonObjectBase) and schema_val['type'] == 'OrderedDict':
continue
if not key in fields or fields[key] is None:
raise AttributeError('Required value "%s" not set' % key)
# Check for inheritance
if self.check_inheritance_chain(type(field_val),
schema_val['type']):
continue
# Check enum types
from pyon.core.registry import enum_classes
if isinstance(field_val,
int) and schema_val['type'] in enum_classes:
if field_val not in enum_classes(
schema_val['type'])._str_map:
raise AttributeError(
'Invalid enum value "%d" for field "%s.%s", should be between 1 and %d'
% (fields[key], type(self).__name__, key,
len(enum_classes(schema_val['type'])._str_map)))
else:
continue
if type(field_val) == tuple and schema_val['type'] == 'list':
continue
if isinstance(field_val,
IonObjectBase) and schema_val['type'] == 'dict':
log.warn(
'TODO: Please convert generic dict attribute type to abstract type for field "%s.%s"'
% (type(self).__name__, key))
continue
# Special case check for ION object being passed where default type is dict or str
if 'decorators' in schema_val:
if content_type_decorator in schema_val['decorators']:
if isinstance(
field_val, IonObjectBase
) and schema_val['type'] == 'dict' or schema_val[
'type'] == 'str':
self.check_content(
key, field_val, schema_val['decorators']
[content_type_decorator])
continue
raise AttributeError(
'Invalid type "%s" for field "%s.%s", should be "%s"' %
(type(fields[key]), type(self).__name__, key,
schema_val['type']))
if type(field_val).__name__ == 'str':
if value_pattern_decorator in schema_val['decorators']:
self.check_string_pattern_match(
key, field_val,
schema_val['decorators'][value_pattern_decorator])
if type(field_val).__name__ in ['int', 'float', 'long']:
if value_range_decorator in schema_val['decorators']:
self.check_numeric_value_range(
key, field_val,
schema_val['decorators'][value_range_decorator])
if 'decorators' in schema_val:
if content_type_decorator in schema_val['decorators']:
if schema_val['type'] == 'list':
self.check_collection_content(
key, field_val,
schema_val['decorators'][content_type_decorator])
elif schema_val['type'] == 'dict' or schema_val[
'type'] == 'OrderedDict':
self.check_collection_content(
key, field_val.values(),
schema_val['decorators'][content_type_decorator])
else:
self.check_content(
key, field_val,
schema_val['decorators'][content_type_decorator])
if content_count_decorator in schema_val['decorators']:
if schema_val['type'] == 'list':
self.check_collection_length(
key, field_val,
schema_val['decorators'][content_count_decorator])
if schema_val['type'] == 'dict' or schema_val[
'type'] == 'OrderedDict':
self.check_collection_length(
key, field_val.values(),
schema_val['decorators'][content_count_decorator])
if isinstance(field_val, IonObjectBase):
field_val._validate()
# Next validate only IonObjects found in child collections.
# Note that this is non-recursive; only for first-level collections.
elif isinstance(field_val, Mapping):
for subkey in field_val:
subval = field_val[subkey]
if isinstance(subval, IonObjectBase):
subval._validate()
elif isinstance(field_val, Iterable):
for subval in field_val:
if isinstance(subval, IonObjectBase):
subval._validate()
def _find_associated_resources(self, resource, association_predicate, target_type=None, res_type=None):
"""
Returns a list of tuples (target_id, Association) based on associations for the given
resource (object), predicate and optional target object type.
This method figures out appropriate association lookup based on the predicate definitions
@param resource Either a resource object or a resource id (then res_type is needed)
"""
assoc_list = []
res_type = res_type or resource.type_
resource_id = resource if type(resource) is str else resource._id
if target_type and type(target_type) not in (list, tuple): # None and empty str left alone
target_type = [target_type]
assoc_direction = ""
if association_predicate and (association_predicate.endswith(">") or association_predicate.endswith("<")):
assoc_direction = association_predicate[-1]
association_predicate = association_predicate[:-1]
# First validate the association predicate
pred = Predicates[association_predicate]
if not pred:
return [] # Unknown association type so return empty list
# Need to check through all of these in this order to account for specific vs base class inclusions
# @ TODO: This algorithm and the order is fragile
if self.is_predicate_association(pred, "domain", res_type) and self._allow_direction(assoc_direction, ">"):
# Case 1: Association in schema with current exact resource type as SUBJECT
assoc_list.extend(self._find_associations(resource_id, association_predicate, target_type, backward=False))
# If no objects were found, try finding as subjects just in case (unless direction requested)
if not assoc_list and not assoc_direction:
assoc_list.extend(
self._find_associations(resource_id, association_predicate, target_type, backward=True)
)
elif self.is_predicate_association(pred, "range", res_type) and self._allow_direction(assoc_direction, "<"):
# Case 2: Association in schema with current exact resource type as OBJECT
assoc_list.extend(self._find_associations(resource_id, association_predicate, target_type, backward=True))
elif self.is_predicate_association_extension(pred, "domain", res_type) and self._allow_direction(
assoc_direction, ">"
):
# Case 3: Association in schema with base type of current resource type as SUBJECT
assoc_list.extend(self._find_associations(resource_id, association_predicate, target_type, backward=False))
# If no objects were found, try finding as subjects just in case.
if not assoc_list and not assoc_direction:
assoc_list.extend(
self._find_associations(resource_id, association_predicate, target_type, backward=True)
)
elif self.is_predicate_association_extension(pred, "range", res_type) and self._allow_direction(
assoc_direction, "<"
):
# Case 4: Association in schema with base type of current resource type as OBJECT
assoc_list.extend(self._find_associations(resource_id, association_predicate, target_type, backward=True))
else:
log.warn("Cannot handle association predicate %s for resource type %s", association_predicate, res_type)
return assoc_list
boo = 0;
log.debug("read pid file: " + pid_file)
while(start_time + DEFAULT_TIMEOUT > time.time()):
try:
file = open(pid_file)
pid = file.read().strip('\0\n\r')
if(pid):
int(pid)
log.info("port agent pid: [%s]" % (pid))
return int(pid)
except ValueError, e:
log.warn("Failed to convert %s to an int '%s" % (pid, e) )
break
except:
log.warn("Failed to open pid file: %s" % (pid_file))
gevent.sleep(1);
log.error("port agent startup failed");
return None;
def _read_config(self):
self._tmp_config.seek(0);
return "".join(self._tmp_config.readlines());
def stop(self):
log.info('Stop port agent')
# When calling stop, IMS grabs a new port agent process via PortAgentProcess.get_process
# So self._pid is None and needs to be initialized
def set_object_field_values(self, obj, resource, ext_exclude, **kwargs):
"""
Iterate through all fields of the given object and set values according
to the field type and decorator definition in the object type schema.
"""
# Step 1: Determine needs to fill fields with resource objects.
field_needs = [] # Fields that need to be set in a subsequent step
resource_needs = set() # Resources to read by id based on needs
assoc_needs = set() # Compound associations to follow
final_target_types = {} # Keeps track of what resource type filter is desired
for field in obj._schema:
# Skip any fields that were specifically to be excluded
if ext_exclude is not None and field in ext_exclude:
continue
# Iterate over all of the decorators for the field
for decorator in obj._schema[field]["decorators"]:
field_start_time = time.time()
# Field gets value from method or service call (local to current executing process)
if decorator == "Method":
deco_value = obj.get_decorator_value(field, decorator)
method_name = deco_value if deco_value else "get_" + field
ret_val = self.execute_method_with_resource(resource._id, method_name, **kwargs)
if ret_val is not None:
setattr(obj, field, ret_val)
elif decorator == "ServiceRequest":
deco_value = obj.get_decorator_value(field, decorator)
if obj._schema[field]["type"] != "ServiceRequest":
log.error("The field %s is an incorrect type for a ServiceRequest decorator.", field)
continue
method_name = deco_value if deco_value else "get_" + field
if method_name.find(".") == -1:
raise Inconsistent(
"The field %s decorated as a ServiceRequest only supports remote operations.", field
)
service_client, operation = get_remote_info(self, method_name)
rmi_call = method_name.split(".")
parms = {"resource_id": resource._id}
parms.update(get_method_arguments(service_client, operation, **kwargs))
ret_val = IonObject(
OT.ServiceRequest,
service_name=rmi_call[0],
service_operation=operation,
request_parameters=parms,
)
setattr(obj, field, ret_val)
# Fill field based on compound association chains. Results in nested lists of resource objects
elif self.is_compound_association(decorator):
target_type = obj.get_decorator_value(field, decorator)
if (
target_type and "," in target_type
): # Can specify multiple type filters, only handles two levels for now
target_type, final_target_type = target_type.split(",")
final_target_types[field] = final_target_type # Keep track for later
predicates = self.get_compound_association_predicates(decorator)
assoc_list = self._find_associated_resources(resource, predicates[0], target_type)
field_needs.append((field, "A", (assoc_list, predicates)))
for target_id, assoc in assoc_list:
assoc_needs.add((target_id, predicates[1]))
# Fill field based on association with list of resource objects
elif self.is_association_predicate(decorator):
target_type = obj.get_decorator_value(field, decorator)
if target_type and "," in target_type: # Can specify list of target types
target_type = target_type.split(",")
assoc_list = self._find_associated_resources(resource, decorator, target_type)
if obj._schema[field]["type"] == "list":
if assoc_list:
field_needs.append((field, "L", assoc_list))
[resource_needs.add(target_id) for target_id, assoc in assoc_list]
elif obj._schema[field]["type"] == "int":
setattr(obj, field, len(assoc_list))
else: # Can be nested object or None
if assoc_list:
first_assoc = assoc_list[0]
if len(assoc_list) != 1:
# WARNING: Swallow random further objects here!
log.warn(
"Extended object field %s uses only 1 of %d associated resources",
field,
len(assoc_list),
)
field_needs.append((field, "O", first_assoc))
resource_needs.add(first_assoc[0])
else:
setattr(obj, field, None)
else:
log.debug("Unknown decorator %s for field %s of resource %s", decorator, field, resource._id)
field_stop_time = time.time()
# log.debug("Time to process field %s(%s) %f secs", field, decorator, field_stop_time - field_start_time)
# field_needs contains a list of what's needed to load in next step (different cases)
if not field_needs:
return
# Step 2: Read second level of compound associations as needed
# @TODO Can only do 2 level compounds for now. Make recursive someday
if assoc_needs:
assocs = self._rr.find_associations(anyside=list(assoc_needs), id_only=False)
self._add_associations(assocs)
# Determine resource ids to read for compound associations
for field, need_type, needs in field_needs:
if need_type == "A":
assoc_list, predicates = needs
for target_id, assoc in assoc_list:
res_type = assoc.ot if target_id == assoc.o else assoc.st
assoc_list1 = self._find_associated_resources(target_id, predicates[1], None, res_type)
for target_id1, assoc1 in assoc_list1:
resource_needs.add(target_id1)
# Step 3: Read resource objects based on needs
res_list = self._rr.read_mult(list(resource_needs))
res_objs = dict(zip(resource_needs, res_list))
# Step 4: Set fields to loaded resource objects based on type
for field, need_type, needs in field_needs:
if need_type == "L": # case list
obj_list = [res_objs[target_id] for target_id, assoc in needs]
setattr(obj, field, obj_list)
elif need_type == "O": # case nested object
target_id, assoc = needs
setattr(obj, field, res_objs[target_id])
elif need_type == "A": # case compound
assoc_list, predicates = needs
obj_list = []
for target_id, assoc in assoc_list:
res_type = assoc.ot if target_id == assoc.o else assoc.st
assoc_list1 = self._find_associated_resources(target_id, predicates[1], None, res_type)
obj_list.append([res_objs[target_id1] for target_id1, assoc1 in assoc_list1])
# Filter the list to remove objects that might match the current resource type
result_obj_list = []
for ol_nested in obj_list:
if ol_nested:
# Only get the object types which don't match the current resource type and may match a final type
if final_target_types.has_key(field):
result_obj_list.extend(
[
target_obj
for target_obj in ol_nested
if (
target_obj.type_ != resource.type_
and final_target_types[field] in target_obj._get_extends()
)
]
)
else:
result_obj_list.extend(
[target_obj for target_obj in ol_nested if (target_obj.type_ != resource.type_)]
)
if obj._schema[field]["type"] == "list":
if result_obj_list:
setattr(obj, field, result_obj_list)
elif obj._schema[field]["type"] == "int":
setattr(obj, field, len(result_obj_list))
else:
if result_obj_list:
if len(result_obj_list) != 1:
# WARNING: Swallow random further objects here!
log.warn(
"Extended object field %s uses only 1 of %d compound associated resources",
field,
len(result_obj_list),
)
setattr(obj, field, result_obj_list[0])
else:
setattr(obj, field, None)
