def test_cluster(self):
"""Test the '/cluster' endpoint."""
start_time = utilities.now()
# Execrcise the collector. Read data from golden files and compute
# a context graph.
ret_value = self.app.get('/cluster')
end_time = utilities.now()
result = json.loads(ret_value.data)
self.verify_resources(result, start_time, end_time)
self.assertEqual(23, self.count_relations(result, 'contains'))
self.assertEqual(3, self.count_relations(result, 'createdFrom'))
self.assertEqual(7, self.count_relations(result, 'loadBalances'))
self.assertEqual(6, self.count_relations(result, 'monitors'))
self.assertEqual(10, self.count_relations(result, 'runs'))
# Verify that all relations contain a timestamp in the range
# [start_time, end_time].
self.assertTrue(isinstance(result.get('relations'), types.ListType))
for r in result['relations']:
self.assertTrue(isinstance(r, types.DictType))
timestamp = r.get('timestamp')
self.assertTrue(utilities.valid_string(timestamp))
self.assertTrue(start_time <= timestamp <= end_time)
# The overall timestamp must be in the expected range.
self.assertTrue(utilities.valid_string(result.get('timestamp')))
self.assertTrue(start_time <= result['timestamp'] <= end_time)
json_output = json.dumps(result, sort_keys=True)
self.assertEqual(2, json_output.count('"alternateLabel": '))
self.assertEqual(85, json_output.count('"createdBy": '))
def compile_light_time(device_id):
"""Process time and light."""
config = get_config_for_device(device_id)
status = get_status_for_device(device_id)
if config and status:
_now = now()
timestamp = get_timestamp(status)
light = get_light_threshold(config)
time_diff = round(get_time_diff(timestamp, _now))
analysis = analyze_time_diff(time_diff, device_id)
light_reading = get_current_light_reading(status)
else:
_now = now()
timestamp = 'I got a 401 or some other error'
light = 'I got a 401 or some other error'
time_diff = 'I got a 401 or some other error'
analysis = 'I got a 401 or some other error'
light_reading = 'I got a 401 or some other error'
# print("time: {}, light:{}, 'time_diff': {}, 'analysis': {}".format(
# timestamp,
# light,
# time_diff,
# analysis)
# )
return ({
'now': _now,
'time': timestamp,
'light': light,
'time difference': time_diff,
'analysis': analysis,
'light_reading': light_reading
})
def test_cluster(self):
"""Test the '/cluster' endpoint."""
start_time = utilities.now()
# Execrcise the collector. Read data from golden files and compute
# a context graph.
ret_value = self.app.get('/cluster')
end_time = utilities.now()
result = json.loads(ret_value.data)
self.verify_resources(result, start_time, end_time)
self.assertEqual(23, self.count_relations(result, 'contains'))
self.assertEqual(3, self.count_relations(result, 'createdFrom'))
self.assertEqual(7, self.count_relations(result, 'loadBalances'))
self.assertEqual(6, self.count_relations(result, 'monitors'))
self.assertEqual(10, self.count_relations(result, 'runs'))
# Verify that all relations contain a timestamp in the range
# [start_time, end_time].
self.assertTrue(isinstance(result.get('relations'), types.ListType))
for r in result['relations']:
self.assertTrue(isinstance(r, types.DictType))
timestamp = r.get('timestamp')
self.assertTrue(utilities.valid_string(timestamp))
self.assertTrue(start_time <= timestamp <= end_time)
# The overall timestamp must be in the expected range.
self.assertTrue(utilities.valid_string(result.get('timestamp')))
self.assertTrue(start_time <= result['timestamp'] <= end_time)
json_output = json.dumps(result, sort_keys=True)
self.assertEqual(2, json_output.count('"alternateLabel": '))
self.assertEqual(85, json_output.count('"createdBy": '))
def add_relation(self, source, target, kind, label=None, metadata=None):
"""Adds a relation to the context graph."""
assert utilities.valid_string(source) and utilities.valid_string(target)
assert utilities.valid_string(kind)
assert utilities.valid_optional_string(label)
assert (metadata is None) or isinstance(metadata, dict)
with self._lock:
# The timestamp of the relation should be inherited from the previous
# context graph.
key = (source, target, kind)
timestamp = self._previous_relations_to_timestamps.get(key)
if not utilities.valid_string(timestamp):
timestamp = utilities.now()
# Add the relation to the context graph data structure.
relation = {
'source': source,
'target': target,
'type': kind,
'timestamp': timestamp
}
self._current_relations_to_timestamps[key] = timestamp
# Add annotations as needed.
relation['annotations'] = {}
if metadata is not None:
relation['annotations']['metadata'] = copy.deep_copy(metadata)
relation['annotations']['label'] = label if label is not None else kind
self._context_relations.append(relation)
def add_relation(self, source, target, kind, label=None, metadata=None):
"""Adds a relation to the context graph."""
assert utilities.valid_string(source) and utilities.valid_string(target)
assert utilities.valid_string(kind)
assert utilities.valid_optional_string(label)
assert (metadata is None) or isinstance(metadata, types.DictType)
with self._lock:
# The timestamp of the relation should be inherited from the previous
# context graph.
key = (source, target, kind)
timestamp = self._previous_relations_to_timestamps.get(key)
if not utilities.valid_string(timestamp):
timestamp = utilities.now()
# Add the relation to the context graph data structure.
relation = {
'source': source,
'target': target,
'type': kind,
'timestamp': timestamp
}
self._current_relations_to_timestamps[key] = timestamp
# Add annotations as needed.
relation['annotations'] = {}
if metadata is not None:
relation['annotations']['metadata'] = copy.deep_copy(metadata)
relation['annotations']['label'] = label if label is not None else kind
if self._version is not None:
relation['annotations']['createdBy'] = self._version
self._context_relations.append(relation)
def monitor_handler(self, exception=None):
""" monitor handler, defines what happens when a new value arrives;
Currently a new values is added to the `values` list """
try:
if exception is None:
# if self.status == pyca.NO_ALARM:
self.last_update = self.timestr()
if (self.monitor_append): self.values.append(self.value)
if (is_debug_on()):
print utilities.now() + " ",
print "monitoring %s %s" % (self.name, self.timestr()),
print self.value
else:
print "%-30s " % (self.name), exception
except Exception, e:
print e
class User(db.Document):
username = db.StringField(db_field="u", required=True)
password = db.StringField(db_field="p", required=True)
email = db.EmailField(db_field="e", required=True)
firstname = db.StringField(db_field="fn", max_length=50)
lastname = db.StringField(db_field="ln", max_length=50)
created = db.IntField(db_field="c", default=now())
def test_resources(self):
"""Test the '/resources' endpoint."""
start_time = utilities.now()
ret_value = self.app.get('/cluster/resources')
end_time = utilities.now()
result = json.loads(ret_value.data)
self.verify_resources(result, start_time, end_time)
self.assertEqual(0, self.count_relations(result, 'contains'))
self.assertEqual(0, self.count_relations(result, 'createdFrom'))
self.assertEqual(0, self.count_relations(result, 'loadBalances'))
self.assertEqual(0, self.count_relations(result, 'monitors'))
self.assertEqual(0, self.count_relations(result, 'runs'))
# The overall timestamp must be in the expected range.
self.assertTrue(utilities.valid_string(result.get('timestamp')))
self.assertTrue(start_time <= result['timestamp'] <= end_time)
def to_context_resources(self):
"""Returns just the resources in Cluster-Insight context graph format."""
with self._lock:
resources = {
'success': True,
'timestamp': self._max_resources_timestamp(utilities.now()),
'resources': self._context_resources,
}
return resources
def to_context_resources(self):
"""Returns just the resources in Cluster-Insight context graph format."""
with self._lock:
resources = {
'success': True,
'timestamp': self._max_resources_timestamp(utilities.now()),
'resources': self._context_resources,
}
return resources
def to_context_graph(self):
"""Returns the context graph in cluster-insight context graph format."""
# return graph in Cluster-Insight context graph format.
with self._lock:
context_graph = {
'success': True,
'timestamp': self._max_relations_timestamp(
self._max_resources_timestamp(utilities.now())),
'resources': self._context_resources,
'relations': self._context_relations
}
return context_graph
def test_make_response(self):
"""Tests make_response()."""
# The timestamp of the first response is the current time.
start_time = utilities.now()
resp = utilities.make_response([], 'resources')
end_time = utilities.now()
# Note that timless_json_hash() ignores the value of the timestamp.
self.assertEqual(
utilities.timeless_json_hash(
{'success': True, 'timestamp': utilities.now(), 'resources': []}),
utilities.timeless_json_hash(resp))
self.assertTrue(start_time <= resp.get('timestamp') <= end_time)
# The timestamp of the second response is the timestamp of the container.
resp = utilities.make_response([CONTAINER], 'resources')
self.assertEqual(
utilities.timeless_json_hash(
{'success': True, 'timestamp': utilities.now(), 'resources':
[CONTAINER]}),
utilities.timeless_json_hash(resp))
self.assertEqual(CONTAINER['timestamp'], resp['timestamp'])
def _make_error(error_message):
"""Returns an error response in the Cluster-Insight context graph format.
Args:
error_message: the error message describing the failure.
Returns:
An error response in the cluster-insight context graph format.
"""
assert isinstance(error_message, types.StringTypes) and error_message
return {'success': False,
'timestamp': utilities.now(),
'error_message': error_message}
def test_timeless_json_hash(self):
"""Tests timeless_json_hash() with multiple similar and dissimilar objects.
"""
a = {'uid': 'A', 'creationTimestamp': '2015-02-20T21:39:34Z'}
# 'b1' and 'b2' differs just by the value of the 'lastHearbeatTime'
# attribute.
b1 = {'uid': 'B', 'lastHeartbeatTime': '2015-03-13T22:32:15Z'}
b2 = {'uid': 'B', 'lastHeartbeatTime': utilities.now()}
# 'c1' and 'c2' differs just by the value of the 'resourceVersion'
# attribute.
c1 = {'uid': 'C', 'resourceVersion': '13'}
c2 = {'uid': 'C', 'resourceVersion': '42'}
# 'wrapped_xxx' objects look like the objects we normally keep in the cache.
# The difference between 'wrapped_a1' and 'wrapped_a2' is the value of the
# 'timestamp' attribute.
wrapped_a1 = utilities.wrap_object(a, 'Node', 'aaa', time.time())
wrapped_a2 = utilities.wrap_object(a, 'Node', 'aaa', time.time() + 100)
# The difference between the 'wrapped_b1', 'wrapped_b2' and 'wrapped_b3'
# objects are the values of the 'timestamp' and 'lastHeartbeatTime'
# attributes.
now = time.time()
wrapped_b1 = utilities.wrap_object(b1, 'Node', 'bbb', now)
wrapped_b2 = utilities.wrap_object(b2, 'Node', 'bbb', now)
wrapped_b3 = utilities.wrap_object(b2, 'Node', 'bbb', now + 100)
# The difference between 'wrapped_c1' and 'wrapped_c2' objects are
# the values of the 'timestamp' and 'resourceVersion' attributes.
wrapped_c1 = utilities.wrap_object(c1, 'Node', 'bbb', now)
wrapped_c2 = utilities.wrap_object(c2, 'Node', 'bbb', now + 100)
self.assertEqual(utilities.timeless_json_hash(wrapped_a1),
utilities.timeless_json_hash(wrapped_a2))
self.assertEqual(utilities.timeless_json_hash(wrapped_b1),
utilities.timeless_json_hash(wrapped_b2))
self.assertEqual(utilities.timeless_json_hash(wrapped_b1),
utilities.timeless_json_hash(wrapped_b3))
self.assertEqual(utilities.timeless_json_hash(wrapped_c1),
utilities.timeless_json_hash(wrapped_c2))
# Verify that the hash values of lists of objects behaves as expected.
self.assertEqual(utilities.timeless_json_hash([wrapped_a1, wrapped_b3]),
utilities.timeless_json_hash([wrapped_a2, wrapped_b1]))
# Verify that the hash value of dissimilar objects is not equal.
self.assertTrue(utilities.timeless_json_hash(wrapped_a1) !=
utilities.timeless_json_hash(wrapped_b1))
def _make_error(error_message):
"""Returns an error response in the Cluster-Insight context graph format.
Args:
error_message: the error message describing the failure.
Returns:
An error response in the cluster-insight context graph format.
"""
assert isinstance(error_message, types.StringTypes) and error_message
return {
'success': False,
'timestamp': utilities.now(),
'error_message': error_message
}
def logprint(text,date=True,print_screen=None,newline=True):
""" appends `text` to the logfile.
Optional (booleans):
date: put time stamp (with millisecond resolution in front of the string)
print_screen: write also to screen
newline: terminate with newline the string"""
if ( (gLogf is None) or (gFilename.startswith(utilities.today()) != True) ): guessFilename()
if (config.PRINT_DATE & date):
text = "%s %s" % (utilities.now(),text)
if (newline): text += "\n"
if (print_screen is None): print_screen = config.LOGFILE_print_screen
if (print_screen):
sys.stdout.write(text)
sys.stdout.flush()
gLogf.write(text)
gLogf.flush()
def to_context_graph(self):
"""Returns the context graph in cluster-insight context graph format."""
# return graph in Cluster-Insight context graph format.
with self._lock:
context_graph = {
'success':
True,
'timestamp':
self._max_relations_timestamp(
self._max_resources_timestamp(utilities.now())),
'resources':
self._context_resources,
'relations':
self._context_relations
}
return context_graph
def max_resources_and_relations_timestamp(self):
"""Computes the maximal timestamp of all resources and relations.
Must be called while holding self._lock.
If there are no resources and no relations, return the current time.
Returns:
Maximum timestamp of all resources and relations.
"""
max_timestamp = None
for r in self._context_resources:
if (max_timestamp is None) or (r['timestamp'] > max_timestamp):
max_timestamp = r['timestamp']
for r in self._context_relations:
if (max_timestamp is None) or (r['timestamp'] > max_timestamp):
max_timestamp = r['timestamp']
return utilities.now() if max_timestamp is None else max_timestamp
def update_unit_status(status, config, time_diff, iterator):
"""Combine values with cell positions in a dictionary."""
s = types.SimpleNamespace()
s.light_threshold_status = {}
s.now_status = {}
s.report_status = {}
s.diff_status = {}
s._now = now()
s.timestamp = get_timestamp(status)
s.light_threshold = get_light_threshold(config)
s.light_threshold_status['value'] = s.light_threshold
# light_threshold_status['cell'] = "C{}".format(str(i))
s.light_threshold_status['cell'] = light_threshold_status_cell(iterator)
s.now_status['value'] = s._now
s.now_status['cell'] = time_checked_cell(iterator)
s.report_status['value'] = s.timestamp
s.report_status['cell'] = report_status_cell(iterator)
try:
s.time_diff = round(get_time_diff(s.timestamp, s._now))
except TypeError:
s.time_diff = -1
if s.time_diff > 32:
check_battery = True
else:
check_battery = False
s.check_battery = check_battery
s.pretty_time = "CHECK BATTERY" if check_battery else get_pretty_time(
s.time_diff)
s.diff_status['value'] = "{}".format(s.pretty_time)
s.diff_status['cell'] = time_since_last_report_cell(iterator)
return s
def test_timeless_json_hash(self):
"""Tests timeless_json_hash() with multiple similar and dissimilar objects.
"""
a = {'uid': 'A', 'creationTimestamp': '2015-02-20T21:39:34Z'}
# 'b1' and 'b2' differs just by the value of the 'lastProbeTime' attribute.
b1 = {'uid': 'B', 'lastProbeTime': '2015-03-13T22:32:15Z'}
b2 = {'uid': 'B', 'lastProbeTime': utilities.now()}
# 'wrapped_xxx' objects look like the objects we normally keep in the cache.
# The difference between 'wrapped_a1' and 'wrapped_a2' is the value of the
# 'timestamp' attribute.
wrapped_a1 = utilities.wrap_object(a, 'Node', 'aaa', time.time())
wrapped_a2 = utilities.wrap_object(a, 'Node', 'aaa', time.time() + 100)
# The difference between the 'wrapped_b1', 'wrapped_b2' and 'wrapped_b3'
# objects are the values of the 'timestamp' and 'lastProbeTime' attributes.
now = time.time()
wrapped_b1 = utilities.wrap_object(b1, 'Node', 'bbb', now)
wrapped_b2 = utilities.wrap_object(b2, 'Node', 'bbb', now)
wrapped_b3 = utilities.wrap_object(b2, 'Node', 'bbb', now + 100)
self.assertEqual(utilities.timeless_json_hash(wrapped_a1),
utilities.timeless_json_hash(wrapped_a2))
self.assertEqual(utilities.timeless_json_hash(wrapped_b1),
utilities.timeless_json_hash(wrapped_b2))
self.assertEqual(utilities.timeless_json_hash(wrapped_b1),
utilities.timeless_json_hash(wrapped_b3))
# Verify that the hash values of lists of objects behaves as expected.
self.assertEqual(utilities.timeless_json_hash([wrapped_a1, wrapped_b3]),
utilities.timeless_json_hash([wrapped_a2, wrapped_b1]))
# Verify that the hash value of dissimilar objects is not equal.
self.assertTrue(utilities.timeless_json_hash(wrapped_a1) !=
utilities.timeless_json_hash(wrapped_b1))
def _do_compute_graph(gs, output_format):
"""Returns the context graph in the specified format.
Args:
gs: the global state.
output_format: one of 'dot', 'context_graph', or 'resources'.
Returns:
A successful response in the specified format.
Raises:
CollectorError: inconsistent or invalid graph data.
"""
assert isinstance(gs, global_state.GlobalState)
assert utilities.valid_string(output_format)
g = ContextGraph()
g.set_relations_to_timestamps(gs.get_relations_to_timestamps())
# Nodes
nodes_list = kubernetes.get_nodes_with_metrics(gs)
if not nodes_list:
return g.dump(output_format)
# Find the timestamp of the oldest node. This will be the timestamp of
# the cluster.
oldest_timestamp = utilities.now()
for node in nodes_list:
assert utilities.is_wrapped_object(node, 'Node')
# note: we cannot call min(oldest_timestamp, node['timestamp']) here
# because min(string) returnes the smallest character in the string.
if node['timestamp'] < oldest_timestamp:
oldest_timestamp = node['timestamp']
# The cluster name may be available through the Kubernetes API someday.
# TODO(rimey): Determine the cluster name.
cluster_name = '_unknown_'
cluster_guid = 'Cluster:' + cluster_name
g.set_title(cluster_name)
g.add_resource(cluster_guid, {'label': cluster_name}, 'Cluster',
oldest_timestamp, {})
# Nodes
for node in nodes_list:
_do_compute_node(cluster_guid, node, g)
# Pods
for pod in kubernetes.get_pods(gs):
_do_compute_pod(cluster_guid, pod, g)
# Services
for service in kubernetes.get_services(gs):
_do_compute_service(gs, cluster_guid, service, g)
# ReplicationControllers
for rcontroller in kubernetes.get_rcontrollers(gs):
_do_compute_rcontroller(gs, cluster_guid, rcontroller, g)
# Other nodes, not on the list, such as the Kubernetes master.
_do_compute_other_nodes(gs, cluster_guid, nodes_list, oldest_timestamp, g)
# Keep the relations_to_timestamps mapping for next call.
gs.set_relations_to_timestamps(g.get_relations_to_timestamps())
g.set_metadata({'timestamp': g.max_resources_and_relations_timestamp()})
# Dump the resulting graph
return g.dump(output_format)
def test_cluster(self):
"""Test the '/cluster' endpoint."""
start_time = utilities.now()
end_time = None
for _ in range(2):
# Exercise the collector. Read data from golden files and compute
# a context graph.
# The second iteration should read from the cache.
ret_value = self.app.get('/cluster')
if end_time is None:
end_time = utilities.now()
result = json.loads(ret_value.data)
# The timestamps of the second iteration should be the same as in the
# first iteration, because the data of the 2nd iteration should be
# fetched from the cache, and it did not change.
# Even if fetching the data caused an explicit reading from the files
# in the second iteration, the data did not change, so it should keep
# its original timestamp.
self.verify_resources(result, start_time, end_time)
self.assertEqual(5, self.count_relations(
result, 'contains', 'Cluster', 'Node'))
self.assertEqual(6, self.count_relations(
result, 'contains', 'Cluster', 'Service'))
self.assertEqual(3, self.count_relations(
result, 'contains', 'Cluster', 'ReplicationController'))
self.assertEqual(16, self.count_relations(
result, 'contains', 'Pod', 'Container'))
self.assertEqual(30, self.count_relations(result, 'contains'))
self.assertEqual(16, self.count_relations(result, 'createdFrom'))
self.assertEqual(7, self.count_relations(result, 'loadBalances'))
self.assertEqual(6, self.count_relations(result, 'monitors'))
self.assertEqual(14, self.count_relations(result, 'runs'))
# Verify that all relations contain a timestamp in the range
# [start_time, end_time].
self.assertTrue(isinstance(result.get('relations'), list))
for r in result['relations']:
self.assertTrue(isinstance(r, dict))
timestamp = r.get('timestamp')
self.assertTrue(utilities.valid_string(timestamp))
self.assertTrue(start_time <= timestamp <= end_time)
# The overall timestamp must be in the expected range.
self.assertTrue(utilities.valid_string(result.get('timestamp')))
self.assertTrue(start_time <= result['timestamp'] <= end_time)
# Wait a little to ensure that the current time is greater than
# end_time
time.sleep(1)
self.assertTrue(utilities.now() > end_time)
# Change the timestamp of the nodes in the cache.
timestamp_before_update = utilities.now()
gs = collector.app.context_graph_global_state
nodes, timestamp_seconds = gs.get_nodes_cache().lookup('')
self.assertTrue(isinstance(nodes, list))
self.assertTrue(start_time <=
utilities.seconds_to_timestamp(timestamp_seconds) <=
end_time)
# Change the first node to force the timestamp in the cache to change.
# We have to change both the properties of the first node and its
# timestamp, so the cache will store the new value (including the new
# timestamp).
self.assertTrue(len(nodes) >= 1)
self.assertTrue(utilities.is_wrapped_object(nodes[0], 'Node'))
nodes[0]['properties']['newAttribute123'] = 'the quick brown fox jumps over'
nodes[0]['timestamp'] = utilities.now()
gs.get_nodes_cache().update('', nodes)
timestamp_after_update = utilities.now()
_, timestamp_seconds = gs.get_nodes_cache().lookup('')
self.assertTrue(timestamp_before_update <=
utilities.seconds_to_timestamp(timestamp_seconds) <=
timestamp_after_update)
# Build the context graph again.
ret_value = self.app.get('/cluster')
result = json.loads(ret_value.data)
self.verify_resources(result, start_time, timestamp_after_update)
# Verify that all relations contain a timestamp in the range
# [start_time, end_time].
self.assertTrue(isinstance(result.get('relations'), list))
for r in result['relations']:
self.assertTrue(isinstance(r, dict))
timestamp = r.get('timestamp')
self.assertTrue(utilities.valid_string(timestamp))
self.assertTrue(start_time <= timestamp <= end_time)
# The overall timestamp must be in the expected range.
self.assertTrue(utilities.valid_string(result.get('timestamp')))
self.assertTrue(timestamp_before_update <= result['timestamp'] <=
timestamp_after_update)
def doFolderOrFiles(folderNameOrFileList,
skipFirst=0,
forceChi=False,
psize=100e-6,
d = 0.1,
xcen = 1024,
ycen = 1024,
lam = 1.,
qbin = 0.01,
tx = 0,
ty = 0,
x=None,
y=None,
folderOut=None,
mask = None,
waitForFiles=True,
imageReader = readMar,
ccdfilenames="*.mccd"
):
""" Perform azimuthal averaging.
folderNameOrFileList: can be either a folder (where the *.mccd will be found)
or a file list
skipFirst: to skip the first files (but careful to order...)
forceChi : calculate chi file even if output chi is present
d = sample detector distance
pixelsize = pixel size (in m)
xcen,ycen = center of the image
tx,ty = angle of detector normal with respect to incoming beam (in deg)
zeros are for perpendicular configuration
qbin = rebinning q (spacing or list)
Pplane = Polarization (1 = horizontal, 0 = vertical)
d = distance of center of detector to sample (in m)
lam = wavelength in Ang
folderOut : if None, same as folderNameOrFileList
imageReader : function that takes a name and return the intensity matrix
ccdfilenames : pattern to look for files
"""
az = None
t_start = util.now()
while ((az is None) or (waitForFiles)):
if (os.path.isdir(folderNameOrFileList)):
f=os.popen("ls -1 %s/%s" % (folderNameOrFileList,ccdfilenames))
temp=f.readlines()
f.close()
files = []
for t in temp:
files.append(t.strip())
files = files[skipFirst:]
if folderOut is None: folderOut = folderNameOrFileList
else:
if type(folderNameOrFileList) != list:
folderNameOrFileList = (folderNameOrFileList,); # if we pass single file
files=folderNameOrFileList[skipFirst:]
if folderOut is None: folderOut = os.path.dirname(folderNameOrFileList[0])
if (az is None):
if not os.path.exists(folderOut): os.makedirs(folderOut)
t0=time.time()
f = files[0]
iData = imageReader(f)
i0_mask = iData != 0
fname = folderOut+"/"+"azimuthal_averaging_info.png"
az = azimuthal_averaging(mask&i0_mask,xcen,ycen,pixelsize=psize,x=x,y=y,d=d,
lam=lam,qbin=qbin,img=iData,report_file=fname)
#az.displaypolar(iData)
print "Time needed for inizialization %.2f"%(time.time()-t0)
fname = folderOut+"/"+"azimuthal_averaging_info.txt"
finfo=open(fname,"w")
finfo.write(az.header)
finfo.close()
if (len(files) == 0):
print "Done %d files I could find, waiting for new files (%s)" % (skipFirst,util.now())
time.sleep(10)
t0=time.time()
t_save=0.
t_read=0.
t_az=0.
skip = 0
flist = []
data = []
err = []
for f in files:
fout = os.path.splitext(os.path.basename(f))[0]
fout = folderOut + "/" + fout + ".chi"
if (os.path.exists(fout) and not forceChi):
skip += 1
continue
else:
t1 = time.time()
iData = imageReader(f)
t_read += (time.time()-t1)
t1 = time.time()
az.do_azimuthal_averaging(iData)
t_az += (time.time()-t1)
t1 = time.time()
az.saveChiFile(fout)
t_save += (time.time()-t1)
flist.append(f)
data.append(az.I)
err.append(az.sig)
if ((len(files)-skip)!=0):
nfiles = len(files)
s="Time needed for %d files: %.2f ms/file"%(nfiles,(time.time()-t0)/nfiles*1e3)
ttot = t_read+t_save+t_az
s+="\n"+ "Fraction of time to read,calc,save : %.2f,%.2f,%.2f" % (t_read/ttot,t_az/ttot,t_save/ttot)
s+="\n"+ "Time per file to read,calc,save (ms): %.2f,%.2f,%.2f" % \
(t_read/nfiles*1e3,t_az/nfiles*1e3,t_save/nfiles*1e3)
print s
finfo=open(fname,"a")
finfo.write(s)
finfo.close()
hname = folderOut + "/" + folderOut.rstrip("/").split("/")[-1]+".h5"
t_end = util.now()
if (~forceChi & (os.path.exists(hname))):
hchi = h5py.File(hname,"r")
flist = [hchi["flist"].value,flist]
data = [hchi["data"].value,data]
err = [hchi["err"].value,err]
hchi.close()
hchi = h5py.File(hname,"w")
hchi.attrs["time_start"]=t_start
hchi.attrs["time_end"]=t_end
hchi.attrs["info"] = az.header
hchi.attrs["time_bench"] = s
hchi.create_dataset("flist",data=flist)
hchi.create_dataset("data",data=data)
hchi.create_dataset("err",data=err)
hchi.create_dataset("q",data=az.q)
hchi.create_dataset("theta",data=az.theta)
hchi.create_dataset("Npixel",data=az.Npixel)
hchi.close()
skipFirst += len(files)-skip
return az
def doImages(listOfImgs,
psize=100e-6,
d = 0.1,
xcen = 1024,
ycen = 1024,
lam = 1.,
qbin = 0.01,
tx = 0,
ty = 0,
x=None,
y=None,
folderOut="./",
mask = None,
force=False,
nJobs = 4,
hdf5out = None
):
""" Perform azimuthal averaging.
listOfImgs: list of images previously read
d = sample detector distance
pixelsize = pixel size (in m)
xcen,ycen = center of the image
tx,ty = angle of detector normal with respect to incoming beam (in deg)
zeros are for perpendicular configuration
qbin = rebinning q (spacing or list)
Pplane = Polarization (1 = horizontal, 0 = vertical)
d = distance of center of detector to sample (in m)
lam = wavelength in Ang
force = if True force reinizialization
hdf5out = if not None, use as outfile name
folderOut :
"""
print "NJOBS",nJobs
t_start = util.now()
t0=time.time()
if (len(listOfImgs) == 0):
return
if (g_az is None):
if (mask is None):
mask=np.ones_like(listOfImgs[0],dtype=np.bool)
t0=time.time()
fname = folderOut+"/"+"azimuthal_averaging_info.png"
az = azimuthal_averaging(mask,xcen,ycen,pixelsize=psize,x=x,y=y,d=d,
lam=lam,qbin=qbin,report_file=fname)
print "Time needed for inizialization %.2f"%(time.time()-t0)
globals()["g_az"] = az
else:
az = g_az
fname = folderOut+"/"+"azimuthal_averaging_info.txt"
t0=time.time()
nq = az.nq
nImg = len(listOfImgs)
if (nJobs > 1):
import jobManager
ag = jobManager.myAgent(nJobs=nJobs,parallel="thread")
#ag = jobManager.myAgent(nMax=nJobs,parallel="process")
n = int(np.ceil(float(nImg)/nJobs))
for i in range(nJobs):
m=i*n
M=(i+1)*n
M=min(M,nImg)
ag.addJob( _doImages,(listOfImgs[m:M],az) )
ag.waitUntilAllDone(update=0.05)
#time.sleep(10)
dataI=np.vstack ( [x[0] for x in ag.data] )
#dataI=np.reshape(dataI, (nImg,az.nq) )
dataE=np.vstack ( [x[1] for x in ag.data] )
#dataE=np.reshape(dataE, (nImg,az.nq) )
else:
dataI,dataE=_doImages(listOfImgs,az)
t_end = util.now()
s="Time needed for %d images: %.2f ms/img"%(nImg,(time.time()-t0)/nImg*1e3)
print s
finfo=open(fname,"a")
finfo.write(s)
finfo.close()
if hdf5out is not None:
hname = folderOut + "/" + hdf5out
hchi = h5py.File(hname,"w")
hchi.attrs["time_start"]=t_start
hchi.attrs["time_end"]=t_end
hchi.attrs["info"] = az.header
hchi.attrs["time_bench"] = s
hchi.create_dataset("data",data=dataI)
hchi.create_dataset("err",data=dataE)
hchi.create_dataset("q",data=az.q)
hchi.create_dataset("theta",data=az.theta)
hchi.create_dataset("Npixel",data=az.Npixel)
hchi.close()
return dataI,dataE,az
def _do_compute_graph(gs, input_queue, output_queue, output_format):
"""Returns the context graph in the specified format.
Args:
gs: the global state.
input_queue: the input queue for the worker threads.
output_queue: output queue containing exceptions data from the worker
threads.
output_format: one of 'graph', 'dot', 'context_graph', or 'resources'.
Returns:
A successful response in the specified format.
Raises:
CollectorError: inconsistent or invalid graph data.
"""
assert isinstance(gs, global_state.GlobalState)
assert isinstance(input_queue, Queue.PriorityQueue)
assert isinstance(output_queue, Queue.Queue)
assert utilities.valid_string(output_format)
g = ContextGraph()
g.set_version(docker.get_version(gs))
g.set_metadata({'timestamp': utilities.now()})
g.set_relations_to_timestamps(gs.get_relations_to_timestamps())
# Nodes
nodes_list = kubernetes.get_nodes_with_metrics(gs)
if not nodes_list:
return g.dump(gs, output_format)
# Find the timestamp of the oldest node. This will be the timestamp of
# the cluster.
oldest_timestamp = utilities.now()
for node in nodes_list:
assert utilities.is_wrapped_object(node, 'Node')
# note: we cannot call min(oldest_timestamp, node['timestamp']) here
# because min(string) returnes the smallest character in the string.
if node['timestamp'] < oldest_timestamp:
oldest_timestamp = node['timestamp']
# Get the cluster name from the first node.
# The cluster name is an approximation. It is not a big deal if it
# is incorrect, since the aggregator knows the cluster name.
cluster_name = utilities.node_id_to_cluster_name(nodes_list[0]['id'])
cluster_guid = 'Cluster:' + cluster_name
g.set_title(cluster_name)
g.add_resource(cluster_guid, {'label': cluster_name}, 'Cluster',
oldest_timestamp, {})
# Nodes
for node in nodes_list:
input_queue.put((
gs.get_random_priority(),
_do_compute_node,
{'gs': gs, 'input_queue': input_queue, 'cluster_guid': cluster_guid,
'node': node, 'g': g}))
# Services
for service in kubernetes.get_services(gs):
input_queue.put((
gs.get_random_priority(),
_do_compute_service,
{'gs': gs, 'cluster_guid': cluster_guid, 'service': service, 'g': g}))
# ReplicationControllers
rcontrollers_list = kubernetes.get_rcontrollers(gs)
for rcontroller in rcontrollers_list:
input_queue.put((
gs.get_random_priority(),
_do_compute_rcontroller,
{'gs': gs, 'cluster_guid': cluster_guid, 'rcontroller': rcontroller,
'g': g}))
# Wait until worker threads finished processing all outstanding requests.
# Once we return from the join(), all output was generated already.
input_queue.join()
# Convert any exception caught by the worker threads to an exception
# raised by the current thread.
if not output_queue.empty():
msg = output_queue.get_nowait() # should not fail.
gs.logger_error(msg)
raise collector_error.CollectorError(msg)
# Keep the relations_to_timestamps mapping for next call.
gs.set_relations_to_timestamps(g.get_relations_to_timestamps())
# Dump the resulting graph
return g.dump(gs, output_format)
def test_cluster(self):
"""Test the '/cluster' endpoint."""
start_time = utilities.now()
end_time = None
for _ in range(2):
# Exercise the collector. Read data from golden files and compute
# a context graph.
# The second iteration should read from the cache.
ret_value = self.app.get('/cluster')
if end_time is None:
end_time = utilities.now()
result = json.loads(ret_value.data)
# The timestamps of the second iteration should be the same as in the
# first iteration, because the data of the 2nd iteration should be
# fetched from the cache, and it did not change.
# Even if fetching the data caused an explicit reading from the files
# in the second iteration, the data did not change, so it should keep
# its original timestamp.
self.verify_resources(result, start_time, end_time)
self.assertEqual(5, self.count_relations(
result, 'contains', 'Cluster', 'Node'))
self.assertEqual(6, self.count_relations(
result, 'contains', 'Cluster', 'Service'))
self.assertEqual(3, self.count_relations(
result, 'contains', 'Cluster', 'ReplicationController'))
self.assertEqual(1, self.count_relations(
result, 'contains', 'Node', 'Container'))
self.assertEqual(4, self.count_relations(
result, 'contains', 'Pod', 'Container'))
self.assertEqual(7, self.count_relations(
result, 'contains', 'Container', 'Process'))
self.assertEqual(26, self.count_relations(result, 'contains'))
self.assertEqual(3, self.count_relations(result, 'createdFrom'))
self.assertEqual(7, self.count_relations(result, 'loadBalances'))
self.assertEqual(6, self.count_relations(result, 'monitors'))
self.assertEqual(14, self.count_relations(result, 'runs'))
# Verify that all relations contain a timestamp in the range
# [start_time, end_time].
self.assertTrue(isinstance(result.get('relations'), types.ListType))
for r in result['relations']:
self.assertTrue(isinstance(r, types.DictType))
timestamp = r.get('timestamp')
self.assertTrue(utilities.valid_string(timestamp))
self.assertTrue(start_time <= timestamp <= end_time)
# The overall timestamp must be in the expected range.
self.assertTrue(utilities.valid_string(result.get('timestamp')))
self.assertTrue(start_time <= result['timestamp'] <= end_time)
json_output = json.dumps(result, sort_keys=True)
self.assertEqual(2, json_output.count('"alternateLabel": '))
self.assertEqual(99, json_output.count('"createdBy": '))
# Wait a little to ensure that the current time is greater than
# end_time
time.sleep(1)
self.assertTrue(utilities.now() > end_time)
# Change the timestamp of the nodes in the cache.
timestamp_before_update = utilities.now()
gs = collector.app.context_graph_global_state
nodes, timestamp_seconds = gs.get_nodes_cache().lookup('')
self.assertTrue(isinstance(nodes, types.ListType))
self.assertTrue(start_time <=
utilities.seconds_to_timestamp(timestamp_seconds) <=
end_time)
# Change the first node to force the timestamp in the cache to change.
# We have to change both the properties of the first node and its
# timestamp, so the cache will store the new value (including the new
# timestamp).
self.assertTrue(len(nodes) >= 1)
self.assertTrue(utilities.is_wrapped_object(nodes[0], 'Node'))
nodes[0]['properties']['newAttribute123'] = 'the quick brown fox jumps over'
nodes[0]['timestamp'] = utilities.now()
gs.get_nodes_cache().update('', nodes)
timestamp_after_update = utilities.now()
_, timestamp_seconds = gs.get_nodes_cache().lookup('')
self.assertTrue(timestamp_before_update <=
utilities.seconds_to_timestamp(timestamp_seconds) <=
timestamp_after_update)
# Build the context graph again.
ret_value = self.app.get('/cluster')
result = json.loads(ret_value.data)
self.verify_resources(result, start_time, timestamp_after_update)
# Verify that all relations contain a timestamp in the range
# [start_time, end_time].
self.assertTrue(isinstance(result.get('relations'), types.ListType))
for r in result['relations']:
self.assertTrue(isinstance(r, types.DictType))
timestamp = r.get('timestamp')
self.assertTrue(utilities.valid_string(timestamp))
self.assertTrue(start_time <= timestamp <= end_time)
# The overall timestamp must be in the expected range.
self.assertTrue(utilities.valid_string(result.get('timestamp')))
self.assertTrue(timestamp_before_update <= result['timestamp'] <=
timestamp_after_update)
def __init__(self,msg,fromfunction=None,showTimestamp=True): self.msg=msg self.time = utilities.now() self.fromfunction=fromfunction self.showTimestamp=showTimestamp
def _do_compute_graph(gs, input_queue, output_queue, output_format):
"""Returns the context graph in the specified format.
Args:
gs: the global state.
input_queue: the input queue for the worker threads.
output_queue: output queue containing exceptions data from the worker
threads.
output_format: one of 'graph', 'dot', 'context_graph', or 'resources'.
Returns:
A successful response in the specified format.
Raises:
CollectorError: inconsistent or invalid graph data.
"""
assert isinstance(gs, global_state.GlobalState)
assert isinstance(input_queue, Queue.PriorityQueue)
assert isinstance(output_queue, Queue.Queue)
assert utilities.valid_string(output_format)
g = ContextGraph()
g.set_version(docker.get_version(gs))
g.set_metadata({'timestamp': utilities.now()})
g.set_relations_to_timestamps(gs.get_relations_to_timestamps())
# Nodes
nodes_list = kubernetes.get_nodes_with_metrics(gs)
if not nodes_list:
return g.dump(gs, output_format)
# Find the timestamp of the oldest node. This will be the timestamp of
# the cluster.
oldest_timestamp = utilities.now()
for node in nodes_list:
assert utilities.is_wrapped_object(node, 'Node')
# note: we cannot call min(oldest_timestamp, node['timestamp']) here
# because min(string) returnes the smallest character in the string.
if node['timestamp'] < oldest_timestamp:
oldest_timestamp = node['timestamp']
# Get the cluster name from the first node.
# The cluster name is an approximation. It is not a big deal if it
# is incorrect, since the aggregator knows the cluster name.
cluster_name = utilities.node_id_to_cluster_name(nodes_list[0]['id'])
cluster_guid = 'Cluster:' + cluster_name
g.set_title(cluster_name)
g.add_resource(cluster_guid, {'label': cluster_name}, 'Cluster',
oldest_timestamp, {})
# Nodes
for node in nodes_list:
input_queue.put((gs.get_random_priority(), _do_compute_node, {
'gs': gs,
'input_queue': input_queue,
'cluster_guid': cluster_guid,
'node': node,
'g': g
}))
# Services
for service in kubernetes.get_services(gs):
input_queue.put((gs.get_random_priority(), _do_compute_service, {
'gs': gs,
'cluster_guid': cluster_guid,
'service': service,
'g': g
}))
# ReplicationControllers
rcontrollers_list = kubernetes.get_rcontrollers(gs)
for rcontroller in rcontrollers_list:
input_queue.put((gs.get_random_priority(), _do_compute_rcontroller, {
'gs': gs,
'cluster_guid': cluster_guid,
'rcontroller': rcontroller,
'g': g
}))
# Wait until worker threads finished processing all outstanding requests.
# Once we return from the join(), all output was generated already.
input_queue.join()
# Convert any exception caught by the worker threads to an exception
# raised by the current thread.
if not output_queue.empty():
msg = output_queue.get_nowait() # should not fail.
gs.logger_error(msg)
raise collector_error.CollectorError(msg)
# Keep the relations_to_timestamps mapping for next call.
gs.set_relations_to_timestamps(g.get_relations_to_timestamps())
# Dump the resulting graph
return g.dump(gs, output_format)