def test_history_last_filter():
for i in range(4):
li = LogItem.new({'log': 'history'})
li.save()
last = li
assert LogItem.history_last() == last
def _guess_log_item(self, resource_obj):
# created state will be changed during commit
if resource_obj.db_obj.state != RESOURCE_STATE.created.name:
action = 'update'
else:
action = 'run'
LogItem.new(
{'resource': resource_obj.name,
'action': action,
'tags': resource_obj.tags}).save_lazy()
def remove(self, force=False):
if force:
self.delete()
else:
self.db_obj.state = RESOURCE_STATE.removed.name
self.db_obj.save_lazy()
LogItem.new(
{'resource': self.name,
'action': 'remove',
'log': 'staged',
'tags': self.tags}).save_lazy()
def remove(self, force=False):
if force:
self.delete()
else:
self.db_obj.state = RESOURCE_STATE.removed.name
self.db_obj.save_lazy()
LogItem.new({
'resource': self.name,
'action': 'remove',
'log': 'staged',
'tags': self.tags
}).save_lazy()
def update(self, args):
for k, v in args.items():
self.db_obj.inputs[k] = v
self.db_obj.save_lazy()
# created state will be changed during commit
if self.db_obj.state != RESOURCE_STATE.created.name:
action = 'update'
else:
action = 'run'
LogItem.new(
{'resource': self.name,
'action': action,
'tags': self.tags}).save_lazy()
def update(self, args):
# TODO: disconnect input when it is updated and end_node
# for some input_to_input relation
self.db_obj.state = RESOURCE_STATE.updated.name
for k, v in args.items():
self.db_obj.inputs[k] = v
self.db_obj.save_lazy()
# run and update are same things from solar pov
# so lets remove this redundancy
LogItem.new(
{'resource': self.name,
'action': 'run',
'tags': self.tags}).save_lazy()
def connect_with_events(self, receiver, mapping=None, events=None,
use_defaults=False):
mapping = get_mapping(self, receiver, mapping)
self._connect_inputs(receiver, mapping)
LogItem.new({'resource': receiver.name,
'action': 'run',
'tags': receiver.tags}).save_lazy()
# signals.connect(self, receiver, mapping=mapping)
# TODO: implement events
if use_defaults:
if self != receiver:
api.add_default_events(self, receiver)
if events:
api.add_events(self.name, events)
def update(self, args):
for k, v in args.items():
self.db_obj.inputs[k] = v
self.db_obj.save_lazy()
# created state will be changed during commit
if self.db_obj.state != RESOURCE_STATE.created.name:
action = 'update'
else:
action = 'run'
LogItem.new({
'resource': self.name,
'action': action,
'tags': self.tags
}).save_lazy()
def create_from_spec(self, name, spec, args=None, tags=None):
args = args or {}
self.name = name
if spec:
if spec.startswith('/'):
# it's full path, don't use repo
self.base_path = spec
metadata = read_meta(spec)
else:
repo, spec = Repository.parse(spec)
metadata = repo.get_metadata(spec)
self.base_path = repo.get_path(spec)
else:
metadata = deepcopy(self._metadata)
self.base_path = spec # TODO: remove this old method?
if tags is None:
tags = []
m_tags = metadata.get('tags', [])
tags.extend(m_tags)
tags.append('resource={}'.format(name))
inputs = metadata.get('input', {})
self.auto_extend_inputs(inputs)
self.db_obj = DBResource.from_dict(
name, {
'id': name,
'name': name,
'actions_path': metadata.get('actions_path', ''),
'actions': metadata.get('actions', {}),
'base_name': metadata.get('base_name', ''),
'base_path': metadata.get('base_path', ''),
'handler': metadata.get('handler', ''),
'version': metadata.get('version', ''),
'meta_inputs': inputs,
'tags': tags,
'state': RESOURCE_STATE.created.name,
'managers': metadata.get('managers', [])
})
self.create_inputs(args)
self.db_obj.save()
LogItem.new({
'resource': self.name,
'action': 'run',
'log': 'staged',
'tags': self.tags
}).save_lazy()
def create_from_spec(self, name, spec, args=None, tags=None):
args = args or {}
self.name = name
if spec:
if spec.startswith('/'):
# it's full path, don't use repo
self.base_path = spec
metadata = read_meta(spec)
else:
repo, spec = Repository.parse(spec)
metadata = repo.get_metadata(spec)
self.base_path = repo.get_path(spec)
else:
metadata = deepcopy(self._metadata)
self.base_path = spec # TODO: remove this old method?
if tags is None:
tags = []
m_tags = metadata.get('tags', [])
tags.extend(m_tags)
tags.append('resource={}'.format(name))
inputs = metadata.get('input', {})
self.auto_extend_inputs(inputs)
self.db_obj = DBResource.from_dict(
name,
{
'id': name,
'name': name,
'actions_path': metadata.get('actions_path', ''),
'actions': metadata.get('actions', {}),
'base_name': metadata.get('base_name', ''),
'base_path': metadata.get('base_path', ''),
'handler': metadata.get('handler', ''),
'version': metadata.get('version', ''),
'meta_inputs': inputs,
'tags': tags,
'state': RESOURCE_STATE.created.name,
'managers': metadata.get('managers', [])
})
self.create_inputs(args)
self.db_obj.save()
LogItem.new({
'resource': self.name,
'action': 'run',
'log': 'staged',
'tags': self.tags}).save_lazy()
def test_multiple_filter():
l1 = LogItem.new({'log': 'history', 'resource': 'a'})
l2 = LogItem.new({'log': 'history', 'resource': 'b'})
l1.save()
l2.save()
assert LogItem.composite.filter({
'log': 'history',
'resource': 'a'
}) == [l1.key]
assert LogItem.composite.filter({
'log': 'history',
'resource': 'b'
}) == [l2.key]
def test_separate_logs():
history = 'history'
staged = 'staged'
history_uids = set()
staged_uids = set()
for i in range(2):
l = LogItem.new({'log': history})
l.save()
history_uids.add(l.key)
for i in range(3):
l = LogItem.new({'log': staged})
l.save()
staged_uids.add(l.key)
assert set(LogItem.composite.filter({'log': history})) == history_uids
assert set(LogItem.composite.filter({'log': staged})) == staged_uids
def to_log_item(self, action):
return LogItem.new(
{'resource': self.resource,
'diff': self.diff,
'connections_diff': self.connections,
'base_path': self.path,
'action': action,
'log': 'staged'})
def create_logitem(resource, action, diffed, connections_diffed,
base_path=''):
return LogItem.new(
{'resource': resource,
'action': action,
'diff': diffed,
'connections_diff': connections_diffed,
'base_path': base_path,
'log': 'staged'})
def connect_with_events(self,
receiver,
mapping=None,
events=None,
use_defaults=False):
mapping = get_mapping(self, receiver, mapping)
self._connect_inputs(receiver, mapping)
LogItem.new({
'resource': receiver.name,
'action': 'run',
'tags': receiver.tags
}).save_lazy()
# signals.connect(self, receiver, mapping=mapping)
# TODO: implement events
if use_defaults:
if self != receiver:
api.add_default_events(self, receiver)
if events:
api.add_events(self.name, events)
def stage_changes():
for li in data.SL():
li.delete()
last = LogItem.history_last()
since = StrInt.greater(last.updated) if last else None
staged_log = utils.solar_map(make_single_stage_item,
resource.load_updated(since), concurrency=10)
staged_log = filter(None, staged_log)
return staged_log
def test_reversed_order_is_preserved():
added = []
for i in range(4):
li = LogItem.new({'log': 'history'})
li.save()
added.append(li.key)
added.reverse()
assert list(
LogItem.history.filter(StrInt.n_max(), StrInt.n_min(),
max_results=2)) == added[:2]
def test_changed_index():
l = LogItem.new({'log': 'staged', 'resource': 'a', 'action': 'run'})
l.save()
assert LogItem.composite.filter({'log': 'staged'}) == [l.key]
l.log = 'history'
l.save()
assert LogItem.composite.filter({'log': 'staged'}) == []
assert LogItem.composite.filter({'log': 'history'}) == [l.key]
def create_logitem(resource, action, populate=True):
"""Create log item in staged log
:param resource: basestring
:param action: basestring
"""
log_item = LogItem.new(
{'resource': resource,
'action': action,
'log': 'staged'})
if populate:
populate_log_item(log_item)
return log_item
def create_logitem(resource, action, populate=True):
"""Create log item in staged log
:param resource: basestring
:param action: basestring
"""
log_item = LogItem.new({
'resource': resource,
'action': action,
'log': 'staged'
})
if populate:
populate_log_item(log_item)
return log_item
def discard_uids(uids):
items = LogItem.multi_get(uids)
for item in items:
if item.action == CHANGES.update.name:
_discard_update(item)
elif item.action == CHANGES.remove.name:
_discard_remove(item)
elif item.action == CHANGES.run.name:
_discard_run(item)
else:
log.debug('Action %s for resource %s is a side'
' effect of another action', item.action, item.res)
item.delete()
def test_staged_not_indexed():
added = []
for i in range(3):
li = LogItem.new({'log': 'staged'})
li.save()
added.append(li)
for li in added[:2]:
li.log = 'history'
li.save()
assert set(LogItem.history.filter(
StrInt.n_max(), StrInt.n_min())) == {li.key
for li in added[:2]}
def revert_uids(uids):
"""Reverts uids
:param uids: iterable not generator
"""
items = LogItem.multi_get(uids)
for item in items:
if item.action == CHANGES.update.name:
_revert_update(item)
elif item.action == CHANGES.remove.name:
_revert_remove(item)
elif item.action == CHANGES.run.name:
_revert_run(item)
else:
log.debug('Action %s for resource %s is a side'
' effect of another action', item.action, item.res)
def send_to_orchestration(tags=None):
dg = nx.MultiDiGraph()
events = {}
changed_nodes = []
if tags:
staged_log = LogItem.log_items_by_tags(tags)
else:
staged_log = data.SL()
for logitem in staged_log:
events[logitem.resource] = evapi.all_events(logitem.resource)
changed_nodes.append(logitem.resource)
state_change = StateChange(logitem.resource, logitem.action)
state_change.insert(changed_nodes, dg)
evapi.build_edges(dg, events)
# what `name` should be?
dg.graph['name'] = 'system_log'
return graph.create_plan_from_graph(dg)
def send_to_orchestration(tags=None):
dg = nx.MultiDiGraph()
events = {}
changed_nodes = []
if tags:
staged_log = LogItem.log_items_by_tags(tags)
else:
staged_log = data.SL()
for logitem in staged_log:
events[logitem.resource] = evapi.all_events(logitem.resource)
changed_nodes.append(logitem.resource)
state_change = StateChange(logitem.resource, logitem.action)
state_change.insert(changed_nodes, dg)
evapi.build_edges(dg, events)
# what `name` should be?
dg.graph['name'] = 'system_log'
return graph.create_plan_from_graph(dg)
def stage_resources(resources_query, action):
"""Create log items for resources selected by query
:param resources_query: iterable with tags or basestring
:param action: basestring
"""
if isinstance(resources_query, basestring):
resources = [load(resources_query)]
else:
resources = load_by_tags(resources_query)
created = []
for resource in resources:
# save - cache doesnt cover all query in the same sesssion
# and this query will be triggered right after staging resources
log_item = LogItem.new(
{'resource': resource.name,
'action': action,
'log': 'staged',
'tags': resource.tags})
log_item.save()
created.append(log_item)
return created
def stage_resources(resources_query, action):
"""Create log items for resources selected by query
:param resources_query: iterable with tags or basestring
:param action: basestring
"""
if isinstance(resources_query, basestring):
resources = [load(resources_query)]
else:
resources = load_by_tags(resources_query)
created = []
for resource in resources:
# save - cache doesnt cover all query in the same sesssion
# and this query will be triggered right after staging resources
log_item = LogItem.new({
'resource': resource.name,
'action': action,
'log': 'staged',
'tags': resource.tags
})
log_item.save()
created.append(log_item)
return created
def SL():
rst = LogItem.bucket.get_index('$bucket', startkey='_',
max_results=100000).results
return filter(bool, LogItem.multi_get(rst))
def SL():
rst = LogItem.composite.filter({'log': 'staged'})
return LogItem.multi_get(rst)
def CL():
rst = LogItem.composite.filter({'log': 'history'})
return LogItem.multi_get(rst)
def clear_history():
LogItem.delete_all()
CommitedResource.delete_all()
def discard_uids(uids):
items = filter(bool, LogItem.multi_get(uids))
for item in items:
discard_single(item)
def CL():
rst = LogItem.composite.filter({'log': 'history'})
return LogItem.multi_get(rst)
def clear_history():
LogItem.delete_all()
CommitedResource.delete_all()
def test_history_last_returns_none():
assert LogItem.history_last() is None
def SL():
rst = LogItem.bucket.get_index('$bucket',
startkey='_',
max_results=100000).results
return filter(bool, LogItem.multi_get(rst))
def SL():
rst = LogItem.composite.filter({'log': 'staged'})
return LogItem.multi_get(rst)
