def add_remove(device: Computer, components: Set[Component]) -> OrderedSet:
"""
Generates the Add and Remove events (but doesn't add them to
session).
:param device: A device which ``components`` attribute contains
the old list of components. The components that
are not in ``components`` will be Removed.
:param components: List of components that are potentially to
be Added. Some of them can already exist
on the device, in which case they won't
be re-added.
:return: A list of Add / Remove events.
"""
# Note that we create the Remove events before the Add ones
events = OrderedSet()
old_components = set(device.components)
adding = components - old_components
if adding:
# For the components we are adding, let's remove them from their old parents
def g_parent(component: Component) -> Device:
return component.parent or Device(
id=0) # Computer with id 0 is our Identity
for parent, _components in groupby(sorted(adding, key=g_parent),
key=g_parent):
if parent.id != 0: # Is not Computer Identity
events.add(
Remove(device=parent,
components=OrderedSet(_components)))
return events
class _QueryPlan(object):
def __init__(self):
self._queries = OrderedSet({None})
self._columns = defaultdict(OrderedSet)
self._children = defaultdict(OrderedSet)
def add_query(self, query, parent_query):
self._queries.add(query)
self._children[parent_query].add(query)
def add_expr(self, query, column):
self._columns[query].add(column)
def query_id(self, query):
return list(self._queries).index(query)
def column_id(self, query, column):
return list(self._columns[query]).index(column)
def query_columns(self, query):
return tuple(self._columns.get(query) or ())
def query_children(self, query):
return tuple(self._children.get(query) or ())
def process_rows(self, rows, session):
children = self.query_children(None)
results = [[((self.query_id(None), row),) for row in rows]]
results.extend(child.__execute__(self, None, rows, session)
for child in children)
return {0: [tuple(chain(*r)) for r in zip(*results)]}
def load(self):
header = OrderedSet(default_header)
parties = {}
for party in self.session.query(self.Party).all():
parties[party.nr] = party.abbr
header.add(party.abbr)
query = "SELECT constituency.nr AS cnr, district.nr AS dnr, judicaldistrict.nr AS jdnr, judicaldistrict.electivecnt, " \
"judicaldistrict.votecnt, judicaldistrict.invalidcnt, votes.pnr, votes.cnt " \
"FROM constituency " \
"INNER JOIN district ON constituency.nr = district.cnr " \
"INNER JOIN judicaldistrict ON district.nr = judicaldistrict.dnr " \
"AND judicaldistrict.enr = '" + str(self.enr) + "' " \
"INNER JOIN votes ON votes.enr = '" + str(self.enr) + "' " \
"AND votes.dnr = district.nr " \
"AND votes.jdnr = judicaldistrict.nr;"
result = self.session.execute(query)
data = {}
for row in result:
key = str(row["cnr"]) + str(row["dnr"]) + str(row["jdnr"])
if key not in data:
data[key] = {
"WK": row["cnr"],
"BZ": row["dnr"],
"SPR": row["jdnr"],
"WBER": row["electivecnt"],
"ABG": row["votecnt"],
"UNG": row["invalidcnt"],
"T": 4,
"WV": 1,
}
data[key][parties[row["pnr"]]] = row["cnt"]
return header, list(data.values())
def load(self):
header = OrderedSet(default_header)
parties = {}
for party in self.session.query(self.Party).all():
parties[party.nr] = party.abbr
header.add(party.abbr)
query = "SELECT constituency.nr AS cnr, district.nr AS dnr, judicaldistrict.nr AS jdnr, judicaldistrict.electivecnt, " \
"judicaldistrict.votecnt, judicaldistrict.invalidcnt, votes.pnr, votes.cnt " \
"FROM constituency " \
"INNER JOIN district ON constituency.nr = district.cnr " \
"INNER JOIN judicaldistrict ON district.nr = judicaldistrict.dnr " \
"AND judicaldistrict.enr = '" + str(self.enr) + "' " \
"INNER JOIN votes ON votes.enr = '" + str(self.enr) + "' " \
"AND votes.dnr = district.nr " \
"AND votes.jdnr = judicaldistrict.nr;"
result = self.session.execute(query)
data = {}
for row in result:
key = str(row["cnr"]) + str(row["dnr"]) + str(row["jdnr"])
if key not in data:
data[key] = {
"WK": row["cnr"],
"BZ": row["dnr"],
"SPR": row["jdnr"],
"WBER": row["electivecnt"],
"ABG": row["votecnt"],
"UNG": row["invalidcnt"],
"T": 4,
"WV": 1,
}
data[key][parties[row["pnr"]]] = row["cnt"]
return header, list(data.values())
def run(self,
device: Device,
components: Iterable[Component] or None) -> (Device, OrderedSet):
"""Synchronizes the device and components with the database.
Identifies if the device and components exist in the database
and updates / inserts them as necessary.
Passed-in parameters have to be transient, or said differently,
not-db-synced objects, or otherwise they would end-up being
added in the session. `Learn more... <http://docs.sqlalchemy.org/
en/latest/orm/session_state_management.html#quickie-intro-to
-object-states>`_.
This performs Add / Remove as necessary.
:param device: The device to add / update to the database.
:param components: Components that are inside of the device.
This method performs Add and Remove actions
so the device ends up with these components.
Components are added / updated accordingly.
If this is empty, all components are removed.
If this is None, it means that we are not
providing info about the components, in which
case we keep the already existing components
of the device –we don't touch them.
:return: A tuple of:
1. The device from the database (with an ID) whose
``components`` field contain the db version
of the passed-in components.
2. A list of Add / Remove (not yet added to session).
"""
db_device = self.execute_register(device)
db_components, actions = OrderedSet(), OrderedSet()
if components is not None: # We have component info (see above)
if not isinstance(db_device, Computer):
# Until a good reason is given, we synthetically forbid
# non-computers with components
raise ValidationError('Only computers can have components.')
blacklist = set() # type: Set[int]
not_new_components = set()
for component in components:
db_component, is_new = self.execute_register_component(component,
blacklist,
parent=db_device)
db_components.add(db_component)
if not is_new:
not_new_components.add(db_component)
# We only want to perform Add/Remove to not new components
actions = self.add_remove(db_device, not_new_components)
db_device.components = db_components
return db_device, actions
def test_files_walk(proj, limit=100):
repo, proj = _get_repo(proj)
print 'processing', jn(PROJECT_PATH, proj)
total_time = 0
paths_ = OrderedSet([''])
try:
tip = repo.get_commit('tip')
for topnode, dirs, files in tip.walk('/'):
for dir in dirs:
paths_.add(dir.path)
for f in dir:
paths_.add(f.path)
for f in files:
paths_.add(f.path)
except RepositoryError as e:
pass
cnt = 0
for f in paths_:
cnt += 1
if limit and limit == cnt:
break
file_path = '/'.join((proj, 'files', 'tip', f))
full_uri = (BASE_URI % file_path)
print '%s visiting %s' % (cnt, full_uri)
s = time.time()
f = o.open(full_uri)
size = len(f.read())
e = time.time() - s
total_time += e
print '%s visited OK size:%s req:%s ms' % (cnt, size, e)
print 'total_time', total_time
print 'average on req', total_time / float(cnt)
def _to_python(self, value, state):
perm_updates = OrderedSet()
perm_additions = OrderedSet()
perm_deletions = OrderedSet()
# build a list of permission to update/delete and new permission
# Read the perm_new_member/perm_del_member attributes and group
# them by they IDs
new_perms_group = defaultdict(dict)
del_perms_group = defaultdict(dict)
for k, v in value.copy().iteritems():
if k.startswith('perm_del_member'):
# delete from org storage so we don't process that later
del value[k]
# part is `id`, `type`
_type, part = k.split('perm_del_member_')
args = part.split('_')
if len(args) == 2:
_key, pos = args
del_perms_group[pos][_key] = v
if k.startswith('perm_new_member'):
# delete from org storage so we don't process that later
del value[k]
# part is `id`, `type`, `perm`
_type, part = k.split('perm_new_member_')
args = part.split('_')
if len(args) == 2:
_key, pos = args
new_perms_group[pos][_key] = v
# store the deletes
for k in sorted(del_perms_group.keys()):
perm_dict = del_perms_group[k]
del_member = perm_dict.get('id')
del_type = perm_dict.get('type')
if del_member and del_type:
perm_deletions.add((del_member, None, del_type))
# store additions in order of how they were added in web form
for k in sorted(new_perms_group.keys()):
perm_dict = new_perms_group[k]
new_member = perm_dict.get('id')
new_type = perm_dict.get('type')
new_perm = perm_dict.get('perm')
if new_member and new_perm and new_type:
perm_additions.add((new_member, new_perm, new_type))
# get updates of permissions
# (read the existing radio button states)
for k, update_value in value.iteritems():
if k.startswith('u_perm_') or k.startswith('g_perm_'):
member = k[7:]
update_type = {'u': 'user',
'g': 'users_group'}[k[0]]
if member == User.DEFAULT_USER:
if str2bool(value.get('repo_private')):
# set none for default when updating to
# private repo protects agains form manipulation
update_value = EMPTY_PERM
perm_updates.add((member, update_value, update_type))
# check the deletes
value['perm_additions'] = list(perm_additions)
value['perm_updates'] = list(perm_updates)
value['perm_deletions'] = list(perm_deletions)
# validate users they exist and they are active !
for member_id, _perm, member_type in perm_additions:
try:
if member_type == 'user':
self.user_db = User.query()\
.filter(User.active == true())\
.filter(User.user_id == member_id).one()
if member_type == 'users_group':
self.user_db = UserGroup.query()\
.filter(UserGroup.users_group_active == true())\
.filter(UserGroup.users_group_id == member_id)\
.one()
except Exception:
log.exception('Updated permission failed: org_exc:')
msg = M(self, 'perm_new_member_type', state)
raise formencode.Invalid(
msg, value, state, error_dict={
'perm_new_member_name': msg}
)
return value
