def test_string_index(self):
class Test(MappedClass):
class __mongometa__:
indexes = [ 'abc' ]
_id = FieldProperty(S.Int)
abc=FieldProperty(S.Int, if_missing=None)
mgr = mapper(Test).collection.m
assert len(mgr.indexes) == 1, mgr.indexes
def dump_cls(depth, cls):
indent = ' '*4*depth
yield indent + '%s.%s' % (cls.__module__, cls.__name__)
m = mapper(cls)
for p in m.properties:
s = indent*2 + ' - ' + str(p)
if hasattr(p, 'field_type'):
s += ' (%s)' % p.field_type
yield s
def dump_cls(depth, cls):
indent = " " * 4 * depth
yield indent + "%s.%s" % (cls.__module__, cls.__name__)
m = mapper(cls)
for p in m.properties:
s = indent * 2 + " - " + str(p)
if hasattr(p, "field_type"):
s += " (%s)" % p.field_type
yield s
def test_mapper(self):
m = mapper(self.Basic)
self.assertEqual(repr(m), '<Mapper Basic:basic>')
doc = self.Basic(a=1, b=[2,3], c=dict(d=4, e=5))
self.session.flush()
q = self.Basic.query.find()
self.assertEqual(q.count(), 1)
m.remove({})
q = self.Basic.query.find()
self.assertEqual(q.count(), 0)
def test_mapper(self):
m = mapper(self.Basic)
self.assert_(repr(m).startswith('<Mapper for '))
doc = self.Basic(a=1, b=[2,3], c=dict(d=4, e=5))
self.session.flush()
q = self.Basic.query.find()
self.assertEqual(q.count(), 1)
m.remove({})
q = self.Basic.query.find()
self.assertEqual(q.count(), 0)
def get_commit_info(commit):
if not isinstance(commit, Commit):
commit = mapper(Commit).create(commit, dict(instrument=False))
session(commit).expunge(commit)
return dict(id=commit._id,
author=commit.authored.name,
author_email=commit.authored.email,
date=commit.authored.date,
author_url=commit.author_url,
shortlink=commit.shorthand_id(),
summary=commit.summary)
def setUp(self):
self.datastore = DS.DataStore(
'mim:///', database='test_db')
session = Session(bind=self.datastore)
self.session = ORMSession(session)
class Parent(object): pass
class Child(object): pass
parent = collection(
'parent', session,
Field('_id', int))
child = collection(
'child', session,
Field('_id', int),
Field('parent_id', int))
mapper(Parent, parent, self.session, properties=dict(
children=RelationProperty(Child)))
mapper(Child, child, self.session, properties=dict(
parent_id=ForeignIdProperty(Parent),
parent = RelationProperty(Parent)))
self.Parent = Parent
self.Child = Child
def test_mapper(self):
m = mapper(self.Basic)
assert repr(m) == '<Mapper Basic:basic>'
self.datastore.db.basic.insert(dict(
a=1, b=[2,3], c=dict(d=4, e=5), f='unknown'))
print(list(self.datastore.db.basic.find()))
obj = self.Basic.query.find().options(instrument=False).first()
print(obj)
q = self.Basic.query.find()
self.assertEqual(q.count(), 1)
m.remove({})
q = self.Basic.query.find()
self.assertEqual(q.count(), 0)
def setUp(self):
self.datastore = DS.DataStore(
'mim:///', database='test_db')
session = Session(bind=self.datastore)
self.session = ORMSession(session)
base = collection(
'test_doc', session,
Field('_id', S.ObjectId),
Field('type', str, if_missing='base'),
Field('a', int),
polymorphic_on='type',
polymorphic_identity='base')
derived = collection(
base,
Field('type', str, if_missing='derived'),
Field('b', int),
polymorphic_identity='derived')
class Base(object): pass
class Derived(Base): pass
mapper(Base, base, self.session)
mapper(Derived, derived, self.session)
self.Base = Base
self.Derived = Derived
def get_commit_info(commit):
if not isinstance(commit, Commit):
commit = mapper(Commit).create(commit, dict(instrument=False))
sess = session(commit)
if sess: sess.expunge(commit)
return dict(
id=commit._id,
author=commit.authored.name,
author_email=commit.authored.email,
date=commit.authored.date,
author_url=commit.author_url,
shortlink=commit.shorthand_id(),
summary=commit.summary
)
def get_projects_for_macro(category=None, display_mode='grid', sort='last_updated',
show_total=False, limit=100, labels='', award='', private=False,
columns=1, show_proj_icon=True, show_download_button=True, show_awards_banner=True,
grid_view_tools='',
initial_q={}):
from allura.lib.widgets.project_list import ProjectList
from allura.lib import utils
from allura import model as M
# 'trove' is internal substitution for 'category' filter in wiki macro
trove = category
limit = int(limit)
q = dict(
deleted=False,
is_nbhd_project=False)
q.update(initial_q)
if labels:
or_labels = labels.split('|')
q['$or'] = [{'labels': {'$all': l.split(',')}} for l in or_labels]
if trove is not None:
trove = M.TroveCategory.query.get(fullpath=trove)
if award:
aw = M.Award.query.find(dict(
created_by_neighborhood_id=c.project.neighborhood_id,
short=award)).first()
if aw:
ids = [grant.granted_to_project_id for grant in
M.AwardGrant.query.find(dict(
granted_by_neighborhood_id=c.project.neighborhood_id,
award_id=aw._id))]
if '_id' in q:
ids = list(set(q['_id']['$in']).intersection(ids))
q['_id'] = {'$in': ids}
if trove is not None:
q['trove_' + trove.type] = trove._id
sort_key, sort_dir = 'last_updated', pymongo.DESCENDING
if sort == 'alpha':
sort_key, sort_dir = 'name', pymongo.ASCENDING
elif sort == 'random':
sort_key, sort_dir = None, None
elif sort == 'last_registered':
sort_key, sort_dir = '_id', pymongo.DESCENDING
elif sort == '_id':
sort_key, sort_dir = '_id', pymongo.DESCENDING
projects = []
if private:
# Only return private projects.
# Can't filter these with a mongo query directly - have to iterate
# through and check the ACL of each project.
for chunk in utils.chunked_find(M.Project, q, sort_key=sort_key,
sort_dir=sort_dir):
projects.extend([p for p in chunk if p.private])
total = len(projects)
if sort == 'random':
projects = random.sample(projects, min(limit, total))
else:
projects = projects[:limit]
else:
total = None
if sort == 'random':
# MongoDB doesn't have a random sort built in, so...
# 1. Do a direct pymongo query (faster than ORM) to fetch just the
# _ids of objects that match our criteria
# 2. Choose a random sample of those _ids
# 3. Do an ORM query to fetch the objects with those _ids
# 4. Shuffle the results
from ming.orm import mapper
m = mapper(M.Project)
collection = M.main_doc_session.db[m.collection.m.collection_name]
docs = list(collection.find(q, {'_id': 1}))
if docs:
ids = [doc['_id'] for doc in
random.sample(docs, min(limit, len(docs)))]
if '_id' in q:
ids = list(set(q['_id']['$in']).intersection(ids))
q['_id'] = {'$in': ids}
projects = M.Project.query.find(q).all()
random.shuffle(projects)
else:
projects = M.Project.query.find(q).limit(limit).sort(sort_key,
sort_dir).all()
pl = ProjectList()
g.resource_manager.register(pl)
response = pl.display(projects=projects, display_mode=display_mode,
columns=columns, show_proj_icon=show_proj_icon,
show_download_button=show_download_button,
show_awards_banner=show_awards_banner,
grid_view_tools=grid_view_tools)
if show_total:
if total is None:
total = 0
for p in M.Project.query.find(q):
if h.has_access(p, 'read')():
total = total + 1
response = '<p class="macro_projects_total">%s Projects</p>%s' % \
(total, response)
return response
def refresh_repo(repo, all_commits=False, notify=True, new_clone=False):
all_commit_ids = commit_ids = list(repo.all_commit_ids())
if not commit_ids:
# the repo is empty, no need to continue
return
new_commit_ids = unknown_commit_ids(commit_ids)
stats_log = h.log_action(log, 'commit')
for ci in new_commit_ids:
stats_log.info('', meta=dict(module='scm-%s' % repo.repo_id, read='0'))
if not all_commits:
# Skip commits that are already in the DB
commit_ids = new_commit_ids
log.info('Refreshing %d commits on %s', len(commit_ids), repo.full_fs_path)
# Refresh commits
seen = set()
for i, oid in enumerate(commit_ids):
repo.refresh_commit_info(oid, seen, not all_commits)
if (i + 1) % 100 == 0:
log.info('Refresh commit info %d: %s', (i + 1), oid)
refresh_commit_repos(all_commit_ids, repo)
# Refresh child references
for i, oid in enumerate(commit_ids):
ci = CommitDoc.m.find(dict(_id=oid), validate=False).next()
refresh_children(ci)
if (i + 1) % 100 == 0:
log.info('Refresh child info %d for parents of %s', (i + 1),
ci._id)
if repo._refresh_precompute:
# Refresh commit runs
commit_run_ids = commit_ids
# Check if the CommitRuns for the repo are in a good state by checking for
# a CommitRunDoc that contains the last known commit. If there isn't one,
# the CommitRuns for this repo are in a bad state - rebuild them entirely.
if commit_run_ids != all_commit_ids:
last_commit = last_known_commit_id(all_commit_ids, new_commit_ids)
log.info('Last known commit id: %s', last_commit)
if not CommitRunDoc.m.find(dict(commit_ids=last_commit)).count():
log.info('CommitRun incomplete, rebuilding with all commits')
commit_run_ids = all_commit_ids
log.info('Starting CommitRunBuilder for %s', repo.full_fs_path)
rb = CommitRunBuilder(commit_run_ids)
rb.run()
rb.cleanup()
log.info('Finished CommitRunBuilder for %s', repo.full_fs_path)
# Refresh trees
# Like diffs below, pre-computing trees for some SCMs is too expensive,
# so we skip it here, then do it on-demand later.
if repo._refresh_precompute:
cache = {}
for i, oid in enumerate(commit_ids):
ci = CommitDoc.m.find(dict(_id=oid), validate=False).next()
cache = refresh_commit_trees(ci, cache)
if (i + 1) % 100 == 0:
log.info('Refresh commit trees %d: %s', (i + 1), ci._id)
# Compute diffs
cache = {}
# For some SCMs, we don't want to pre-compute the diffs because that
# would be too expensive, so we skip them here and do them on-demand
# with caching.
if repo._refresh_precompute:
for i, oid in enumerate(commit_ids):
cid = CommitDoc.m.find(dict(_id=oid), validate=False).next()
ci = mapper(Commit).create(cid, dict(instrument=False))
ci.set_context(repo)
compute_diffs(repo._id, cache, ci)
if (i + 1) % 100 == 0:
log.info('Compute diffs %d: %s', (i + 1), ci._id)
if repo._refresh_precompute:
model_cache = ModelCache()
lcid_cache = {}
for i, oid in enumerate(reversed(commit_ids)):
ci = model_cache.get(Commit, dict(_id=oid))
ci.set_context(repo)
compute_lcds(ci, model_cache, lcid_cache)
ThreadLocalORMSession.flush_all()
if (i + 1) % 100 == 0:
log.info('Compute last commit info %d: %s', (i + 1), ci._id)
if not all_commits and not new_clone:
for commit in commit_ids:
new = repo.commit(commit)
user = User.by_email_address(new.committed.email)
if user is None:
user = User.by_username(new.committed.name)
if user is not None:
g.statsUpdater.newCommit(new, repo.app_config.project, user)
log.info('Refresh complete for %s', repo.full_fs_path)
g.post_event('repo_refreshed', len(commit_ids), all_commits, new_clone)
# Send notifications
if notify:
send_notifications(repo, commit_ids)
# well (i.e., . and $ not allowed)
entries = [{'name': name, 'commit_id': value}
for name, value in entries.iteritems()]
lcd = cls(
commit_id=tree.commit._id,
path=path,
entries=entries,
)
model_cache.set(cls, {'path': path, 'commit_id': tree.commit._id}, lcd)
return lcd
@LazyProperty
def by_name(self):
return {n.name: n.commit_id for n in self.entries}
mapper(Commit, CommitDoc, repository_orm_session)
mapper(Tree, TreeDoc, repository_orm_session)
mapper(LastCommit, LastCommitDoc, repository_orm_session)
class ModelCache(object):
'''
Cache model instances based on query params passed to get.
'''
def __init__(self, max_instances=None, max_queries=None):
'''
By default, each model type can have 2000 instances and
8000 queries. You can override these for specific model
types by passing in a dict() for either max_instances or
'commit_id': value
} for name, value in entries.iteritems()]
lcd = cls(
commit_id=tree.commit._id,
path=path,
entries=entries,
)
model_cache.set(cls, {'path': path, 'commit_id': tree.commit._id}, lcd)
return lcd
@LazyProperty
def by_name(self):
return {n.name: n.commit_id for n in self.entries}
mapper(Commit, CommitDoc, repository_orm_session)
mapper(Tree, TreeDoc, repository_orm_session)
mapper(LastCommit, LastCommitDoc, repository_orm_session)
class ModelCache(object):
'''
Cache model instances based on query params passed to get.
'''
def __init__(self, max_instances=None, max_queries=None):
'''
By default, each model type can have 2000 instances and
8000 queries. You can override these for specific model
types by passing in a dict() for either max_instances or
max_queries keyed by the class(es) with the max values.
Classes not in the dict() will use the default 2000/8000
project_id=p_id,
app=parts[1],
artifact=parts[2])
elif len(parts) == 2:
return dict(nbhd=p_nbhd,
project=p_shortname,
project_id=p_id,
app=parts[0],
artifact=parts[1])
elif len(parts) == 1:
return dict(nbhd=p_nbhd,
project=p_shortname,
project_id=p_id,
app=None,
artifact=parts[0])
else:
return None
# Mapper definitions
mapper(ArtifactReference, ArtifactReferenceDoc, main_orm_session)
mapper(Shortlink,
ShortlinkDoc,
main_orm_session,
properties=dict(ref_id=ForeignIdProperty(ArtifactReference),
project_id=ForeignIdProperty('Project'),
app_config_id=ForeignIdProperty('AppConfig'),
project=RelationProperty('Project'),
app_config=RelationProperty('AppConfig'),
ref=RelationProperty(ArtifactReference)))
def get_fields(self, entity):
"""Get all of the fields for a given entity."""
if inspect.isfunction(entity):
entity = entity()
return [prop.name for prop in mapper(entity).properties if isinstance(prop, ORMProperty)]
def get_field(self, entity, name):
"""Get a field with the given field name."""
return mapper(entity).property_index[name]
def get_relations(self, entity):
"""Get all of the field names in an enity which are related to other entities."""
return [prop.name for prop in mapper(entity).properties if isinstance(prop, RelationProperty)]
project=parts[0],
project_id=p_id,
app=parts[1],
artifact=parts[2])
elif len(parts) == 2:
return dict(
nbhd=p_nbhd,
project=p_shortname,
project_id=p_id,
app=parts[0],
artifact=parts[1])
elif len(parts) == 1:
return dict(
nbhd=p_nbhd,
project=p_shortname,
project_id=p_id,
app=None,
artifact=parts[0])
else:
return None
# Mapper definitions
mapper(ArtifactReference, ArtifactReferenceDoc, main_orm_session)
mapper(Shortlink, ShortlinkDoc, main_orm_session, properties=dict(
ref_id=ForeignIdProperty(ArtifactReference),
project_id=ForeignIdProperty('Project'),
app_config_id=ForeignIdProperty('AppConfig'),
project=RelationProperty('Project'),
app_config=RelationProperty('AppConfig'),
ref=RelationProperty(ArtifactReference)))
def get_projects_for_macro(category=None,
sort='last_updated',
show_total=False,
limit=100,
labels='',
award='',
private=False,
columns=1,
show_proj_icon=True,
show_download_button=False,
show_awards_banner=True,
initial_q={}):
from allura.lib.widgets.project_list import ProjectList
from allura.lib import utils
from allura import model as M
# 'trove' is internal substitution for 'category' filter in wiki macro
trove = category
limit = int(limit)
q = dict(deleted=False, is_nbhd_project=False)
q.update(initial_q)
if labels:
or_labels = labels.split('|')
q['$or'] = [{'labels': {'$all': l.split(',')}} for l in or_labels]
if trove is not None:
trove = M.TroveCategory.query.get(fullpath=trove)
if award:
aw = M.Award.query.find(
dict(created_by_neighborhood_id=c.project.neighborhood_id,
short=award)).first()
if aw:
ids = [
grant.granted_to_project_id
for grant in M.AwardGrant.query.find(
dict(granted_by_neighborhood_id=c.project.neighborhood_id,
award_id=aw._id))
]
if '_id' in q:
ids = list(set(q['_id']['$in']).intersection(ids))
q['_id'] = {'$in': ids}
if trove is not None:
q['trove_' + trove.type] = trove._id
sort_key, sort_dir = 'last_updated', pymongo.DESCENDING
if sort == 'alpha':
sort_key, sort_dir = 'name', pymongo.ASCENDING
elif sort == 'random':
sort_key, sort_dir = None, None
elif sort == 'last_registered':
sort_key, sort_dir = '_id', pymongo.DESCENDING
elif sort == '_id':
sort_key, sort_dir = '_id', pymongo.DESCENDING
projects = []
if private:
# Only return private projects.
# Can't filter these with a mongo query directly - have to iterate
# through and check the ACL of each project.
for chunk in utils.chunked_find(M.Project,
q,
sort_key=sort_key,
sort_dir=sort_dir):
projects.extend([p for p in chunk if p.private])
total = len(projects)
if sort == 'random':
projects = random.sample(projects, min(limit, total))
else:
projects = projects[:limit]
else:
total = None
if sort == 'random':
# MongoDB doesn't have a random sort built in, so...
# 1. Do a direct pymongo query (faster than ORM) to fetch just the
# _ids of objects that match our criteria
# 2. Choose a random sample of those _ids
# 3. Do an ORM query to fetch the objects with those _ids
# 4. Shuffle the results
from ming.orm import mapper
m = mapper(M.Project)
collection = M.main_doc_session.db[m.collection.m.collection_name]
docs = list(collection.find(q, {'_id': 1}))
if docs:
ids = [
doc['_id']
for doc in random.sample(docs, min(limit, len(docs)))
]
if '_id' in q:
ids = list(set(q['_id']['$in']).intersection(ids))
q['_id'] = {'$in': ids}
projects = M.Project.query.find(q).all()
random.shuffle(projects)
else:
projects = M.Project.query.find(q).limit(limit).sort(
sort_key, sort_dir).all()
pl = ProjectList()
g.resource_manager.register(pl)
response = pl.display(
projects=projects,
columns=columns,
show_proj_icon=show_proj_icon,
show_download_button=show_download_button,
show_awards_banner=show_awards_banner,
)
if show_total:
if total is None:
total = 0
for p in M.Project.query.find(q):
if h.has_access(p, 'read')():
total = total + 1
response = '<p class="macro_projects_total">%s Projects</p>%s' % \
(total, response)
return response
def refresh_repo(repo, all_commits=False, notify=True, new_clone=False):
all_commit_ids = commit_ids = list(repo.all_commit_ids())
if not commit_ids:
# the repo is empty, no need to continue
return
new_commit_ids = unknown_commit_ids(commit_ids)
stats_log = h.log_action(log, "commit")
for ci in new_commit_ids:
stats_log.info("", meta=dict(module="scm-%s" % repo.repo_id, read="0"))
if not all_commits:
# Skip commits that are already in the DB
commit_ids = new_commit_ids
log.info("Refreshing %d commits on %s", len(commit_ids), repo.full_fs_path)
# Refresh commits
seen = set()
for i, oid in enumerate(commit_ids):
repo.refresh_commit_info(oid, seen, not all_commits)
if (i + 1) % 100 == 0:
log.info("Refresh commit info %d: %s", (i + 1), oid)
refresh_commit_repos(all_commit_ids, repo)
# Refresh child references
for i, oid in enumerate(commit_ids):
ci = CommitDoc.m.find(dict(_id=oid), validate=False).next()
refresh_children(ci)
if (i + 1) % 100 == 0:
log.info("Refresh child info %d for parents of %s", (i + 1), ci._id)
if repo._refresh_precompute:
# Refresh commit runs
commit_run_ids = commit_ids
# Check if the CommitRuns for the repo are in a good state by checking for
# a CommitRunDoc that contains the last known commit. If there isn't one,
# the CommitRuns for this repo are in a bad state - rebuild them entirely.
if commit_run_ids != all_commit_ids:
last_commit = last_known_commit_id(all_commit_ids, new_commit_ids)
log.info("Last known commit id: %s", last_commit)
if not CommitRunDoc.m.find(dict(commit_ids=last_commit)).count():
log.info("CommitRun incomplete, rebuilding with all commits")
commit_run_ids = all_commit_ids
log.info("Starting CommitRunBuilder for %s", repo.full_fs_path)
rb = CommitRunBuilder(commit_run_ids)
rb.run()
rb.cleanup()
log.info("Finished CommitRunBuilder for %s", repo.full_fs_path)
# Refresh trees
# Like diffs below, pre-computing trees for some SCMs is too expensive,
# so we skip it here, then do it on-demand later.
if repo._refresh_precompute:
cache = {}
for i, oid in enumerate(commit_ids):
ci = CommitDoc.m.find(dict(_id=oid), validate=False).next()
cache = refresh_commit_trees(ci, cache)
if (i + 1) % 100 == 0:
log.info("Refresh commit trees %d: %s", (i + 1), ci._id)
# Compute diffs
cache = {}
# For some SCMs, we don't want to pre-compute the diffs because that
# would be too expensive, so we skip them here and do them on-demand
# with caching.
if repo._refresh_precompute:
for i, oid in enumerate(commit_ids):
cid = CommitDoc.m.find(dict(_id=oid), validate=False).next()
ci = mapper(Commit).create(cid, dict(instrument=False))
ci.set_context(repo)
compute_diffs(repo._id, cache, ci)
if (i + 1) % 100 == 0:
log.info("Compute diffs %d: %s", (i + 1), ci._id)
if repo._refresh_precompute:
model_cache = ModelCache()
lcid_cache = {}
for i, oid in enumerate(reversed(commit_ids)):
ci = model_cache.get(Commit, dict(_id=oid))
ci.set_context(repo)
compute_lcds(ci, model_cache, lcid_cache)
ThreadLocalORMSession.flush_all()
if (i + 1) % 100 == 0:
log.info("Compute last commit info %d: %s", (i + 1), ci._id)
if not all_commits and not new_clone:
for commit in commit_ids:
new = repo.commit(commit)
user = User.by_email_address(new.committed.email)
if user is None:
user = User.by_username(new.committed.name)
if user is not None:
g.statsUpdater.newCommit(new, repo.app_config.project, user)
log.info("Refresh complete for %s", repo.full_fs_path)
g.post_event("repo_refreshed", len(commit_ids), all_commits, new_clone)
# Send notifications
if notify:
send_notifications(repo, commit_ids)
