def test_no_dereference_context_manager_object_id(self): """Ensure that DBRef items in ListFields aren't dereferenced. """ connect('mongoenginetest') class User(Document): name = StringField() class Group(Document): ref = ReferenceField(User, dbref=False) generic = GenericReferenceField() members = ListField(ReferenceField(User, dbref=False)) User.drop_collection() Group.drop_collection() for i in xrange(1, 51): User(name='user %s' % i).save() user = User.objects.first() Group(ref=user, members=User.objects, generic=user).save() with no_dereference(Group) as NoDeRefGroup: self.assertTrue(Group._fields['members']._auto_dereference) self.assertFalse(NoDeRefGroup._fields['members']._auto_dereference) with no_dereference(Group) as Group: group = Group.objects.first() self.assertTrue(all([not isinstance(m, User) for m in group.members])) self.assertFalse(isinstance(group.ref, User)) self.assertFalse(isinstance(group.generic, User)) self.assertTrue(all([isinstance(m, User) for m in group.members])) self.assertTrue(isinstance(group.ref, User)) self.assertTrue(isinstance(group.generic, User))
def test_no_dereference_context_manager_dbref(self): """Ensure that DBRef items in ListFields aren't dereferenced. """ connect('mongoenginetest') class User(Document): name = StringField() class Group(Document): ref = ReferenceField(User, dbref=True) generic = GenericReferenceField() members = ListField(ReferenceField(User, dbref=True)) User.drop_collection() Group.drop_collection() for i in range(1, 51): User(name='user %s' % i).save() user = User.objects.first() Group(ref=user, members=User.objects, generic=user).save() with no_dereference(Group) as NoDeRefGroup: self.assertTrue(Group._fields['members']._auto_dereference) self.assertFalse(NoDeRefGroup._fields['members']._auto_dereference) with no_dereference(Group) as Group: group = Group.objects.first() self.assertTrue( all([not isinstance(m, User) for m in group.members])) self.assertFalse(isinstance(group.ref, User)) self.assertFalse(isinstance(group.generic, User)) self.assertTrue(all([isinstance(m, User) for m in group.members])) self.assertTrue(isinstance(group.ref, User)) self.assertTrue(isinstance(group.generic, User))
def test_no_dereference_context_manager_object_id(self): """Ensure that DBRef items in ListFields aren't dereferenced.""" connect("mongoenginetest") class User(Document): name = StringField() class Group(Document): ref = ReferenceField(User, dbref=False) generic = GenericReferenceField() members = ListField(ReferenceField(User, dbref=False)) User.drop_collection() Group.drop_collection() for i in range(1, 51): User(name="user %s" % i).save() user = User.objects.first() Group(ref=user, members=User.objects, generic=user).save() with no_dereference(Group) as NoDeRefGroup: assert Group._fields["members"]._auto_dereference assert not NoDeRefGroup._fields["members"]._auto_dereference with no_dereference(Group) as Group: group = Group.objects.first() for m in group.members: assert not isinstance(m, User) assert not isinstance(group.ref, User) assert not isinstance(group.generic, User) for m in group.members: assert isinstance(m, User) assert isinstance(group.ref, User) assert isinstance(group.generic, User)
def ticket_view(request, ombudsman_slug, ticket_slug): if request.method == 'POST' and request.user.is_authenticated(): p = request.POST comment = Comment() comment.text = p['comment'].replace('\r\n', '<br>') comment.created_by = request.user.to_dbref() comment.created_in = datetime.now() Ticket.objects(slug = ticket_slug).update_one(push__comments = comment) with no_dereference(Ticket) as Ticket_all: obj = get_document_or_404(Ticket_all, slug = ticket_slug) obj.description = obj.description.replace('\r\n', '<br>') context = Context({'page_title': obj.title, 'current_page': 'ombudsman', 'obj': obj}); return render(request, 'ticket_view.html', context)
def analytics(): sort = request.args.get('s', 'r') date_from = request.args.get('f', False) date_to = request.args.get('t', False) t = timezone(g.current_account.timezone) print(sort) buletins = g.current_account.bulletins stories = [] for buletin in buletins: for content in buletin.content: if date_from and buletin.publish_at < datetime.strptime( date_from, '%m/%d/%Y'): continue if date_to and buletin.publish_at > datetime.strptime( date_to, '%m/%d/%Y'): continue s = {} s['id'] = content.id s['title'] = content.title s['publish_at'] = utc.localize(buletin.publish_at).astimezone(t) #print(content.content) with no_dereference(Story): s['readers'] = len(content.readers) s['readers_by_fragment'] = [] #f_p = 1 #s_content = sorted(content.content, key=lambda x: x.order) #for f in s_content: # if f.type != Fragment.TYPE_ANSWER: # s['readers_by_fragment'].append({'id':f_p, 'num_readers':f.num_readers}) # f_p += 1 stories.append(s) if sort == 'r': stories.sort(key=lambda x: x['readers'], reverse=True) else: stories.sort(key=lambda x: x['publish_at'], reverse=True) return render_template('analytics/index.html', stories=stories, sort=sort, date_from=date_from, date_to=date_to)
def ombudsman_view(request, slug): if request.method == 'POST' and request.user.is_authenticated() and request.user.group.name == 'users': p = request.POST ombudsman = OmbudsmanOffice.objects(slug = slug).fields('users').limit(1)[0] if 'action' in p and p['action'] == 'join': SubscribeUserInOmbudsmanOffice(slug, request.user.to_dbref()) elif 'action' in p and p['action'] == 'unjoin': UnsubscribeUserInOmbudsmanOffice(slug, request.user.to_dbref()) elif set(['title', 'description', 'category']).issubset(p): ticket = Ticket() ticket.title = p['title'] ticket.slug = '' ticket.description = p['description'] ticket.category = p['category'] ticket.belongs_to = ombudsman.to_dbref() ticket.created_by = request.user.to_dbref() ticket.comments = [] ticket.log = Log() ticket.save() ticket_slug = slugify('{0}-{1}'.format(ticket.title, ticket.obj_id)) ticket.update( set__slug = ticket_slug ) return HttpResponseRedirect('/ombudsman/{0}/ticket/{1}'.format(ombudsman.slug, ticket_slug)) with no_dereference(OmbudsmanOffice) as OmbudsmanOffice_all: obj = get_document_or_404(OmbudsmanOffice_all, slug = slug) obj.total_tickets = len(Ticket.objects(belongs_to = obj.id)) obj.total_users = len(obj.users) if obj.about: obj.about = obj.about.replace('\r\n', '<br>') context = Context({'page_title': obj.name, 'current_page': 'ombudsman', 'obj': obj}); return render(request, 'ombudsman_view.html', context)
def get(self, project): """Retrieve overview graph for a project. --- operationId: get_graph parameters: - name: project in: path type: string pattern: '^[a-zA-Z0-9_]{3,30}$' required: true description: project name/slug - name: columns in: query type: array items: type: string required: true description: comma-separated list of column names to plot (in MongoDB dot notation) - name: filters in: query type: array items: type: string description: list of `column__operator:value` filters \ with `column` in dot notation and `operator` in mongoengine format \ (http://docs.mongoengine.org/guide/querying.html#query-operators). \ `column` needs to be a valid field in `content.data`. - name: page in: query type: integer default: 1 description: page to retrieve (in batches of `per_page`) - name: per_page in: query type: integer default: 200 minimum: 2 maximum: 200 description: number of results to return per page responses: 200: description: x-y-data in plotly format schema: type: array items: type: object properties: x: type: array items: type: number y: type: array items: type: number """ mask = ['content.data', 'identifier'] columns = request.args.get('columns').split(',') filters = request.args.get('filters', '').split(',') page = int(request.args.get('page', 1)) PER_PAGE_MAX = 200 per_page = int(request.args.get('per_page', PER_PAGE_MAX)) per_page = PER_PAGE_MAX if per_page > PER_PAGE_MAX else per_page with no_dereference(Contributions) as ContributionsDeref: objects = ContributionsDeref.objects(project=project).only(*mask) data = [{'x': [], 'y': [], 'text': []} for col in columns] # C__gte:0.42,C__lte:2.10,ΔE-QP.direct__lte:11.3 -> content__data__C__value__lte if filters: query = {} for f in filters: if '__' in f and ':' in f: k, v = f.split(':') col, op = k.rsplit('__', 1) col = col.replace(".", "__") key = f'content__data__{col}__value__{op}' query[key] = float(v) objects = objects(**query) for obj in objects.paginate(page=page, per_page=per_page).items: d = nested_to_record(obj['content']['data'], sep='.') if all(f'{c}.display' in d.keys() for c in columns): for idx, col in enumerate(columns): val = d.get(f'{col}.display') if val: data[idx]['x'].append(obj.identifier) data[idx]['y'].append(val.split(' ')[0]) data[idx]['text'].append(str(obj.id)) return data
def build(): with no_dereference(Contributions) as Contribs: # TODO get a random max_docs slice to avoid collisions in parallel Fargate tasks # remove dangling and unset missing notebooks nbs_total, nbs_count = -1, -1 ctrbs_cnt = Contribs.objects._cursor.collection.estimated_document_count( ) nbs_cnt = Notebooks.objects._cursor.collection.estimated_document_count( ) if ctrbs_cnt != nbs_cnt: contribs = Contribs.objects(notebook__exists=True).only("notebook") nids = [contrib.notebook.id for contrib in contribs] if len(nids) < nbs_cnt: nbs = Notebooks.objects(id__nin=nids).only("id") nbs_total = nbs.count() max_docs = 2500 nbs[:max_docs].delete() nbs_count = nbs_total if nbs_total < max_docs else max_docs else: missing_nids = set(nids) - set( Notebooks.objects.distinct("id")) if missing_nids: upd_contribs = Contribs.objects( notebook__in=list(missing_nids)) nupd_total = upd_contribs.count() nupd = upd_contribs.update(unset__notebook="") print( f"unset notebooks for {nupd}/{nupd_total} contributions" ) # build missing notebooks max_docs = NotebooksResource.max_limit cids = request.args.get("cids", "").split(",")[:max_docs] if cids[0]: documents = Contribs.objects(id__in=cids) else: documents = Contribs.objects(notebook__exists=False)[:max_docs] total = documents.count() count = 0 for document in documents: if document.notebook is not None: # NOTE document.notebook.delete() doesn't trigger pre_delete signal? nb = Notebooks.objects.get(id=document.notebook.id) nb.delete() cells = [ # define client only once in kernel # avoids API calls for regex expansion for query parameters nbf.new_code_cell("\n".join([ "if 'client' not in locals():", "\tclient = Client(", '\t\theaders={"X-Authenticated-Groups": "admin"},', f'\t\thost="{MPCONTRIBS_API_HOST}"', "\t)", ])), nbf.new_code_cell( f'client.get_contribution("{document.id}").pretty()'), ] if document.tables: cells.append(nbf.new_markdown_cell("## Tables")) for table in document.tables: cells.append( nbf.new_code_cell("\n".join([ f'df = client.get_table("{table.id}")', "df.plot(**df.attrs)", ]))) if document.structures: cells.append(nbf.new_markdown_cell("## Structures")) for structure in document.structures: cells.append( nbf.new_code_cell( f'client.get_structure("{structure.id}")')) cid = str(document.id) outputs = execute_cells(cid, cells) if not outputs: raise ValueError(f"notebook generation for {cid} failed!") for idx, output in outputs.items(): cells[idx]["outputs"] = output doc = deepcopy(seed_nb) doc["cells"] += cells[1:] # skip localhost Client document.notebook = Notebooks(**doc).save() document.save(signal_kwargs={"skip": True}) count += 1 return f"{count}/{total} notebooks built & {nbs_count}/{nbs_total} notebooks deleted"
def archive_articles(limit=None): """ Archive articles that pollute the production database. """ raise NotImplementedError('REVIEW for RELDB.') # cf. https://docs.djangoproject.com/en/dev/topics/db/multi-db/#selecting-a-database-to-delete-from # NOQA counts = { 'duplicates': 0, 'orphaned': 0, 'bad_articles': 0, 'archived_dupes': 0, } if limit is None: limit = config.ARTICLE_ARCHIVE_BATCH_SIZE with no_dereference(Article) as ArticleOnly: if config.ARTICLE_ARCHIVE_OLDER_THAN > 0: older_than = now() - timedelta( days=config.ARTICLE_ARCHIVE_OLDER_THAN) duplicates = ArticleOnly.objects( duplicate_of__ne=None, date_published__lt=older_than).limit(limit) orphaned = ArticleOnly.objects( orphaned=True, date_published__lt=older_than).limit(limit) else: duplicates = ArticleOnly.objects( duplicate_of__ne=None).limit(limit) orphaned = ArticleOnly.objects(orphaned=True).limit(limit) duplicates.no_cache() orphaned.no_cache() counts['duplicates'] = duplicates.count() counts['orphaned'] = orphaned.count() if counts['duplicates']: current = 0 LOGGER.info(u'Archiving of %s duplicate article(s) started.', counts['duplicates']) with benchmark('Archiving of %s duplicate article(s)' % counts['duplicates']): for article in duplicates: archive_article_one_internal(article, counts) current += 1 if current % 50 == 0: LOGGER.info(u'Archived %s/%s duplicate articles so far.', current, counts['duplicates']) if counts['orphaned']: current = 0 LOGGER.info(u'Archiving of %s orphaned article(s) started.', counts['orphaned']) with benchmark('Archiving of %s orphaned article(s)' % counts['orphaned']): for article in orphaned: archive_article_one_internal(article, counts) current += 1 if current % 50 == 0: LOGGER.info(u'Archived %s/%s orphaned articles so far.', current, counts['duplicates']) if counts['duplicates'] or counts['orphaned']: synchronize_statsd_articles_gauges(full=True) LOGGER.info( '%s already archived and %s bad articles were found ' u'during the operation.', counts['archived_dupes'], counts['bad_articles']) else: LOGGER.info(u'No article to archive.')
def archive_articles(limit=None): """ Archive articles that pollute the production database. """ raise NotImplementedError('REVIEW for RELDB.') # cf. https://docs.djangoproject.com/en/dev/topics/db/multi-db/#selecting-a-database-to-delete-from # NOQA counts = { 'duplicates': 0, 'orphaned': 0, 'bad_articles': 0, 'archived_dupes': 0, } if limit is None: limit = config.ARTICLE_ARCHIVE_BATCH_SIZE with no_dereference(Article) as ArticleOnly: if config.ARTICLE_ARCHIVE_OLDER_THAN > 0: older_than = now() - timedelta( days=config.ARTICLE_ARCHIVE_OLDER_THAN) duplicates = ArticleOnly.objects( duplicate_of__ne=None, date_published__lt=older_than).limit(limit) orphaned = ArticleOnly.objects( orphaned=True, date_published__lt=older_than).limit(limit) else: duplicates = ArticleOnly.objects(duplicate_of__ne=None ).limit(limit) orphaned = ArticleOnly.objects(orphaned=True).limit(limit) duplicates.no_cache() orphaned.no_cache() counts['duplicates'] = duplicates.count() counts['orphaned'] = orphaned.count() if counts['duplicates']: current = 0 LOGGER.info(u'Archiving of %s duplicate article(s) started.', counts['duplicates']) with benchmark('Archiving of %s duplicate article(s)' % counts['duplicates']): for article in duplicates: archive_article_one_internal(article, counts) current += 1 if current % 50 == 0: LOGGER.info(u'Archived %s/%s duplicate articles so far.', current, counts['duplicates']) if counts['orphaned']: current = 0 LOGGER.info(u'Archiving of %s orphaned article(s) started.', counts['orphaned']) with benchmark('Archiving of %s orphaned article(s)' % counts['orphaned']): for article in orphaned: archive_article_one_internal(article, counts) current += 1 if current % 50 == 0: LOGGER.info(u'Archived %s/%s orphaned articles so far.', current, counts['duplicates']) if counts['duplicates'] or counts['orphaned']: synchronize_statsd_articles_gauges(full=True) LOGGER.info('%s already archived and %s bad articles were found ' u'during the operation.', counts['archived_dupes'], counts['bad_articles']) else: LOGGER.info(u'No article to archive.')
def archive_articles(limit=None): counts = { 'duplicates': 0, 'orphaned': 0, 'bad_articles': 0, 'archived_dupes': 0, } if limit is None: limit = config.ARTICLE_ARCHIVE_BATCH_SIZE with no_dereference(Article) as ArticleOnly: if config.ARTICLE_ARCHIVE_OLDER_THAN > 0: older_than = now() - timedelta( days=config.ARTICLE_ARCHIVE_OLDER_THAN) duplicates = ArticleOnly.objects( duplicate_of__ne=None, date_published__lt=older_than).limit(limit) orphaned = ArticleOnly.objects( orphaned=True, date_published__lt=older_than).limit(limit) else: duplicates = ArticleOnly.objects(duplicate_of__ne=None ).limit(limit) orphaned = ArticleOnly.objects(orphaned=True).limit(limit) duplicates.no_cache() orphaned.no_cache() counts['duplicates'] = duplicates.count() counts['orphaned'] = orphaned.count() if counts['duplicates']: current = 0 LOGGER.info(u'Archiving of %s duplicate article(s) started.', counts['duplicates']) with benchmark('Archiving of %s duplicate article(s)' % counts['duplicates']): for article in duplicates: archive_article_one_internal(article, counts) current += 1 if current % 50 == 0: LOGGER.info(u'Archived %s/%s duplicate articles so far.', current, counts['duplicates']) if counts['orphaned']: current = 0 LOGGER.info(u'Archiving of %s orphaned article(s) started.', counts['orphaned']) with benchmark('Archiving of %s orphaned article(s)' % counts['orphaned']): for article in orphaned: archive_article_one_internal(article, counts) current += 1 if current % 50 == 0: LOGGER.info(u'Archived %s/%s orphaned articles so far.', current, counts['duplicates']) if counts['duplicates'] or counts['orphaned']: synchronize_statsd_articles_gauges(full=True) LOGGER.info('%s already archived and %s bad articles were found ' u'during the operation.', counts['archived_dupes'], counts['bad_articles']) else: LOGGER.info(u'No article to archive.')