Example #1
0
    def get(self, request, *args, **kwargs):
        out = super(DocketView, self).get(request, *args, **kwargs).data

        out['source'] = self.item.source
        stats = out['stats']
        stats['similar_dockets'] = []
        summaries = []

        out['upstream_urls'] = []
        if out['source'] == 'regulations.gov':
            out['upstream_urls'].append({
                'url':
                'http://www.regulations.gov/#!docketDetail;D=' + self.item.id,
                'label':
                'Regulations.gov'
            })
        elif out['source'] == 'sec_cftc':
            if 'Source_URL' in self.item.details:
                out['upstream_urls'].append({
                    'url':
                    self.item.details['Source_URL'],
                    'label':
                    'SEC.gov' if self.item.agency == 'SEC' else 'CFTC.gov'
                })
            for replaced in self.item.suppression.get('replaces', []):
                out['upstream_urls'].append({
                    'url':
                    'http://www.regulations.gov/#!docketDetail;D=' + replaced,
                    'label':
                    'Regulations.gov'
                })

        if stats['count'] > 0:
            # do a similar thing with FR documents
            if stats.get('doc_info', {}).get('fr_docs', None):
                fr_doc_ids = [
                    doc['id'] for doc in stats['doc_info']['fr_docs']
                ]
                fr_search = Doc.objects(id__in=fr_doc_ids)
                fr_docs = dict([(fr_doc.id, fr_doc) for fr_doc in fr_search])

                for doc in stats['doc_info']['fr_docs']:
                    if doc['id'] in fr_docs:
                        fr_doc = fr_docs[doc['id']]
                        doc['stats'] = {
                            'date_range': fr_doc.stats['date_range'],
                            'count': fr_doc.stats['count']
                        } if fr_doc.stats else {
                            'count': 0
                        }

                        if fr_doc.annotations.get('fr_data', None):
                            doc['summary'] = fr_doc.annotations['fr_data'].get(
                                'abstract', None)

                        if not doc.get('summary', None):
                            doc['summary'] = fr_doc.get_summary()

                        doc['comments_open'] = 'Comment_Due_Date' in fr_doc.details and force_date(
                            fr_doc.details['Comment_Due_Date']
                        ) > datetime.datetime.now()

                        if doc['summary']:
                            summaries.append(doc['summary'])
                    else:
                        doc['stats'] = {'count': 0, 'comments_open': False}
                        doc['summary'] = None

                # remove duplicates, if any
                tmp = stats['doc_info']['fr_docs']
                included = set()
                stats['doc_info']['fr_docs'] = []
                for doc in tmp:
                    if doc['id'] not in included:
                        stats['doc_info']['fr_docs'].append(doc)
                        included.add(doc['id'])

            summary_text = "\n".join(summaries)
            if summary_text:
                similar_dockets = get_similar_dockets(
                    summary_text, kwargs[self.aggregation_field])[:3]
                if similar_dockets:
                    sd = dict([(docket.id, docket.title)
                               for docket in Docket.objects(
                                   id__in=similar_dockets).only('id', 'title')
                               ])
                    stats['similar_dockets'] = [{
                        'id': docket,
                        'title': sd[docket]
                    } for docket in similar_dockets]

        agency = self.item.agency
        if not agency:
            agency = re.split("[-_]", self.item.id)[0]

        if agency:
            agency_meta = list(Agency.objects(id=agency).only("name"))
            if agency_meta:
                out['agency'] = {
                    'id': agency,
                    'name': agency_meta[0].name,
                    'url': '/agency/%s' % agency
                }
            else:
                agency = None

        if not agency:
            out['agency'] = None

        return Response(out)
Example #2
0
    def get(self, request, *args, **kwargs):
        out = super(DocketView, self).get(request, *args, **kwargs).data

        out['source'] = self.item.source
        stats = out['stats']
        stats['similar_dockets'] = []
        summaries = []

        out['upstream_urls'] = []
        if out['source'] == 'regulations.gov':
            out['upstream_urls'].append({
                'url': 'http://www.regulations.gov/#!docketDetail;D=' + self.item.id,
                'label': 'Regulations.gov'
            })
        elif out['source'] == 'sec_cftc':
            if 'Source_URL' in self.item.details:
                out['upstream_urls'].append({
                    'url': self.item.details['Source_URL'],
                    'label': 'SEC.gov' if self.item.agency == 'SEC' else 'CFTC.gov'
                })
            for replaced in self.item.suppression.get('replaces', []):
                out['upstream_urls'].append({
                    'url': 'http://www.regulations.gov/#!docketDetail;D=' + replaced,
                    'label': 'Regulations.gov'
                })


        if stats['count'] > 0:
            # do a similar thing with FR documents
            if stats.get('doc_info', {}).get('fr_docs', None):
                fr_doc_ids = [doc['id'] for doc in stats['doc_info']['fr_docs']]
                fr_search = Doc.objects(id__in=fr_doc_ids)
                fr_docs = dict([(fr_doc.id, fr_doc) for fr_doc in fr_search])

                for doc in stats['doc_info']['fr_docs']:
                    if doc['id'] in fr_docs:
                        fr_doc = fr_docs[doc['id']]
                        doc['stats'] = {
                            'date_range': fr_doc.stats['date_range'],
                            'count': fr_doc.stats['count']
                        } if fr_doc.stats else {'count': 0}
                        
                        if fr_doc.annotations.get('fr_data', None):
                            doc['summary'] = fr_doc.annotations['fr_data'].get('abstract', None)

                        if not doc.get('summary', None):
                            doc['summary'] = fr_doc.get_summary()

                        doc['comments_open'] = 'Comment_Due_Date' in fr_doc.details and force_date(fr_doc.details['Comment_Due_Date']) > datetime.datetime.now()

                        if doc['summary']:
                            summaries.append(doc['summary'])
                    else:
                        doc['stats'] = {'count': 0, 'comments_open': False}
                        doc['summary'] = None

                # remove duplicates, if any
                tmp = stats['doc_info']['fr_docs']
                included = set()
                stats['doc_info']['fr_docs'] = []
                for doc in tmp:
                    if doc['id'] not in included:
                        stats['doc_info']['fr_docs'].append(doc)
                        included.add(doc['id'])

            summary_text = "\n".join(summaries)
            if summary_text:
                similar_dockets = get_similar_dockets(summary_text, kwargs[self.aggregation_field])[:3]
                if similar_dockets:
                    sd = dict([(docket.id, docket.title) for docket in Docket.objects(id__in=similar_dockets).only('id', 'title')])
                    stats['similar_dockets'] = [{
                        'id': docket,
                        'title': sd[docket]
                    } for docket in similar_dockets]

        agency = self.item.agency
        if not agency:
            agency = re.split("[-_]", self.item.id)[0]
        
        if agency:
            agency_meta = list(Agency.objects(id=agency).only("name"))
            if agency_meta:
                out['agency'] = {
                    'id': agency,
                    'name': agency_meta[0].name,
                    'url': '/agency/%s' % agency
                }
            else:
                agency = None
        
        if not agency:
            out['agency'] = None

        return Response(out)
Example #3
0
    def get(self, request, *args, **kwargs):
        out = super(DocketView, self).get(request, *args, **kwargs).data

        stats = out['stats']
        stats['similar_dockets'] = []
        summaries = []

        if stats['count'] > 0:
            # do a similar thing with FR documents
            if stats.get('doc_info', {}).get('fr_docs', None):
                fr_doc_ids = [doc['id'] for doc in stats['doc_info']['fr_docs']]
                fr_search = Doc.objects(id__in=fr_doc_ids)
                fr_docs = dict([(fr_doc.id, fr_doc) for fr_doc in fr_search])

                for doc in stats['doc_info']['fr_docs']:
                    if doc['id'] in fr_docs:
                        fr_doc = fr_docs[doc['id']]
                        doc['stats'] = {
                            'date_range': fr_doc.stats['date_range'],
                            'count': fr_doc.stats['count']
                        } if fr_doc.stats else {'count': 0}
                        doc['summary'] = fr_doc.get_summary()
                        doc['comments_open'] = 'Comment_Due_Date' in fr_doc.details and fr_doc.details['Comment_Due_Date'] > datetime.datetime.now()

                        if doc['summary']:
                            summaries.append(doc['summary'])
                    else:
                        doc['stats'] = {'count': 0, 'comments_open': False}
                        doc['summary'] = None

                # remove duplicates, if any
                tmp = stats['doc_info']['fr_docs']
                included = set()
                stats['doc_info']['fr_docs'] = []
                for doc in tmp:
                    if doc['id'] not in included:
                        stats['doc_info']['fr_docs'].append(doc)
                        included.add(doc['id'])

            summary_text = "\n".join(summaries)
            if summary_text:
                similar_dockets = get_similar_dockets(summary_text, kwargs[self.aggregation_field])[:3]
                if similar_dockets:
                    sd = dict([(docket.id, docket.title) for docket in Docket.objects(id__in=similar_dockets).only('id', 'title')])
                    stats['similar_dockets'] = [{
                        'id': docket,
                        'title': sd[docket]
                    } for docket in similar_dockets]

        agency = self.item.agency
        if not agency:
            agency = re.split("[-_]", self.item.id)[0]
        
        if agency:
            agency_meta = list(Agency.objects(id=agency).only("name"))
            if agency_meta:
                out['agency'] = {
                    'id': agency,
                    'name': agency_meta[0].name,
                    'url': '/agency/%s' % agency
                }
            else:
                agency = None
        
        if not agency:
            out['agency'] = None

        return Response(out)