def fetch(self, params, offset=None, limit=None): results = [] if params.get('country'): params['country_json'] = '{"0":"' + params['country'] + '"}' params.update(self.params) # type is default resp = requests.get(self.URL, params=params) soup = Soup(resp.text, 'html.parser') contents = soup.findAll('ul', {'class': 'searchResults'}) if not contents: return results, 0 content = contents[0] for item in content.findAll('li', {'class': ['searchResultItem']}): itemcontent = item.find( 'div', {'class': ['searchResultItem_content', 'media_body']}) urlcontent = item.find('div', {'class': 'searchResultItem_download'}) datecontent = item.find('span', {'class': 'searchResultItem_date'}) title = itemcontent.find('a').get_text() pdfurl = urlcontent.find('a')['href'] raw_date = datecontent.find('b').get_text() # 4 July 2018 date = datetime.datetime.strptime(raw_date, '%d %B %Y') data = Lead( # FIXME: use proper key id=pdfurl, title=title.strip(), published_on=date.date(), url=pdfurl, source='UNHCR Portal', source_type='', website='data2.unhcr.org') results.append(data) return results, len(results)
def get_queryset(self): # TODO: Optimize this queryset return Lead.get_for(self.request.user).select_related( 'project', 'project__analysis_framework', 'project__analysis_framework__organization', )
def fetch(self, params, offset=None, limit=None): results = [] resp = requests.get(self.URL, params=params) soup = Soup(resp.text, 'html.parser') contents = soup.findAll('div', {'class': 'wrapper-type'}) if not contents: return results, 0 content = contents[0] for item in content.findAll('div', {'class': 'views-row'}): bottomcontent = item.find('div', {'class': 'content-bottom'}) topcontent = item.find('div', {'class': 'content-top'}) date = topcontent.find('span', {'class': 'updated-date'}).text date = datetime.datetime.strptime(date, '%d/%m/%Y') title = topcontent.find('div', {'class': 'field-item'}).text link = bottomcontent.find('div', {'class': 'field-item'}).find('a') data = Lead( # FIXME: use proper key id=link['href'], title=title.strip(), published_on=date.date(), url=link['href'], source='Briefing Notes', source_type=Lead.WEBSITE, website='www.acaps.org/special-reports') results.append(data) return results, len(results)
def fetch(self, params, page=None, limit=None): country = params.get('country') if not country: return [], 0 results = [] resp = requests.get(self.URL) soup = Soup(resp.text, 'html.parser') content = soup.find('tbody') for row in content.findAll('tr'): elem = row.find('a') name = elem.get_text() name = re.sub('(.*)', '', name) title = row.findAll('td')[-1].get_text() if name.strip() == country.strip(): # add as lead url = elem['href'] if url[0] == '/': # means relative path url = self.website + url data = Lead(title=title.strip(), url=url, source='PDNA portal', source_type=Lead.WEBSITE, website=self.website) results.append(data) return results, len(results)
def fetch(self, params, page=None, limit=None): results = [] if not params or not params.get('feed-url'): return results feed = feedparser.parse(params['feed-url']) title_field = params.get('title-field') date_field = params.get('date-field') source_field = params.get('source-field') url_field = params.get('url-field') website = params.get('website') for entry in feed.entries: title = title_field and entry.get(title_field) date = date_field and entry.get(date_field) source = source_field and entry.get(source_field) url = url_field and entry.get(url_field) data = Lead( title=title, published_on=date, source=source, url=url, website=website, source_type=Lead.RSS, ) results.append(data) return results
def get_leads(self, params) -> Tuple[List[Lead], int]: def _parse_date(date_raw) -> Union[None, datetime.date]: if type(date_raw) == datetime.date: return date_raw elif type(date_raw) == datetime.datetime: return date_raw.date() else: published_on = str_to_date(date_raw) if published_on: return published_on.date() leads_data, total_count = self.fetch(copy.deepcopy(params)) if not leads_data: return [], total_count organization_search = OrganizationSearch([ label for d in leads_data for label in [d['source'], d['author']] ]) leads = [] for ldata in leads_data: published_on = _parse_date(ldata['published_on']) lead = Lead( id=ldata.get('id', random_key()), title=ldata['title'], published_on=published_on, url=ldata['url'], source_raw=ldata['source'], author_raw=ldata['author'], source=organization_search.get(ldata['source']), author=organization_search.get(ldata['author']), source_type=ldata['source_type'], ) if ldata.get('author') is not None: lead._authors = list( filter(None, [organization_search.get(ldata['author'])])) # Add emm info if ldata.get('emm_triggers') is not None: lead._emm_triggers = ldata['emm_triggers'] if ldata.get('emm_entities') is not None: lead._emm_entities = ldata['emm_entities'] leads.append(lead) return leads, total_count
def get_queryset(self): leads = Lead.get_for(self.request.user) lead_id = self.request.GET.get('similar') if lead_id: similar_lead = Lead.objects.get(id=lead_id) leads = leads.filter(project=similar_lead.project).annotate( similarity=TrigramSimilarity('title', similar_lead.title) ).filter(similarity__gt=0.3).order_by('-similarity') return leads
def fetch(self, params, offset=None, limit=None): results = [] # Example: http://apidoc.rwlabs.org/#filter post_params = {} post_params['fields'] = { 'include': ['url_alias', 'title', 'date.original', 'source', 'source.homepage'] } if params.get('country'): post_params['filter'] = { 'field': 'country.iso3', 'value': params['country'], } if params.get('search'): post_params['query'] = { 'value': params['search'], 'fields': ['title'], 'operator': 'AND', } if offset: post_params['offset'] = offset if limit: post_params['limit'] = limit post_params['sort'] = ['date.original:desc', 'title:asc'] resp = requests.post(self.URL, json=post_params).json() count = resp['totalCount'] for datum in resp['data']: fields = datum['fields'] lead = Lead( id=str(datum['id']), title=fields['title'], published_on=fields['date']['original'], url=fields['url_alias'], source=fields['source'][0]['name'], website='www.reliefweb.int', ) results.append(lead) return results, count
def has_permission(self, request, view): project_id = view.kwargs.get('project_id') lead_id = view.kwargs.get('lead_id') entry_id = view.kwargs.get('entry_id') analysis_pillar_id = view.kwargs.get('analysis_pillar_id') if project_id: return Project.get_for_member( request.user).filter(id=project_id).exists() elif lead_id: return Lead.get_for(request.user).filter(id=lead_id).exists() elif entry_id: return Entry.get_for(request.user).filter(id=entry_id).exists() elif analysis_pillar_id: return AnalysisPillar.objects.filter( analysis__project__projectmembership__member=request.user, id=analysis_pillar_id).exists() return True
def fetch(self, params, page=None, limit=None): results = [] resp = requests.get(self.URL, params=params) soup = Soup(resp.text, 'html.parser') contents = soup.find('table').find('tbody').findAll('tr') for row in contents: tds = row.findAll('td') title = tds[0].get_text().replace('_', ' ') date = tds[1].find('span').attrs['content'][:10] # just date str # noqa date = datetime.datetime.strptime(date, '%Y-%m-%d') url = tds[0].find('a').attrs['href'] data = Lead( title=title.strip(), published_on=date.date(), url=url, source="Research Resource Center", source_type=Lead.WEBSITE, website=self.URL ) results.append(data) return results, len(results)
def fetch(self, params, page=None, limit=None): results = [] if not params or not params.get('feed-url'): return results, 0 r = requests.get(params['feed-url']) xml = etree.fromstring(r.content) items = xml.findall('channel/item') title_field = params.get('title-field') date_field = params.get('date-field') source_field = params.get('source-field') url_field = params.get('url-field') website_field = params.get('website-field') for item in items: def get_field(field): if not field: return '' element = item.find(field) return element.text title = get_field(title_field) date = get_field(date_field) source = get_field(source_field) url = get_field(url_field) website = get_field(website_field) data = Lead( # FIXME: use proper key id=url, title=title, published_on=date, source=source, url=url, website=website, source_type=Lead.RSS, ) results.append(data) return results, len(results)
def get(self, request, uuid=None, filename=None): queryset = File.objects.prefetch_related('lead_set') file = get_object_or_404(queryset, uuid=uuid) user = request.user leads_pk = file.lead_set.values_list('pk', flat=True) if (file.is_public or Lead.get_for(user).filter(pk__in=leads_pk).exists() or Entry.get_for(user).filter(image=file).exists() or Entry. get_for(user).filter(image_raw=request.build_absolute_uri( reverse('file', kwargs={'file_id': file.pk}), )).exists() # TODO: Add Profile ): if file.file: return redirect(request.build_absolute_uri(file.file.url)) return response.Response({ 'error': 'File doesn\'t exists', }, status=status.HTTP_404_NOT_FOUND) return response.Response({ 'error': 'Access Forbidden, Contact Admin', }, status=status.HTTP_403_FORBIDDEN)
def fetch(self, params, page=None, limit=None): results = [] url = self.URL if params.get('country'): url = self.URL + '/loc/' + params['country'] resp = requests.get(url, params={}) soup = Soup(resp.text, 'html.parser') contents = soup.find('div', {'id': 'content'}).find('tbody') for row in contents.findAll('tr'): tds = row.findAll('td') title = tds[0].find('a').get_text().strip() datestr = tds[3].get_text().strip() date = datetime.strptime(datestr, '%m/%d/%Y') url = tds[4].find('a')['href'] data = Lead( title=title, published_on=date.date(), url=url, source='Humanitarian Response', website=self.URL, source_type=Lead.WEBSITE ) results.append(data) return results, len(results)
def fetch(self, params, page=None, limit=None): results = [] if page: params['page'] = page resp = requests.get(self.URL, params=params) soup = Soup(resp.text, 'html.parser') contents = soup.find('div', {'class': 'view-content'}) if not contents: return results, len(results) # iterate and get leads for row in contents.findAll('div', {'class': 'views-row'}): content = row.find('h3').find('a') title = content.get_text() url = content['href'] data = Lead( title=title.strip(), url=url, source='WFP Assessments', source_type=Lead.WEBSITE, website='www.wfp.org' ) results.append(data) return results, len(results)
def get_queryset(self): return Lead.get_for(self.request.user)