def fetch_results():
index = 0
while True:
content = requests.get(URL.format(index)).json()
content = content['content']
if len(content) == 0:
break
else:
print('GOT {} chars'.format(len(content)))
content = pq(content)
boxes = content.find('.modal')
if len(boxes) == 0:
break
else:
print('GOT {} boxes'.format(len(boxes)))
for box in boxes:
box = pq(box)
description = pq(box.find('.moreInfoInner'))
description = sanitize_html(description)
print('GOT {}'.format(pq(box.find('#modal-title')).text()))
yield dict(
publication_id=0,
tender_id='0',
tender_type=None,
tender_type_he=pq(
box.find('.generalInfo-jobs li:nth-child(1) span')).text(),
decision=None,
page_title=pq(box.find('#modal-title')).text(),
page_url=SRC_URL,
publisher=pq(box.find('.publisher_link')).text(),
start_date=pq(
box.find('.generalInfo-jobs li:nth-child(2) span')).text(),
description=description,
)
index += 1
def func(row):
if row.get('parsed') is None:
return
url = row['page_url']
print(url)
page_text = requests.get(url).text
page = pq(page_text)
rules = [
('decision', '#MMDTendersStatusField span'),
('start_date', '.PublishingDate'),
('claim_date', '.LastDateForSubmission'),
('description', '#ctl00_PlaceHolderMain_ctl13_ctl00__ControlWrapper_RichHtmlField'),
]
output = {}
for rule, selector in rules:
elements = page.find(selector)
if len(elements):
output[rule] = sanitize_html(pq(elements[0])).strip()
else:
output[rule] = None
files = [l for l in page_text.split('\n') if 'var GovXDLFWrapperObj = ' in l]
if len(files) > 0:
files = files[0].split('var GovXDLFWrapperObj = ')[1].split(';//]]>')[0]
files = json.loads(files)['Items']
files = [
dict(
link=f['fileURL'],
description=f['FileTitle'],
)
for f in files
]
output['documents'] = files
row['parsed'] = output
def get_decision_list():
session = requests.Session()
session.headers['User-Agent'] = 'datagov-external-client'
response = session.get(SEARCH_PAGE_RESULTS_URL.format(skip=0)).json()
results = response['results']
count = 0
while True:
for result in results:
content_pq = pq(result['Content']) if result['Content'] else None
links = get_links(result['Content'], session)
content_pq = sanitize_html(content_pq)
yield {
'text':
content_pq,
'linked_docs':
links,
'doc_published_date':
result['DocPublishedDate'],
'doc_update_date':
result['DocUpdateDate'],
'id':
result['ItemUniqueId'],
'office':
result['ConnectedOffices'][0]["Title"]
if result.get('ConnectedOffices') else '',
'government':
result['PmoGovernmentDesc'][0]
if result.get('PmoGovernmentDesc') else
(result.get('PmoGovernment')[0]
if result.get('PmoGovernment') else None),
'policy_type':
result['PolicyTypeDesc'][0]
if result.get('PolicyTypeDesc') else '',
'procedure_number':
result['ProcedureNumberNumeric'],
'procedure_number_str':
result['ProcedureNumber'],
'publish_date':
result['PublishDate'],
'publish_date_prod':
result['PublishProd'],
'title':
result['Title'],
'unit':
result['UnitsDesc'][0] if result.get('UnitsDesc') else
(result.get('Units')[0] if result.get('Units') else None),
'update_date':
result['UpdateDate'],
'url_id':
result['UrlName'],
'score':
1
}
count += 1
response = session.get(
SEARCH_PAGE_RESULTS_URL.format(skip=count)).json()
results = response['results']
if not results:
return
def scraper():
for link in all_links():
url = BASE_URL + link
main = pq(requests.get(url).text)
title = pq(main.find('h1.main-title')[0]).text()
text = pq(main.find('article.node'))
start_date = text.find('div.field-type-datetime')
start_date = pq(start_date).text()
start_date = '/'.join(x for x in start_date.split('/'))
description = sanitize_html(
pq(text.find('.field-type-text-with-summary')))
try:
claim_date = pq(text.find('strong')[-1]).text()
claim_date = claim_date_re.findall(claim_date)[0]
except IndexError:
try:
claim_date = claim_date_re.findall(description)[-1]
except:
print('SKIPPING', url)
continue
claim_date = [int(p) for p in claim_date_re_parts.findall(claim_date)]
if claim_date[2] < 1000:
claim_date[2] += 2000
claim_date = '/'.join(str(p) for p in claim_date)
tender_type = None
if not tender_type:
for x in ["מכרז ל", "במכרז ל", "למכרז ל", "הזמנה לקבלת הצעות"]:
if x in title:
tender_type = 'office'
break
if not tender_type:
for x in ["פטור ממכרז", "בפטור"]:
if x in title:
tender_type = 'exemptions'
break
if not tender_type:
for x in ["קול קורא", "הודעה על", "הליך לקבלת הצעות"]:
if x in title:
tender_type = 'call_for_bids'
break
if not tender_type:
continue
yield dict(
page_url=url,
page_title=title,
start_date=start_date,
claim_date=claim_date,
description=description,
tender_type=tender_type,
publication_id=0,
tender_type_he='קול קורא',
tender_id='joint' + link.replace('/', '-'),
publisher='הג׳וינט',
)
def func(row):
details = pq(s.get(row['page_url']).text)
for key, selector, *_ in details_headers:
if selector.endswith('li'):
row[key] = [pq(x).text() for x in details.find(selector)]
else:
el = pq(details.find(selector))
if key == 'description':
row[key] = sanitize_html(el)
else:
row[key] = pq(el).text()
def scraper(gcd):
for start_date, link in all_links(gcd):
url = BASE_URL + link
gcd.driver.get(url)
time.sleep(1)
page = pq(gcd.driver.page_source)
title = pq(page.find('.page-content-container h1')).text().strip()
content = pq(page.find('.page-content-item'))
description = sanitize_html(content)
content = content.text().strip()
if not content or not title:
logging.warning('FAILED TO FIND TITLE OR CONTENT FOR %s', url)
continue
try:
claim_date = claim_date_re.findall(content)[0]
except:
logging.warning('FAILED TO FIND CLAIM DATE for %s', url)
claim_date = None
if claim_date:
claim_date = [
int(p) for p in claim_date_re_parts.findall(claim_date)
]
if claim_date[2] < 1000:
claim_date[2] += 2000
claim_date = '/'.join(str(p) for p in claim_date)
start_date = '/'.join(x for x in start_date.split('.'))
yield dict(
page_url=url,
page_title=title,
start_date=start_date,
claim_date=claim_date,
description=description,
tender_type='call_for_bids',
publication_id=0,
tender_type_he='קול קורא',
tender_id='kkl' + link.replace('/', '-'),
publisher='הקרן הקיימת לישראל',
)