def scrape_court(self, site, full_crawl=False):
download_error = False
# Get the court object early for logging
# opinions.united_states.federal.ca9_u --> ca9
court_str = site.court_id.split('.')[-1].split('_')[0]
court = Court.objects.get(pk=court_str)
dup_checker = DupChecker(court, full_crawl=full_crawl)
abort = dup_checker.abort_by_url_hash(site.url, site.hash)
if not abort:
if site.cookies:
logger.info("Using cookies: %s" % site.cookies)
for i, item in enumerate(site):
msg, r = get_binary_content(
item['download_urls'],
site.cookies,
site._get_adapter_instance(),
method=site.method
)
if msg:
logger.warn(msg)
ErrorLog(log_level='WARNING',
court=court,
message=msg).save()
continue
content = site.cleanup_content(r.content)
current_date = item['case_dates']
try:
next_date = site[i + 1]['case_dates']
except IndexError:
next_date = None
# request.content is sometimes a str, sometimes unicode, so
# force it all to be bytes, pleasing hashlib.
sha1_hash = hashlib.sha1(force_bytes(content)).hexdigest()
onwards = dup_checker.press_on(
Audio,
current_date,
next_date,
lookup_value=sha1_hash,
lookup_by='sha1'
)
if dup_checker.emulate_break:
break
if onwards:
# Not a duplicate, carry on
logger.info('Adding new document found at: %s' %
item['download_urls'].encode('utf-8'))
dup_checker.reset()
docket, audio_file, error = self.make_objects(
item, court, sha1_hash, content,
)
if error:
download_error = True
continue
self.save_everything(
items={
'docket': docket,
'audio_file': audio_file,
},
index=False,
)
process_audio_file.apply_async(
(audio_file.pk,),
countdown=random.randint(0, 3600)
)
logger.info(
"Successfully added audio file {pk}: {name}".format(
pk=audio_file.pk,
name=item['case_names'].encode('utf-8')
)
)
# Update the hash if everything finishes properly.
logger.info("%s: Successfully crawled oral arguments." %
site.court_id)
if not download_error and not full_crawl:
# Only update the hash if no errors occurred.
dup_checker.update_site_hash(site.hash)
def scrape_court(self, site, full_crawl=False):
download_error = False
# Get the court object early for logging
# opinions.united_states.federal.ca9_u --> ca9
court_str = site.court_id.split('.')[-1].split('_')[0]
court = Court.objects.get(pk=court_str)
dup_checker = DupChecker(court, full_crawl=full_crawl)
abort = dup_checker.abort_by_url_hash(site.url, site.hash)
if not abort:
if site.cookies:
logger.info("Using cookies: %s" % site.cookies)
for i, item in enumerate(site):
msg, r = get_binary_content(
item['download_urls'],
site.cookies,
site._get_adapter_instance(),
method=site.method
)
if msg:
logger.warn(msg)
ErrorLog(log_level='WARNING',
court=court,
message=msg).save()
continue
content = site.cleanup_content(r.content)
current_date = item['case_dates']
try:
next_date = site[i + 1]['case_dates']
except IndexError:
next_date = None
# request.content is sometimes a str, sometimes unicode, so
# force it all to be bytes, pleasing hashlib.
sha1_hash = hashlib.sha1(force_bytes(content)).hexdigest()
if (court_str == 'nev' and
item['precedential_statuses'] == 'Unpublished'):
# Nevada's non-precedential cases have different SHA1
# sums every time.
lookup_params = {'lookup_value': item['download_urls'],
'lookup_by': 'download_url'}
else:
lookup_params = {'lookup_value': sha1_hash,
'lookup_by': 'sha1'}
onwards = dup_checker.press_on(Opinion, current_date, next_date,
**lookup_params)
if dup_checker.emulate_break:
break
if onwards:
# Not a duplicate, carry on
logger.info('Adding new document found at: %s' %
item['download_urls'].encode('utf-8'))
dup_checker.reset()
docket, opinion, cluster, error = self.make_objects(
item, court, sha1_hash, content
)
if error:
download_error = True
continue
self.save_everything(
items={
'docket': docket,
'opinion': opinion,
'cluster': cluster
},
index=False
)
extract_doc_content.delay(
opinion.pk,
callback=subtask(extract_by_ocr),
citation_countdown=random.randint(0, 3600)
)
logger.info("Successfully added doc {pk}: {name}".format(
pk=opinion.pk,
name=item['case_names'].encode('utf-8'),
))
# Update the hash if everything finishes properly.
logger.info("%s: Successfully crawled opinions." % site.court_id)
if not download_error and not full_crawl:
# Only update the hash if no errors occurred.
dup_checker.update_site_hash(site.hash)
def scrape_court(self, site, full_crawl=False):
download_error = False
# Get the court object early for logging
# opinions.united_states.federal.ca9_u --> ca9
court_str = site.court_id.split('.')[-1].split('_')[0]
court = Court.objects.get(pk=court_str)
dup_checker = DupChecker(court, full_crawl=full_crawl)
abort = dup_checker.abort_by_url_hash(site.url, site.hash)
if not abort:
if site.cookies:
logger.info("Using cookies: %s" % site.cookies)
for i, item in enumerate(site):
msg, r = get_binary_content(item['download_urls'],
site.cookies,
site._get_adapter_instance(),
method=site.method)
if msg:
logger.warn(msg)
ErrorLog(log_level='WARNING', court=court,
message=msg).save()
continue
content = site.cleanup_content(r.content)
current_date = item['case_dates']
try:
next_date = site[i + 1]['case_dates']
except IndexError:
next_date = None
# request.content is sometimes a str, sometimes unicode, so
# force it all to be bytes, pleasing hashlib.
sha1_hash = hashlib.sha1(force_bytes(content)).hexdigest()
if (court_str == 'nev'
and item['precedential_statuses'] == 'Unpublished'):
# Nevada's non-precedential cases have different SHA1
# sums every time.
lookup_params = {
'lookup_value': item['download_urls'],
'lookup_by': 'download_url'
}
else:
lookup_params = {
'lookup_value': sha1_hash,
'lookup_by': 'sha1'
}
onwards = dup_checker.press_on(Opinion, current_date,
next_date, **lookup_params)
if dup_checker.emulate_break:
break
if onwards:
# Not a duplicate, carry on
logger.info('Adding new document found at: %s' %
item['download_urls'].encode('utf-8'))
dup_checker.reset()
docket, opinion, cluster, error = self.make_objects(
item, court, sha1_hash, content)
if error:
download_error = True
continue
self.save_everything(items={
'docket': docket,
'opinion': opinion,
'cluster': cluster
},
index=False)
extract_doc_content.delay(
opinion.pk,
callback=subtask(extract_by_ocr),
citation_countdown=random.randint(0, 3600))
logger.info("Successfully added doc {pk}: {name}".format(
pk=opinion.pk,
name=item['case_names'].encode('utf-8'),
))
# Update the hash if everything finishes properly.
logger.info("%s: Successfully crawled opinions." % site.court_id)
if not download_error and not full_crawl:
# Only update the hash if no errors occurred.
dup_checker.update_site_hash(site.hash)
def download_and_save():
"""This function is run in many threads simultaneously. Each thread
runs so long as there are items in the queue. Once an item is found, it's
downloaded and saved.
The number of items that can be concurrently saved is determined by the
number of threads that are running this function.
"""
while True:
item = queue.get()
logger.info("%s: Attempting to add item at: %s" %
(threading.current_thread().name, item['url']))
try:
msg, r = get_binary_content(
item['url'],
{},
)
except:
logger.info("%s: Unable to get item at: %s" %
(threading.current_thread().name, item['url']))
queue.task_done()
if msg:
logger.warn(msg)
queue.task_done()
continue
sha1_hash = hashlib.sha1(r.content).hexdigest()
if Audio.objects.filter(sha1=sha1_hash).exists():
# Simpsons did it! Try the next one.
logger.info("%s: Item already exists, moving to next item." %
threading.current_thread().name)
queue.task_done()
continue
else:
# New item, onwards!
logger.info('%s: Adding new document found at: %s' %
(threading.current_thread().name, item['url']))
audio_file = Audio(
source='H',
sha1=sha1_hash,
case_name=item['case_name'],
download_url=item['url'],
processing_complete=False,
)
if item['judges']:
audio_file.judges = item['judges']
if item['docket_number']:
audio_file.docket.docket_number = item['docket_number']
court = Court.objects.get(pk=item['court_code'])
docket = Docket(
case_name=item['case_name'],
court=court,
date_argued=item['date_argued'],
)
# Make and associate the file object
try:
cf = ContentFile(r.content)
extension = get_extension(r.content)
if extension not in ['.mp3', '.wma']:
extension = '.' + item['url'].rsplit('.', 1)[1]
# See bitbucket issue #215 for why this must be
# lower-cased.
file_name = trunc(item['case_name'].lower(), 75) + extension
audio_file.local_path_original_file.save(file_name, cf,
save=False)
except:
msg = 'Unable to save binary. Deleted document: %s.\n%s' % \
(item['case_name'], traceback.format_exc())
logger.critical(msg)
queue.task_done()
docket.save()
audio_file.docket = docket
audio_file.save(index=False)
random_delay = random.randint(0, 3600)
process_audio_file.apply_async(
(audio_file.pk,),
countdown=random_delay
)
logger.info("%s: Successfully added audio file %s: %s" %
(threading.current_thread().name,
audio_file.pk,
audio_file.case_name))
def scrape_court(self, site, full_crawl=False):
download_error = False
# Get the court object early for logging
# opinions.united_states.federal.ca9_u --> ca9
court_str = site.court_id.split('.')[-1].split('_')[0]
court = Court.objects.get(pk=court_str)
dup_checker = DupChecker(court, full_crawl=full_crawl)
abort = dup_checker.abort_by_url_hash(site.url, site.hash)
if not abort:
if site.cookies:
logger.info("Using cookies: %s" % site.cookies)
for i, item in enumerate(site):
msg, r = get_binary_content(item['download_urls'],
site.cookies,
site._get_adapter_instance(),
method=site.method)
if msg:
logger.warn(msg)
ErrorLog(log_level='WARNING', court=court,
message=msg).save()
continue
content = site.cleanup_content(r.content)
current_date = item['case_dates']
try:
next_date = site[i + 1]['case_dates']
except IndexError:
next_date = None
# request.content is sometimes a str, sometimes unicode, so
# force it all to be bytes, pleasing hashlib.
sha1_hash = hashlib.sha1(force_bytes(content)).hexdigest()
onwards = dup_checker.press_on(Audio,
current_date,
next_date,
lookup_value=sha1_hash,
lookup_by='sha1')
if dup_checker.emulate_break:
break
if onwards:
# Not a duplicate, carry on
logger.info('Adding new document found at: %s' %
item['download_urls'].encode('utf-8'))
dup_checker.reset()
docket, audio_file, error = self.make_objects(
item,
court,
sha1_hash,
content,
)
if error:
download_error = True
continue
self.save_everything(
items={
'docket': docket,
'audio_file': audio_file,
},
index=False,
)
process_audio_file.apply_async(
(audio_file.pk, ), countdown=random.randint(0, 3600))
logger.info(
"Successfully added audio file {pk}: {name}".format(
pk=audio_file.pk,
name=item['case_names'].encode('utf-8')))
# Update the hash if everything finishes properly.
logger.info("%s: Successfully crawled oral arguments." %
site.court_id)
if not download_error and not full_crawl:
# Only update the hash if no errors occurred.
dup_checker.update_site_hash(site.hash)
def download_and_save():
"""This function is run in many threads simultaneously. Each thread
runs so long as there are items in the queue. Once an item is found, it's
downloaded and saved.
The number of items that can be concurrently saved is determined by the
number of threads that are running this function.
"""
while True:
item = queue.get()
logger.info("%s: Attempting to add item at: %s" %
(threading.current_thread().name, item['url']))
try:
msg, r = get_binary_content(
item['url'],
{},
)
except:
logger.info("%s: Unable to get item at: %s" %
(threading.current_thread().name, item['url']))
queue.task_done()
if msg:
logger.warn(msg)
queue.task_done()
continue
sha1_hash = hashlib.sha1(r.content).hexdigest()
if Audio.objects.filter(sha1=sha1_hash).exists():
# Simpsons did it! Try the next one.
logger.info("%s: Item already exists, moving to next item." %
threading.current_thread().name)
queue.task_done()
continue
else:
# New item, onwards!
logger.info('%s: Adding new document found at: %s' %
(threading.current_thread().name, item['url']))
audio_file = Audio(
source='H',
sha1=sha1_hash,
case_name=item['case_name'],
download_url=item['url'],
processing_complete=False,
)
if item['judges']:
audio_file.judges = item['judges']
if item['docket_number']:
audio_file.docket.docket_number = item['docket_number']
court = Court.objects.get(pk=item['court_code'])
docket = Docket(
case_name=item['case_name'],
court=court,
date_argued=item['date_argued'],
)
# Make and associate the file object
try:
cf = ContentFile(r.content)
extension = get_extension(r.content)
if extension not in ['.mp3', '.wma']:
extension = '.' + item['url'].rsplit('.', 1)[1]
# See bitbucket issue #215 for why this must be
# lower-cased.
file_name = trunc(item['case_name'].lower(), 75) + extension
audio_file.local_path_original_file.save(file_name,
cf,
save=False)
except:
msg = 'Unable to save binary. Deleted document: %s.\n%s' % \
(item['case_name'], traceback.format_exc())
logger.critical(msg)
queue.task_done()
docket.save()
audio_file.docket = docket
audio_file.save(index=False)
random_delay = random.randint(0, 3600)
process_audio_file.apply_async((audio_file.pk, ),
countdown=random_delay)
logger.info("%s: Successfully added audio file %s: %s" %
(threading.current_thread().name, audio_file.pk,
audio_file.case_name))