def migrate_opinions_oral_args_and_dockets(self):
"""Migrate the core objects across, diffing as you go.
:param start_date: Items changed after this date will be processed.
:return: None
"""
self.stdout.write("Migrating dockets, audio files, and opinions...")
# Find dockets modified after date or with sub-items modified after
# date.
q = Q(date_modified__gte=self.start)
q |= Q(documents__date_modified__gte=self.start)
q |= Q(audio_files__date_modified__gte=self.start)
old_dockets = DocketOld.objects.using('old').filter(q)
for old_docket in old_dockets:
try:
old_audio = old_docket.audio_files.all()[0]
except IndexError:
old_audio = None
try:
old_document = old_docket.documents.all()[0]
except IndexError:
old_document = None
if old_document is None and old_audio is None:
continue
if old_document is not None:
old_citation = old_document.citation
old_docket.case_name, old_docket.case_name_full, old_docket.case_name_short = self._get_case_names(
old_citation.case_name)
else:
# Fall back on the docket if needed. Assumes they docket and
# document case_names are always the same.
old_docket.case_name, old_docket.case_name_full, old_docket.case_name_short = self._get_case_names(
old_docket.case_name)
if old_audio is not None:
old_audio.case_name, old_audio.case_name_full, old_audio.case_name_short = self._get_case_names(
old_audio.case_name)
# Courts are in place thanks to initial data. Get the court.
court = CourtNew.objects.get(pk=old_docket.court_id)
# Do Dockets
try:
existing_docket = (DocketNew.objects.using('default').get(
pk=old_docket.pk))
except DocketNew.DoesNotExist:
existing_docket = None
if existing_docket is not None:
# Intersection. No need for complicated merge as all differences
# have been resolved by hand.
new_docket = existing_docket
else:
# New docket in old system. Create it in the new system.
new_docket = DocketNew(
pk=old_docket.pk,
date_modified=old_docket.date_modified,
date_created=old_docket.date_modified,
court=court,
case_name=old_docket.case_name,
case_name_full=old_docket.case_name_full,
case_name_short=old_docket.case_name_short,
slug=self._none_to_blank(old_docket.slug),
docket_number=self._none_to_blank(
old_citation.docket_number),
date_blocked=old_docket.date_blocked,
blocked=old_docket.blocked,
)
if old_audio is not None:
new_docket.date_argued = old_audio.date_argued
new_docket.save(using='default')
# Do Documents/Clusters
if old_document is not None:
try:
existing_oc = (
OpinionClusterNew.objects.using('default').get(
pk=old_document.pk))
except OpinionClusterNew.DoesNotExist:
existing_oc = None
try:
existing_o = (OpinionNew.objects.using('default').get(
pk=old_document.pk))
except OpinionNew.DoesNotExist:
existing_o = None
if existing_oc is not None or existing_o is not None:
# Run the conflict algo.
if self.find_conflicts(old_document, old_citation,
old_docket, existing_oc,
existing_o):
self.stdout.write("Found conflict. Resolve that.")
else:
# No conflicts. Update the existing item.
self.add_oc_and_o(old_document, old_citation,
old_docket, new_docket)
else:
# New item. Just add it.
self.add_oc_and_o(old_document, old_citation, old_docket,
new_docket)
# Finally we do Audio. No checks needed because we haven't changed
# anything on the new server.
if old_audio is not None:
new_audio_file = AudioNew(
pk=old_audio.pk,
docket=new_docket,
source=old_audio.source,
case_name=old_audio.case_name,
case_name_short=old_audio.case_name_short,
case_name_full=old_audio.case_name_full,
judges=self._none_to_blank(old_audio.judges),
date_created=old_audio.time_retrieved,
date_modified=old_audio.date_modified,
sha1=old_audio.sha1,
download_url=old_audio.download_url,
local_path_mp3=old_audio.local_path_mp3,
local_path_original_file=old_audio.
local_path_original_file,
duration=old_audio.duration,
processing_complete=old_audio.processing_complete,
date_blocked=old_audio.date_blocked,
blocked=old_audio.blocked,
)
new_audio_file.save(
using='default',
index=False,
)
def migrate_opinions_oral_args_and_dockets(self):
self.stdout.write("Migrating dockets, audio files, and opinions to new "
"database...")
q = DocketOld.objects.using('old').all()
old_dockets = queryset_generator(q)
num_dockets = q.count()
progress = 0
self._print_progress(progress, num_dockets)
for old_docket in old_dockets:
# First do the docket, then create the cluster and opinion objects.
try:
old_audio = old_docket.audio_files.all()[0]
except IndexError:
old_audio = None
try:
old_document = old_docket.documents.all()[0]
except IndexError:
old_document = None
if old_document is not None:
old_citation = old_document.citation
old_doc_case_name, old_doc_case_name_full, old_doc_case_name_short = self._get_case_names(old_citation.case_name)
if old_audio is not None:
old_audio_case_name, old_audio_case_name_full, old_audio_case_name_short = self._get_case_names(old_audio.case_name)
court = CourtNew.objects.get(pk=old_docket.court_id) # Courts are in place thanks to initial data.
new_docket = DocketNew(
pk=old_docket.pk,
date_modified=old_docket.date_modified,
date_created=old_docket.date_modified,
court=court,
case_name=old_doc_case_name,
case_name_full=old_doc_case_name_full,
case_name_short=old_doc_case_name_short,
slug=self._none_to_blank(old_docket.slug),
docket_number=self._none_to_blank(old_citation.docket_number),
date_blocked=old_docket.date_blocked,
blocked=old_docket.blocked,
)
if old_audio is not None:
new_docket.date_argued = old_audio.date_argued
new_docket.save(using='default')
if old_document is not None:
new_opinion_cluster = OpinionClusterNew(
pk=old_document.pk,
docket=new_docket,
judges=self._none_to_blank(old_document.judges),
date_modified=old_document.date_modified,
date_created=old_document.date_modified,
date_filed=old_document.date_filed,
slug=self._none_to_blank(old_citation.slug),
citation_id=old_document.citation_id,
case_name_short=old_doc_case_name_short,
case_name=old_doc_case_name,
case_name_full=old_doc_case_name_full,
federal_cite_one=self._none_to_blank(
old_citation.federal_cite_one),
federal_cite_two=self._none_to_blank(
old_citation.federal_cite_two),
federal_cite_three=self._none_to_blank(
old_citation.federal_cite_three),
state_cite_one=self._none_to_blank(
old_citation.state_cite_one),
state_cite_two=self._none_to_blank(
old_citation.state_cite_two),
state_cite_three=self._none_to_blank(
old_citation.state_cite_three),
state_cite_regional=self._none_to_blank(
old_citation.state_cite_regional),
specialty_cite_one=self._none_to_blank(
old_citation.specialty_cite_one),
scotus_early_cite=self._none_to_blank(
old_citation.scotus_early_cite),
lexis_cite=self._none_to_blank(old_citation.lexis_cite),
westlaw_cite=self._none_to_blank(old_citation.westlaw_cite),
neutral_cite=self._none_to_blank(old_citation.neutral_cite),
scdb_id=self._none_to_blank(
old_document.supreme_court_db_id),
source=old_document.source,
nature_of_suit=old_document.nature_of_suit,
citation_count=old_document.citation_count,
precedential_status=old_document.precedential_status,
date_blocked=old_document.date_blocked,
blocked=old_document.blocked,
)
new_opinion_cluster.save(
using='default',
index=False,
)
new_opinion = OpinionNew(
pk=old_document.pk,
cluster=new_opinion_cluster,
date_modified=old_document.date_modified,
date_created=old_document.time_retrieved,
type='010combined',
sha1=old_document.sha1,
download_url=old_document.download_url,
local_path=old_document.local_path,
plain_text=old_document.plain_text,
html=self._none_to_blank(old_document.html),
html_lawbox=self._none_to_blank(old_document.html_lawbox),
html_with_citations=old_document.html_with_citations,
extracted_by_ocr=old_document.extracted_by_ocr,
)
new_opinion.save(
using='default',
index=False,
)
if old_audio is not None:
new_audio_file = AudioNew(
pk=old_audio.pk,
docket=new_docket,
source=old_audio.source,
case_name=old_audio_case_name,
case_name_short=old_audio_case_name_short,
case_name_full=old_audio_case_name_full,
judges=self._none_to_blank(old_audio.judges),
date_created=old_audio.time_retrieved,
date_modified=old_audio.date_modified,
sha1=old_audio.sha1,
download_url=old_audio.download_url,
local_path_mp3=old_audio.local_path_mp3,
local_path_original_file=old_audio.local_path_original_file,
duration=old_audio.duration,
processing_complete=old_audio.processing_complete,
date_blocked=old_audio.date_blocked,
blocked=old_audio.blocked,
)
new_audio_file.save(
using='default',
index=False,
)
progress += 1
self._print_progress(progress, num_dockets)
self.stdout.write(u'') # Newline
def migrate_opinions_oral_args_and_dockets(self):
"""Migrate the core objects across, diffing as you go.
:param start_date: Items changed after this date will be processed.
:return: None
"""
self.stdout.write("Migrating dockets, audio files, and opinions...")
# Find dockets modified after date or with sub-items modified after
# date.
q = Q(date_modified__gte=self.start)
q |= Q(documents__date_modified__gte=self.start)
q |= Q(audio_files__date_modified__gte=self.start)
old_dockets = DocketOld.objects.using('old').filter(q)
for old_docket in old_dockets:
try:
old_audio = old_docket.audio_files.all()[0]
except IndexError:
old_audio = None
try:
old_document = old_docket.documents.all()[0]
except IndexError:
old_document = None
if old_document is None and old_audio is None:
continue
if old_document is not None:
old_citation = old_document.citation
old_docket.case_name, old_docket.case_name_full, old_docket.case_name_short = self._get_case_names(
old_citation.case_name)
else:
# Fall back on the docket if needed. Assumes they docket and
# document case_names are always the same.
old_docket.case_name, old_docket.case_name_full, old_docket.case_name_short = self._get_case_names(
old_docket.case_name)
if old_audio is not None:
old_audio.case_name, old_audio.case_name_full, old_audio.case_name_short = self._get_case_names(
old_audio.case_name)
# Courts are in place thanks to initial data. Get the court.
court = CourtNew.objects.get(pk=old_docket.court_id)
# Do Dockets
try:
existing_docket = (DocketNew.objects
.using('default')
.get(pk=old_docket.pk))
except DocketNew.DoesNotExist:
existing_docket = None
if existing_docket is not None:
# Intersection. No need for complicated merge as all differences
# have been resolved by hand.
new_docket = existing_docket
else:
# New docket in old system. Create it in the new system.
new_docket = DocketNew(
pk=old_docket.pk,
date_modified=old_docket.date_modified,
date_created=old_docket.date_modified,
court=court,
case_name=old_docket.case_name,
case_name_full=old_docket.case_name_full,
case_name_short=old_docket.case_name_short,
slug=self._none_to_blank(old_docket.slug),
docket_number=self._none_to_blank(
old_citation.docket_number),
date_blocked=old_docket.date_blocked,
blocked=old_docket.blocked,
)
if old_audio is not None:
new_docket.date_argued = old_audio.date_argued
new_docket.save(using='default')
# Do Documents/Clusters
if old_document is not None:
try:
existing_oc = (OpinionClusterNew.objects
.using('default')
.get(pk=old_document.pk))
except OpinionClusterNew.DoesNotExist:
existing_oc = None
try:
existing_o = (OpinionNew.objects
.using('default')
.get(pk=old_document.pk))
except OpinionNew.DoesNotExist:
existing_o = None
if existing_oc is not None or existing_o is not None:
# Run the conflict algo.
if self.find_conflicts(old_document, old_citation,
old_docket, existing_oc,
existing_o):
self.stdout.write("Found conflict. Resolve that.")
else:
# No conflicts. Update the existing item.
self.add_oc_and_o(old_document, old_citation,
old_docket, new_docket)
else:
# New item. Just add it.
self.add_oc_and_o(old_document, old_citation, old_docket,
new_docket)
# Finally we do Audio. No checks needed because we haven't changed
# anything on the new server.
if old_audio is not None:
new_audio_file = AudioNew(
pk=old_audio.pk,
docket=new_docket,
source=old_audio.source,
case_name=old_audio.case_name,
case_name_short=old_audio.case_name_short,
case_name_full=old_audio.case_name_full,
judges=self._none_to_blank(old_audio.judges),
date_created=old_audio.time_retrieved,
date_modified=old_audio.date_modified,
sha1=old_audio.sha1,
download_url=old_audio.download_url,
local_path_mp3=old_audio.local_path_mp3,
local_path_original_file=old_audio.local_path_original_file,
duration=old_audio.duration,
processing_complete=old_audio.processing_complete,
date_blocked=old_audio.date_blocked,
blocked=old_audio.blocked,
)
new_audio_file.save(
using='default',
index=False,
)
