def migrate_opinions_oral_args_and_dockets(self): """Migrate the core objects across, diffing as you go. :param start_date: Items changed after this date will be processed. :return: None """ self.stdout.write("Migrating dockets, audio files, and opinions...") # Find dockets modified after date or with sub-items modified after # date. q = Q(date_modified__gte=self.start) q |= Q(documents__date_modified__gte=self.start) q |= Q(audio_files__date_modified__gte=self.start) old_dockets = DocketOld.objects.using('old').filter(q) for old_docket in old_dockets: try: old_audio = old_docket.audio_files.all()[0] except IndexError: old_audio = None try: old_document = old_docket.documents.all()[0] except IndexError: old_document = None if old_document is None and old_audio is None: continue if old_document is not None: old_citation = old_document.citation old_docket.case_name, old_docket.case_name_full, old_docket.case_name_short = self._get_case_names( old_citation.case_name) else: # Fall back on the docket if needed. Assumes they docket and # document case_names are always the same. old_docket.case_name, old_docket.case_name_full, old_docket.case_name_short = self._get_case_names( old_docket.case_name) if old_audio is not None: old_audio.case_name, old_audio.case_name_full, old_audio.case_name_short = self._get_case_names( old_audio.case_name) # Courts are in place thanks to initial data. Get the court. court = CourtNew.objects.get(pk=old_docket.court_id) # Do Dockets try: existing_docket = (DocketNew.objects.using('default').get( pk=old_docket.pk)) except DocketNew.DoesNotExist: existing_docket = None if existing_docket is not None: # Intersection. No need for complicated merge as all differences # have been resolved by hand. new_docket = existing_docket else: # New docket in old system. Create it in the new system. new_docket = DocketNew( pk=old_docket.pk, date_modified=old_docket.date_modified, date_created=old_docket.date_modified, court=court, case_name=old_docket.case_name, case_name_full=old_docket.case_name_full, case_name_short=old_docket.case_name_short, slug=self._none_to_blank(old_docket.slug), docket_number=self._none_to_blank( old_citation.docket_number), date_blocked=old_docket.date_blocked, blocked=old_docket.blocked, ) if old_audio is not None: new_docket.date_argued = old_audio.date_argued new_docket.save(using='default') # Do Documents/Clusters if old_document is not None: try: existing_oc = ( OpinionClusterNew.objects.using('default').get( pk=old_document.pk)) except OpinionClusterNew.DoesNotExist: existing_oc = None try: existing_o = (OpinionNew.objects.using('default').get( pk=old_document.pk)) except OpinionNew.DoesNotExist: existing_o = None if existing_oc is not None or existing_o is not None: # Run the conflict algo. if self.find_conflicts(old_document, old_citation, old_docket, existing_oc, existing_o): self.stdout.write("Found conflict. Resolve that.") else: # No conflicts. Update the existing item. self.add_oc_and_o(old_document, old_citation, old_docket, new_docket) else: # New item. Just add it. self.add_oc_and_o(old_document, old_citation, old_docket, new_docket) # Finally we do Audio. No checks needed because we haven't changed # anything on the new server. if old_audio is not None: new_audio_file = AudioNew( pk=old_audio.pk, docket=new_docket, source=old_audio.source, case_name=old_audio.case_name, case_name_short=old_audio.case_name_short, case_name_full=old_audio.case_name_full, judges=self._none_to_blank(old_audio.judges), date_created=old_audio.time_retrieved, date_modified=old_audio.date_modified, sha1=old_audio.sha1, download_url=old_audio.download_url, local_path_mp3=old_audio.local_path_mp3, local_path_original_file=old_audio. local_path_original_file, duration=old_audio.duration, processing_complete=old_audio.processing_complete, date_blocked=old_audio.date_blocked, blocked=old_audio.blocked, ) new_audio_file.save( using='default', index=False, )
def migrate_opinions_oral_args_and_dockets(self): self.stdout.write("Migrating dockets, audio files, and opinions to new " "database...") q = DocketOld.objects.using('old').all() old_dockets = queryset_generator(q) num_dockets = q.count() progress = 0 self._print_progress(progress, num_dockets) for old_docket in old_dockets: # First do the docket, then create the cluster and opinion objects. try: old_audio = old_docket.audio_files.all()[0] except IndexError: old_audio = None try: old_document = old_docket.documents.all()[0] except IndexError: old_document = None if old_document is not None: old_citation = old_document.citation old_doc_case_name, old_doc_case_name_full, old_doc_case_name_short = self._get_case_names(old_citation.case_name) if old_audio is not None: old_audio_case_name, old_audio_case_name_full, old_audio_case_name_short = self._get_case_names(old_audio.case_name) court = CourtNew.objects.get(pk=old_docket.court_id) # Courts are in place thanks to initial data. new_docket = DocketNew( pk=old_docket.pk, date_modified=old_docket.date_modified, date_created=old_docket.date_modified, court=court, case_name=old_doc_case_name, case_name_full=old_doc_case_name_full, case_name_short=old_doc_case_name_short, slug=self._none_to_blank(old_docket.slug), docket_number=self._none_to_blank(old_citation.docket_number), date_blocked=old_docket.date_blocked, blocked=old_docket.blocked, ) if old_audio is not None: new_docket.date_argued = old_audio.date_argued new_docket.save(using='default') if old_document is not None: new_opinion_cluster = OpinionClusterNew( pk=old_document.pk, docket=new_docket, judges=self._none_to_blank(old_document.judges), date_modified=old_document.date_modified, date_created=old_document.date_modified, date_filed=old_document.date_filed, slug=self._none_to_blank(old_citation.slug), citation_id=old_document.citation_id, case_name_short=old_doc_case_name_short, case_name=old_doc_case_name, case_name_full=old_doc_case_name_full, federal_cite_one=self._none_to_blank( old_citation.federal_cite_one), federal_cite_two=self._none_to_blank( old_citation.federal_cite_two), federal_cite_three=self._none_to_blank( old_citation.federal_cite_three), state_cite_one=self._none_to_blank( old_citation.state_cite_one), state_cite_two=self._none_to_blank( old_citation.state_cite_two), state_cite_three=self._none_to_blank( old_citation.state_cite_three), state_cite_regional=self._none_to_blank( old_citation.state_cite_regional), specialty_cite_one=self._none_to_blank( old_citation.specialty_cite_one), scotus_early_cite=self._none_to_blank( old_citation.scotus_early_cite), lexis_cite=self._none_to_blank(old_citation.lexis_cite), westlaw_cite=self._none_to_blank(old_citation.westlaw_cite), neutral_cite=self._none_to_blank(old_citation.neutral_cite), scdb_id=self._none_to_blank( old_document.supreme_court_db_id), source=old_document.source, nature_of_suit=old_document.nature_of_suit, citation_count=old_document.citation_count, precedential_status=old_document.precedential_status, date_blocked=old_document.date_blocked, blocked=old_document.blocked, ) new_opinion_cluster.save( using='default', index=False, ) new_opinion = OpinionNew( pk=old_document.pk, cluster=new_opinion_cluster, date_modified=old_document.date_modified, date_created=old_document.time_retrieved, type='010combined', sha1=old_document.sha1, download_url=old_document.download_url, local_path=old_document.local_path, plain_text=old_document.plain_text, html=self._none_to_blank(old_document.html), html_lawbox=self._none_to_blank(old_document.html_lawbox), html_with_citations=old_document.html_with_citations, extracted_by_ocr=old_document.extracted_by_ocr, ) new_opinion.save( using='default', index=False, ) if old_audio is not None: new_audio_file = AudioNew( pk=old_audio.pk, docket=new_docket, source=old_audio.source, case_name=old_audio_case_name, case_name_short=old_audio_case_name_short, case_name_full=old_audio_case_name_full, judges=self._none_to_blank(old_audio.judges), date_created=old_audio.time_retrieved, date_modified=old_audio.date_modified, sha1=old_audio.sha1, download_url=old_audio.download_url, local_path_mp3=old_audio.local_path_mp3, local_path_original_file=old_audio.local_path_original_file, duration=old_audio.duration, processing_complete=old_audio.processing_complete, date_blocked=old_audio.date_blocked, blocked=old_audio.blocked, ) new_audio_file.save( using='default', index=False, ) progress += 1 self._print_progress(progress, num_dockets) self.stdout.write(u'') # Newline
def migrate_opinions_oral_args_and_dockets(self): """Migrate the core objects across, diffing as you go. :param start_date: Items changed after this date will be processed. :return: None """ self.stdout.write("Migrating dockets, audio files, and opinions...") # Find dockets modified after date or with sub-items modified after # date. q = Q(date_modified__gte=self.start) q |= Q(documents__date_modified__gte=self.start) q |= Q(audio_files__date_modified__gte=self.start) old_dockets = DocketOld.objects.using('old').filter(q) for old_docket in old_dockets: try: old_audio = old_docket.audio_files.all()[0] except IndexError: old_audio = None try: old_document = old_docket.documents.all()[0] except IndexError: old_document = None if old_document is None and old_audio is None: continue if old_document is not None: old_citation = old_document.citation old_docket.case_name, old_docket.case_name_full, old_docket.case_name_short = self._get_case_names( old_citation.case_name) else: # Fall back on the docket if needed. Assumes they docket and # document case_names are always the same. old_docket.case_name, old_docket.case_name_full, old_docket.case_name_short = self._get_case_names( old_docket.case_name) if old_audio is not None: old_audio.case_name, old_audio.case_name_full, old_audio.case_name_short = self._get_case_names( old_audio.case_name) # Courts are in place thanks to initial data. Get the court. court = CourtNew.objects.get(pk=old_docket.court_id) # Do Dockets try: existing_docket = (DocketNew.objects .using('default') .get(pk=old_docket.pk)) except DocketNew.DoesNotExist: existing_docket = None if existing_docket is not None: # Intersection. No need for complicated merge as all differences # have been resolved by hand. new_docket = existing_docket else: # New docket in old system. Create it in the new system. new_docket = DocketNew( pk=old_docket.pk, date_modified=old_docket.date_modified, date_created=old_docket.date_modified, court=court, case_name=old_docket.case_name, case_name_full=old_docket.case_name_full, case_name_short=old_docket.case_name_short, slug=self._none_to_blank(old_docket.slug), docket_number=self._none_to_blank( old_citation.docket_number), date_blocked=old_docket.date_blocked, blocked=old_docket.blocked, ) if old_audio is not None: new_docket.date_argued = old_audio.date_argued new_docket.save(using='default') # Do Documents/Clusters if old_document is not None: try: existing_oc = (OpinionClusterNew.objects .using('default') .get(pk=old_document.pk)) except OpinionClusterNew.DoesNotExist: existing_oc = None try: existing_o = (OpinionNew.objects .using('default') .get(pk=old_document.pk)) except OpinionNew.DoesNotExist: existing_o = None if existing_oc is not None or existing_o is not None: # Run the conflict algo. if self.find_conflicts(old_document, old_citation, old_docket, existing_oc, existing_o): self.stdout.write("Found conflict. Resolve that.") else: # No conflicts. Update the existing item. self.add_oc_and_o(old_document, old_citation, old_docket, new_docket) else: # New item. Just add it. self.add_oc_and_o(old_document, old_citation, old_docket, new_docket) # Finally we do Audio. No checks needed because we haven't changed # anything on the new server. if old_audio is not None: new_audio_file = AudioNew( pk=old_audio.pk, docket=new_docket, source=old_audio.source, case_name=old_audio.case_name, case_name_short=old_audio.case_name_short, case_name_full=old_audio.case_name_full, judges=self._none_to_blank(old_audio.judges), date_created=old_audio.time_retrieved, date_modified=old_audio.date_modified, sha1=old_audio.sha1, download_url=old_audio.download_url, local_path_mp3=old_audio.local_path_mp3, local_path_original_file=old_audio.local_path_original_file, duration=old_audio.duration, processing_complete=old_audio.processing_complete, date_blocked=old_audio.date_blocked, blocked=old_audio.blocked, ) new_audio_file.save( using='default', index=False, )