def update(dry_run=False): created_dict = {'osf': []} for hier in BEPRESS_CHANGES['create']: new_text = hier.split(':')[-1] bepress_parent = BEPRESS_PROVIDER.subjects.get(text=hier.split(':')[-2]) logger.info('Creating osf - {}'.format(new_text)) bepress_subject = Subject.objects.create(parent=bepress_parent, provider=BEPRESS_PROVIDER, text=new_text) created_dict['osf'].append(new_text) for custom_parent in bepress_parent.aliases.all(): if not bepress_parent.children.count() > 1 or ( custom_parent.children.exists() and set(bepress_parent.children.exclude(text=new_text).values_list('text', flat=True)).issubset(set(custom_parent.children.values_list('text', flat=True)))): # Either children were included in the custom taxonomy or they didn't exist before, probably logger.info('Creating {} - {}'.format(custom_parent.provider._id, new_text)) Subject.objects.create(parent=custom_parent, provider=custom_parent.provider, text=new_text, bepress_subject=bepress_subject) if custom_parent.provider._id not in created_dict: created_dict[custom_parent.provider._id] = [] created_dict[custom_parent.provider._id].append(new_text) for old, new in BEPRESS_CHANGES['rename'].items(): logger.info('Renaming `{}`->`{}`'.format(old, new)) to_update = Subject.objects.filter(text=old) affected_preprints = set(to_update.exclude(preprints__isnull=True).values_list('preprints__guids___id', flat=True)) to_update.update(text=new) for preprint_id in affected_preprints: logger.info('Notifying SHARE about preprint {} change'.format(preprint_id)) if not dry_run: on_preprint_updated(preprint_id) for provider_id, list_of_subjs in created_dict.items(): logger.info('Created {} new subjects on {}: "{}"'.format(len(list_of_subjs), provider_id, ', '.join(list_of_subjs)))
def test_send_to_share_is_true(self, mock_share, preprint): on_preprint_updated(preprint._id) data = json.loads(mock_share.calls[-1].request.body.decode()) assert data['data']['attributes']['data']['@graph'] assert mock_share.calls[-1].request.headers[ 'Authorization'] == 'Bearer Snowmobiling'
def migrate(dry=True): assert settings.SHARE_URL, 'SHARE_URL must be set to migrate.' assert settings.SHARE_API_TOKEN, 'SHARE_API_TOKEN must be set to migrate.' targets = get_targets() target_count = len(targets) successes = [] failures = [] count = 0 logger.info('Preparing to migrate {} preprints.'.format(target_count)) for preprint_id in targets: count += 1 logger.info('{}/{} - {}'.format(count, target_count, preprint_id)) try: if not dry: on_preprint_updated(preprint_id, update_share=True) # Sleep in order to be nice to EZID time.sleep(1) except Exception as e: # TODO: This reliably fails for certain nodes with # IncompleteRead(0 bytes read) failures.append(preprint_id) logger.warn('Encountered exception {} while posting to SHARE for preprint {}'.format(e, preprint_id)) else: successes.append(preprint_id) logger.info('Successes: {}'.format(successes)) logger.info('Failures: {}'.format(failures))
def update_share_preprint_modified_dates(dry_run=False): for preprint in Preprint.objects.filter(date_modified__lt=F('node__modified')): if dry_run: logger.info('Would have sent ' + preprint._id + ' data to SHARE') else: on_preprint_updated(preprint._id) logger.info(preprint._id + ' data sent to SHARE')
def migrate(): assert settings.SHARE_URL, 'SHARE_URL must be set to migrate.' assert settings.SHARE_API_TOKEN, 'SHARE_API_TOKEN must be set to migrate.' targets = get_targets() target_count = len(targets) successes = [] failures = [] count = 0 logger.info('Preparing to migrate {} preprints.'.format(target_count)) for preprint_id in targets: count += 1 logger.info('{}/{} - {}'.format(count, target_count, preprint_id)) try: on_preprint_updated(preprint_id) except Exception as e: # TODO: This reliably fails for certain nodes with # IncompleteRead(0 bytes read) failures.append(preprint_id) logger.warn( 'Encountered exception {} while posting to SHARE for preprint {}' .format(e, preprint_id)) else: successes.append(preprint_id) logger.info('Successes: {}'.format(successes)) logger.info('Failures: {}'.format(failures))
def update_share_preprint_modified_dates(dry_run=False): for preprint in PreprintService.objects.filter( date_modified__lt=F('node__date_modified')): if dry_run: logger.info('Would have sent ' + preprint._id + ' data to SHARE') else: on_preprint_updated(preprint._id) logger.info(preprint._id + ' data sent to SHARE')
def delete(self, request, *args, **kwargs): preprint = self.get_object() if settings.SHARE_URL and settings.SHARE_API_TOKEN: on_preprint_updated(preprint._id) update_admin_log( user_id=self.request.user.id, object_id=preprint._id, object_repr='Preprint', message='Preprint Reindexed (SHARE): {}'.format(preprint._id), action_flag=REINDEX_SHARE ) return redirect(reverse_preprint(self.kwargs.get('guid')))
def update(dry_run=False): created_dict = {'osf': []} for hier in BEPRESS_CHANGES['create']: new_text = hier.split(':')[-1] bepress_parent = BEPRESS_PROVIDER.subjects.get( text=hier.split(':')[-2]) logger.info('Creating osf - {}'.format(new_text)) bepress_subject = Subject.objects.create(parent=bepress_parent, provider=BEPRESS_PROVIDER, text=new_text) created_dict['osf'].append(new_text) for custom_parent in bepress_parent.aliases.all(): if not bepress_parent.children.count() > 1 or ( custom_parent.children.exists() and set( bepress_parent.children.exclude( text=new_text).values_list( 'text', flat=True)).issubset( set( custom_parent.children.values_list( 'text', flat=True)))): # Either children were included in the custom taxonomy or they didn't exist before, probably logger.info('Creating {} - {}'.format( custom_parent.provider._id, new_text)) Subject.objects.create(parent=custom_parent, provider=custom_parent.provider, text=new_text, bepress_subject=bepress_subject) if custom_parent.provider._id not in created_dict: created_dict[custom_parent.provider._id] = [] created_dict[custom_parent.provider._id].append(new_text) for old, new in BEPRESS_CHANGES['rename'].items(): logger.info('Renaming `{}`->`{}`'.format(old, new)) to_update = Subject.objects.filter(text=old) affected_preprints = set( to_update.exclude(preprint_services__isnull=True).values_list( 'preprint_services__guids___id', flat=True)) to_update.update(text=new) for preprint_id in affected_preprints: logger.info( 'Notifying SHARE about preprint {} change'.format(preprint_id)) if not dry_run: on_preprint_updated(preprint_id) for provider_id, list_of_subjs in created_dict.items(): logger.info('Created {} new subjects on {}: "{}"'.format( len(list_of_subjs), provider_id, ', '.join(list_of_subjs)))
def map_preprints_to_custom_subjects(custom_provider, merge_dict, dry_run=False): for preprint in PreprintService.objects.filter(provider=custom_provider): logger.info('Preparing to migrate preprint {}'.format(preprint.id)) old_hier = preprint.subject_hierarchy subjects_to_map = [hier[-1] for hier in old_hier] merged_subject_ids = set( Subject.objects.filter(provider=custom_provider, text__in=[ merge_dict[k] for k in set(merge_dict.keys()) & set([s.text for s in subjects_to_map]) ]).values_list('id', flat=True)) subject_ids_to_map = set(s.id for s in subjects_to_map if s.text not in merge_dict.keys()) aliased_subject_ids = set( Subject.objects.filter(bepress_subject__id__in=subject_ids_to_map, provider=custom_provider).values_list( 'id', flat=True)) | merged_subject_ids aliased_hiers = [ s.object_hierarchy for s in Subject.objects.filter(id__in=aliased_subject_ids) ] old_subjects = list(preprint.subjects.values_list('id', flat=True)) preprint.subjects.clear() for hier in aliased_hiers: validate_subject_hierarchy([s._id for s in hier]) for s in hier: preprint.subjects.add(s) # Update preprint in SHARE if not dry_run: on_preprint_updated(preprint._id, old_subjects=old_subjects, update_share=True) preprint.reload() new_hier = [ s.object_hierarchy for s in preprint.subjects.exclude( children__in=preprint.subjects.all()) ] logger.info( 'Successfully migrated preprint {}.\n\tOld hierarchy:{}\n\tNew hierarchy:{}' .format(preprint.id, old_hier, new_hier))
def map_preprints_to_custom_subjects(custom_provider, merge_dict, dry_run=False): for preprint in PreprintService.objects.filter(provider=custom_provider): logger.info('Preparing to migrate preprint {}'.format(preprint.id)) old_hier = preprint.subject_hierarchy subjects_to_map = [hier[-1] for hier in old_hier] merged_subject_ids = set(Subject.objects.filter(provider=custom_provider, text__in=[merge_dict[k] for k in set(merge_dict.keys()) & set([s.text for s in subjects_to_map])]).values_list('id', flat=True)) subject_ids_to_map = set(s.id for s in subjects_to_map if s.text not in merge_dict.keys()) aliased_subject_ids = set(Subject.objects.filter(bepress_subject__id__in=subject_ids_to_map, provider=custom_provider).values_list('id', flat=True)) | merged_subject_ids aliased_hiers = [s.object_hierarchy for s in Subject.objects.filter(id__in=aliased_subject_ids)] old_subjects = list(preprint.subjects.values_list('id', flat=True)) preprint.subjects.clear() for hier in aliased_hiers: validate_subject_hierarchy([s._id for s in hier]) for s in hier: preprint.subjects.add(s) # Update preprint in SHARE if not dry_run: on_preprint_updated(preprint._id, old_subjects=old_subjects, update_share=True) preprint.reload() new_hier = [s.object_hierarchy for s in preprint.subjects.exclude(children__in=preprint.subjects.all())] logger.info('Successfully migrated preprint {}.\n\tOld hierarchy:{}\n\tNew hierarchy:{}'.format(preprint.id, old_hier, new_hier))
def test_send_to_share_is_true(self, mock_requests): self.preprint.provider.access_token = 'Snowmobiling' self.preprint.provider.save() on_preprint_updated(self.preprint._id) assert mock_requests.post.called
def test_send_to_share_is_true(self, mock_requests): self.preprint.provider.access_token = 'Snowmobiling' self.preprint.provider.save() on_preprint_updated(self.preprint._id) assert mock_requests.post.called