def rename(docs, target=None, **kwargs): assert target, "To run this migration you need a target." for doc in docs: raw = RawDocument({ 'doc': doc.doc, 'docID': doc.docID, 'source': target, 'filetype': doc.filetype, 'timestamps': doc.timestamps, 'versions': doc.versions }) assert doc.source != target, "Can't rename {} to {}, names are the same.".format( doc.source, target) if not kwargs.get('dry'): tasks.process_raw(raw) tasks.process_normalized(tasks.normalize(raw, raw['source']), raw) logger.info('Processed document from {} with id {}'.format( doc.source, raw['docID'])) es.delete(index=settings.ELASTIC_INDEX, doc_type=doc.source, id=raw['docID'], ignore=[404]) es.delete(index='share_v1', doc_type=doc.source, id=raw['docID'], ignore=[404]) logger.info('Deleted document from {} with id {}'.format( doc.source, raw['docID']))
def delete(doc, sources=None, **kwargs): assert sources, "To run this migration you need a source." doc.timeout(5).delete() es.delete(index=settings.ELASTIC_INDEX, doc_type=sources, id=doc.docID, ignore=[404]) es.delete(index='share_v1', doc_type=sources, id=doc.docID, ignore=[404]) logger.info('Deleted document from {} with id {}'.format(sources, doc.docID))
def rename(source, target, dry=True): assert source != target, "Can't rename {} to {}, names are the same".format(source, target) count = 0 exceptions = [] for doc in documents(source): count += 1 try: raw = RawDocument({ 'doc': doc.doc, 'docID': doc.docID, 'source': target, 'filetype': doc.filetype, 'timestamps': doc.timestamps, 'versions': doc.versions }) if not dry: process_raw(raw) process_normalized(normalize(raw, raw['source']), raw) logger.info('Processed document from {} with id {}'.format(source, raw['docID'])) except Exception as e: logger.exception(e) exceptions.append(e) else: if not dry: es.delete(index=settings.ELASTIC_INDEX, doc_type=source, id=raw['docID'], ignore=[404]) es.delete(index='share_v1', doc_type=source, id=raw['docID'], ignore=[404]) logger.info('Deleted document from {} with id {}'.format(source, raw['docID'])) if dry: logger.info('Dry run complete') for ex in exceptions: logger.exception(e) logger.info('{} documents processed, with {} exceptions'.format(count, len(exceptions)))
def delete_by_source(source): count = 0 exceptions = [] for doc in documents(source): count += 1 try: doc.delete() es.delete(index=settings.ELASTIC_INDEX, doc_type=source, id=doc.docID, ignore=[404]) es.delete(index='share_v1', doc_type=source, id=doc.docID, ignore=[404]) except Exception as e: logger.exception(e) exceptions.append(e) for ex in exceptions: logger.exception(e) logger.info('{} documents processed, with {} exceptions'.format(count, len(exceptions)))
def rename(doc, target=None, **kwargs): assert target, "To run this migration you need a target." raw = RawDocument({ 'doc': doc.doc, 'docID': doc.docID, 'source': target, 'filetype': doc.filetype, 'timestamps': doc.timestamps, 'versions': doc.versions }) assert doc.source != target, "Can't rename {} to {}, names are the same.".format(doc.source, target) if not kwargs.get('dry'): tasks.process_raw(raw) tasks.process_normalized(tasks.normalize(raw, raw['source']), raw) logger.info('Processed document from {} with id {}'.format(doc.source, raw['docID'])) es.delete(index=settings.ELASTIC_INDEX, doc_type=doc.source, id=raw['docID'], ignore=[404]) es.delete(index='share_v1', doc_type=doc.source, id=raw['docID'], ignore=[404]) logger.info('Deleted document from {} with id {}'.format(doc.source, raw['docID']))