def load(patterns, full_reindex): ''' Load one or more CADA CSV files matching patterns ''' header('Loading CSV files') for pattern in patterns: for filename in iglob(pattern): echo('Loading {}'.format(white(filename))) with open(filename) as f: reader = csv.reader(f) # Skip header reader.next() for idx, row in enumerate(reader, 1): try: advice = csv.from_row(row) skipped = False if not full_reindex: index(advice) echo('.' if idx % 50 else white(idx), nl=False) except Exception: echo(cyan('s') if idx % 50 else white('{0}(s)'.format(idx)), nl=False) skipped = True if skipped: echo(white('{}(s)'.format(idx)) if idx % 50 else '') else: echo(white(idx) if idx % 50 else '') success('Processed {0} rows'.format(idx)) if full_reindex: reindex()
def fix(csvfile): '''Apply a fix (ie. remove plain names)''' header('Apply fixes from {}', csvfile.name) bads = [] reader = csv.reader(csvfile) reader.next() # Skip header for id, _, sources, dests in reader: advice = Advice.objects.get(id=id) sources = [s.strip() for s in sources.split(',') if s.strip()] dests = [d.strip() for d in dests.split(',') if d.strip()] if not len(sources) == len(dests): bads.append(id) continue for source, dest in zip(sources, dests): echo('{0}: Replace {1} with {2}', white(id), white(source), white(dest)) advice.subject = advice.subject.replace(source, dest) advice.content = advice.content.replace(source, dest) advice.save() index(advice) for id in bads: echo('{0}: Replacements length not matching', white(id)) success('Done')
def test_search_with_content(client, advice_factory): for advice in advice_factory.create_batch(3): search.index(advice) search.es.indices.refresh(index=search.es.index_name) response = client.get(url_for('api.search')) assert response.status_code == 200 assert len(response.json['advices']) == 3
def fix(csvfile): '''Apply a fix (ie. remove plain names)''' header('Apply fixes from {}', csvfile.name) bads = [] reader = csv.reader(csvfile) reader.next() # Skip header for id, _, sources, dests in reader: advice = Advice.objects.get(id=id) sources = [s.strip() for s in sources.split(',') if s.strip()] dests = [d.strip() for d in dests.split(',') if d.strip()] if not len(sources) == len(dests): bads.append(id) continue for source, dest in zip(sources, dests): echo('{0}: Replace {1} with {2}', white(id), white(source), white(dest)) advice.subject = advice.subject.replace(source, dest) advice.content = advice.content.replace(source, dest) advice.save() index(advice) for id in bads: echo('{0}: Replacements length not matching', white(id)) success('Done')
def load(patterns, full_reindex): ''' Load one or more CADA CSV files matching patterns ''' header('Loading CSV files') for pattern in patterns: for filename in iglob(pattern): echo('Loading {}'.format(white(filename))) with open(filename) as f: reader = csv.reader(f) # Skip header reader.next() for idx, row in enumerate(reader, 1): try: advice = csv.from_row(row) skipped = False if not full_reindex: index(advice) echo('.' if idx % 50 else white(idx), nl=False) except Exception: echo(cyan('s') if idx % 50 else white('{0}(s)'.format(idx)), nl=False) skipped = True if skipped: echo(white('{}(s)'.format(idx)) if idx % 50 else '') else: echo(white(idx) if idx % 50 else '') success('Processed {0} rows'.format(idx)) if full_reindex: reindex()
def reindex(): '''Reindex all advices''' print('Deleting index {0}'.format(es.index_name)) if es.indices.exists(es.index_name): es.indices.delete(index=es.index_name) es.initialize() for idx, advice in enumerate(Advice.objects, 1): index(advice) print('.' if idx % 50 else idx, end='') es.indices.refresh(index=es.index_name) print('\nIndexed {0} advices'.format(idx))
def reindex(): '''Reindex all advices''' print('Deleting index {0}'.format(es.index_name)) if es.indices.exists(es.index_name): es.indices.delete(index=es.index_name) es.initialize() idx = 0 for idx, advice in enumerate(Advice.objects, 1): index(advice) print('.' if idx % 50 else idx, end='') es.indices.refresh(index=es.index_name) print('\nIndexed {0} advices'.format(idx))
def reindex(): '''Reindex all advices''' header('Reindexing all advices') echo('Deleting index {0}', white(es.index_name)) if es.indices.exists(es.index_name): es.indices.delete(index=es.index_name) es.initialize() idx = 0 for idx, advice in enumerate(Advice.objects, 1): index(advice) echo('.' if idx % 50 else white(idx), nl=False) echo(white(idx) if idx % 50 else '') es.indices.refresh(index=es.index_name) success('Indexed {0} advices', idx)
def reindex(): '''Reindex all advices''' header('Reindexing all advices') echo('Deleting index {0}', white(es.index_name)) if es.indices.exists(es.index_name): es.indices.delete(index=es.index_name) es.initialize() idx = 0 for idx, advice in enumerate(Advice.objects, 1): index(advice) echo('.' if idx % 50 else white(idx), nl=False) echo(white(idx) if idx % 50 else '') es.indices.refresh(index=es.index_name) success('Indexed {0} advices', idx)
def fix(filename): bads = [] with open(filename) as csvfile: reader = csv.reader(csvfile) reader.next() # Skip header for id, _, sources, dests in reader: advice = Advice.objects.get(id=id) sources = [s.strip() for s in sources.split(',') if s.strip()] dests = [d.strip() for d in dests.split(',') if d.strip()] if not len(sources) == len(dests): bads.append(id) continue for source, dest in zip(sources, dests): print('{0}: Replace {1} with {2}'.format(id, source, dest)) advice.subject = advice.subject.replace(source, dest) advice.content = advice.content.replace(source, dest) advice.save() index(advice) print('') for id in bads: print('{0}: Replacements length not matching'.format(id)) print('\nDone')
def fix(filename): bads = [] with open(filename) as csvfile: reader = csv.reader(csvfile) reader.next() # Skip header for id, _, sources, dests in reader: advice = Advice.objects.get(id=id) sources = [s.strip() for s in sources.split(',') if s.strip()] dests = [d.strip() for d in dests.split(',') if d.strip()] if not len(sources) == len(dests): bads.append(id) continue for source, dest in zip(sources, dests): print('{0}: Replace {1} with {2}'.format(id, source, dest)) advice.subject = advice.subject.replace(source, dest) advice.content = advice.content.replace(source, dest) advice.save() index(advice) print('') for id in bads: print('{0}: Replacements length not matching'.format(id)) print('\nDone')
def test_search_with_content(client, advice_factory): for advice in advice_factory.create_batch(3): search.index(advice) search.es.indices.refresh(index=search.es.index_name) assert client.get(url_for('site.search')).status_code == 200
def test_render_home_with_content(client, advice_factory): for advice in advice_factory.create_batch(3): search.index(advice) search.es.indices.refresh(index=search.es.index_name) assert client.get(url_for('site.home')).status_code == 200