def load(patterns, full_reindex):
'''
Load one or more CADA CSV files matching patterns
'''
header('Loading CSV files')
for pattern in patterns:
for filename in iglob(pattern):
echo('Loading {}'.format(white(filename)))
with open(filename) as f:
reader = csv.reader(f)
# Skip header
reader.next()
for idx, row in enumerate(reader, 1):
try:
advice = csv.from_row(row)
skipped = False
if not full_reindex:
index(advice)
echo('.' if idx % 50 else white(idx), nl=False)
except Exception:
echo(cyan('s') if idx %
50 else white('{0}(s)'.format(idx)),
nl=False)
skipped = True
if skipped:
echo(white('{}(s)'.format(idx)) if idx % 50 else '')
else:
echo(white(idx) if idx % 50 else '')
success('Processed {0} rows'.format(idx))
if full_reindex:
reindex()
def fix(csvfile):
'''Apply a fix (ie. remove plain names)'''
header('Apply fixes from {}', csvfile.name)
bads = []
reader = csv.reader(csvfile)
reader.next() # Skip header
for id, _, sources, dests in reader:
advice = Advice.objects.get(id=id)
sources = [s.strip() for s in sources.split(',') if s.strip()]
dests = [d.strip() for d in dests.split(',') if d.strip()]
if not len(sources) == len(dests):
bads.append(id)
continue
for source, dest in zip(sources, dests):
echo('{0}: Replace {1} with {2}', white(id), white(source), white(dest))
advice.subject = advice.subject.replace(source, dest)
advice.content = advice.content.replace(source, dest)
advice.save()
index(advice)
for id in bads:
echo('{0}: Replacements length not matching', white(id))
success('Done')
def test_search_with_content(client, advice_factory):
for advice in advice_factory.create_batch(3):
search.index(advice)
search.es.indices.refresh(index=search.es.index_name)
response = client.get(url_for('api.search'))
assert response.status_code == 200
assert len(response.json['advices']) == 3
def fix(csvfile):
'''Apply a fix (ie. remove plain names)'''
header('Apply fixes from {}', csvfile.name)
bads = []
reader = csv.reader(csvfile)
reader.next() # Skip header
for id, _, sources, dests in reader:
advice = Advice.objects.get(id=id)
sources = [s.strip() for s in sources.split(',') if s.strip()]
dests = [d.strip() for d in dests.split(',') if d.strip()]
if not len(sources) == len(dests):
bads.append(id)
continue
for source, dest in zip(sources, dests):
echo('{0}: Replace {1} with {2}', white(id), white(source), white(dest))
advice.subject = advice.subject.replace(source, dest)
advice.content = advice.content.replace(source, dest)
advice.save()
index(advice)
for id in bads:
echo('{0}: Replacements length not matching', white(id))
success('Done')
def load(patterns, full_reindex):
'''
Load one or more CADA CSV files matching patterns
'''
header('Loading CSV files')
for pattern in patterns:
for filename in iglob(pattern):
echo('Loading {}'.format(white(filename)))
with open(filename) as f:
reader = csv.reader(f)
# Skip header
reader.next()
for idx, row in enumerate(reader, 1):
try:
advice = csv.from_row(row)
skipped = False
if not full_reindex:
index(advice)
echo('.' if idx % 50 else white(idx), nl=False)
except Exception:
echo(cyan('s') if idx % 50 else white('{0}(s)'.format(idx)), nl=False)
skipped = True
if skipped:
echo(white('{}(s)'.format(idx)) if idx % 50 else '')
else:
echo(white(idx) if idx % 50 else '')
success('Processed {0} rows'.format(idx))
if full_reindex:
reindex()
def reindex():
'''Reindex all advices'''
print('Deleting index {0}'.format(es.index_name))
if es.indices.exists(es.index_name):
es.indices.delete(index=es.index_name)
es.initialize()
for idx, advice in enumerate(Advice.objects, 1):
index(advice)
print('.' if idx % 50 else idx, end='')
es.indices.refresh(index=es.index_name)
print('\nIndexed {0} advices'.format(idx))
def reindex():
'''Reindex all advices'''
print('Deleting index {0}'.format(es.index_name))
if es.indices.exists(es.index_name):
es.indices.delete(index=es.index_name)
es.initialize()
idx = 0
for idx, advice in enumerate(Advice.objects, 1):
index(advice)
print('.' if idx % 50 else idx, end='')
es.indices.refresh(index=es.index_name)
print('\nIndexed {0} advices'.format(idx))
def reindex():
'''Reindex all advices'''
header('Reindexing all advices')
echo('Deleting index {0}', white(es.index_name))
if es.indices.exists(es.index_name):
es.indices.delete(index=es.index_name)
es.initialize()
idx = 0
for idx, advice in enumerate(Advice.objects, 1):
index(advice)
echo('.' if idx % 50 else white(idx), nl=False)
echo(white(idx) if idx % 50 else '')
es.indices.refresh(index=es.index_name)
success('Indexed {0} advices', idx)
def reindex():
'''Reindex all advices'''
header('Reindexing all advices')
echo('Deleting index {0}', white(es.index_name))
if es.indices.exists(es.index_name):
es.indices.delete(index=es.index_name)
es.initialize()
idx = 0
for idx, advice in enumerate(Advice.objects, 1):
index(advice)
echo('.' if idx % 50 else white(idx), nl=False)
echo(white(idx) if idx % 50 else '')
es.indices.refresh(index=es.index_name)
success('Indexed {0} advices', idx)
def fix(filename):
bads = []
with open(filename) as csvfile:
reader = csv.reader(csvfile)
reader.next() # Skip header
for id, _, sources, dests in reader:
advice = Advice.objects.get(id=id)
sources = [s.strip() for s in sources.split(',') if s.strip()]
dests = [d.strip() for d in dests.split(',') if d.strip()]
if not len(sources) == len(dests):
bads.append(id)
continue
for source, dest in zip(sources, dests):
print('{0}: Replace {1} with {2}'.format(id, source, dest))
advice.subject = advice.subject.replace(source, dest)
advice.content = advice.content.replace(source, dest)
advice.save()
index(advice)
print('')
for id in bads:
print('{0}: Replacements length not matching'.format(id))
print('\nDone')
def fix(filename):
bads = []
with open(filename) as csvfile:
reader = csv.reader(csvfile)
reader.next() # Skip header
for id, _, sources, dests in reader:
advice = Advice.objects.get(id=id)
sources = [s.strip() for s in sources.split(',') if s.strip()]
dests = [d.strip() for d in dests.split(',') if d.strip()]
if not len(sources) == len(dests):
bads.append(id)
continue
for source, dest in zip(sources, dests):
print('{0}: Replace {1} with {2}'.format(id, source, dest))
advice.subject = advice.subject.replace(source, dest)
advice.content = advice.content.replace(source, dest)
advice.save()
index(advice)
print('')
for id in bads:
print('{0}: Replacements length not matching'.format(id))
print('\nDone')
def test_search_with_content(client, advice_factory):
for advice in advice_factory.create_batch(3):
search.index(advice)
search.es.indices.refresh(index=search.es.index_name)
assert client.get(url_for('site.search')).status_code == 200
def test_render_home_with_content(client, advice_factory):
for advice in advice_factory.create_batch(3):
search.index(advice)
search.es.indices.refresh(index=search.es.index_name)
assert client.get(url_for('site.home')).status_code == 200