def test_export_csv_empty(client):
response = client.get(url_for('site.export_csv'))
assert response.status_code == 200
assert response.mimetype == 'text/csv'
assert response.is_streamed
reader = csv.reader(io.BytesIO(response.data))
assert len([row for row in reader]) == 1
def load(patterns, full_reindex):
'''
Load one or more CADA CSV files matching patterns
'''
header('Loading CSV files')
for pattern in patterns:
for filename in iglob(pattern):
echo('Loading {}'.format(white(filename)))
with open(filename) as f:
reader = csv.reader(f)
# Skip header
reader.next()
for idx, row in enumerate(reader, 1):
try:
advice = csv.from_row(row)
skipped = False
if not full_reindex:
index(advice)
echo('.' if idx % 50 else white(idx), nl=False)
except Exception:
echo(cyan('s') if idx %
50 else white('{0}(s)'.format(idx)),
nl=False)
skipped = True
if skipped:
echo(white('{}(s)'.format(idx)) if idx % 50 else '')
else:
echo(white(idx) if idx % 50 else '')
success('Processed {0} rows'.format(idx))
if full_reindex:
reindex()
def fix(csvfile):
'''Apply a fix (ie. remove plain names)'''
header('Apply fixes from {}', csvfile.name)
bads = []
reader = csv.reader(csvfile)
reader.next() # Skip header
for id, _, sources, dests in reader:
advice = Advice.objects.get(id=id)
sources = [s.strip() for s in sources.split(',') if s.strip()]
dests = [d.strip() for d in dests.split(',') if d.strip()]
if not len(sources) == len(dests):
bads.append(id)
continue
for source, dest in zip(sources, dests):
echo('{0}: Replace {1} with {2}', white(id), white(source), white(dest))
advice.subject = advice.subject.replace(source, dest)
advice.content = advice.content.replace(source, dest)
advice.save()
index(advice)
for id in bads:
echo('{0}: Replacements length not matching', white(id))
success('Done')
def test_export_csv_empty(client):
response = client.get(url_for('site.export_csv'))
assert response.status_code == 200
assert response.mimetype == 'text/csv'
assert response.is_streamed
reader = csv.reader(io.BytesIO(response.data))
assert len([row for row in reader]) == 1
def fix(csvfile):
'''Apply a fix (ie. remove plain names)'''
header('Apply fixes from {}', csvfile.name)
bads = []
reader = csv.reader(csvfile)
reader.next() # Skip header
for id, _, sources, dests in reader:
advice = Advice.objects.get(id=id)
sources = [s.strip() for s in sources.split(',') if s.strip()]
dests = [d.strip() for d in dests.split(',') if d.strip()]
if not len(sources) == len(dests):
bads.append(id)
continue
for source, dest in zip(sources, dests):
echo('{0}: Replace {1} with {2}', white(id), white(source), white(dest))
advice.subject = advice.subject.replace(source, dest)
advice.content = advice.content.replace(source, dest)
advice.save()
index(advice)
for id in bads:
echo('{0}: Replacements length not matching', white(id))
success('Done')
def load(patterns, full_reindex):
'''
Load one or more CADA CSV files matching patterns
'''
header('Loading CSV files')
for pattern in patterns:
for filename in iglob(pattern):
echo('Loading {}'.format(white(filename)))
with open(filename) as f:
reader = csv.reader(f)
# Skip header
reader.next()
for idx, row in enumerate(reader, 1):
try:
advice = csv.from_row(row)
skipped = False
if not full_reindex:
index(advice)
echo('.' if idx % 50 else white(idx), nl=False)
except Exception:
echo(cyan('s') if idx % 50 else white('{0}(s)'.format(idx)), nl=False)
skipped = True
if skipped:
echo(white('{}(s)'.format(idx)) if idx % 50 else '')
else:
echo(white(idx) if idx % 50 else '')
success('Processed {0} rows'.format(idx))
if full_reindex:
reindex()
def test_export_csv_with_content(client, advice_factory):
nb_advices = 3
advice_factory.create_batch(nb_advices)
response = client.get(url_for('site.export_csv'))
assert response.status_code == 200
assert response.mimetype == 'text/csv'
assert response.is_streamed
reader = csv.reader(io.BytesIO(response.data))
assert len([row for row in reader]) == 1 + nb_advices
def test_export_csv_with_content(client, advice, advice_factory):
nb_advices = 3
total_advices = nb_advices + 1 # One is already created to test all parts
advice_factory.create_batch(nb_advices)
response = client.get(url_for('site.export_csv'))
assert response.status_code == 200
assert response.mimetype == 'text/csv'
assert response.is_streamed
reader = csv.reader(io.BytesIO(response.data))
assert len([row for row in reader]) == 1 + total_advices # Include headers
def load(patterns):
'''Load a CADA CSV file'''
for pattern in patterns:
for filename in iglob(pattern):
print('Loading', filename)
with open(filename) as f:
reader = csv.reader(f)
# Skip header
reader.next()
for idx, row in enumerate(reader, 1):
csv.from_row(row)
print('.' if idx % 50 else idx, end='')
print('\nProcessed {0} rows'.format(idx))
reindex()
def load(patterns, full_reindex):
'''Load a CADA CSV file'''
for pattern in patterns:
for filename in iglob(pattern):
print('Loading', filename)
with open(filename) as f:
reader = csv.reader(f)
# Skip header
reader.next()
for idx, row in enumerate(reader, 1):
csv.from_row(row)
print('.' if idx % 50 else idx, end='')
print('\nProcessed {0} rows'.format(idx))
if full_reindex:
reindex()
def fix(filename):
bads = []
with open(filename) as csvfile:
reader = csv.reader(csvfile)
reader.next() # Skip header
for id, _, sources, dests in reader:
advice = Advice.objects.get(id=id)
sources = [s.strip() for s in sources.split(',') if s.strip()]
dests = [d.strip() for d in dests.split(',') if d.strip()]
if not len(sources) == len(dests):
bads.append(id)
continue
for source, dest in zip(sources, dests):
print('{0}: Replace {1} with {2}'.format(id, source, dest))
advice.subject = advice.subject.replace(source, dest)
advice.content = advice.content.replace(source, dest)
advice.save()
print('')
for id in bads:
print('{0}: Replacements length not matching'.format(id))
print('\nDone')
def fix(filename):
bads = []
with open(filename) as csvfile:
reader = csv.reader(csvfile)
reader.next() # Skip header
for id, _, sources, dests in reader:
advice = Advice.objects.get(id=id)
sources = [s.strip() for s in sources.split(',') if s.strip()]
dests = [d.strip() for d in dests.split(',') if d.strip()]
if not len(sources) == len(dests):
bads.append(id)
continue
for source, dest in zip(sources, dests):
print('{0}: Replace {1} with {2}'.format(id, source, dest))
advice.subject = advice.subject.replace(source, dest)
advice.content = advice.content.replace(source, dest)
advice.save()
index(advice)
print('')
for id in bads:
print('{0}: Replacements length not matching'.format(id))
print('\nDone')