예제 #1
0
def test_export_csv_empty(client):
    response = client.get(url_for('site.export_csv'))
    assert response.status_code == 200
    assert response.mimetype == 'text/csv'
    assert response.is_streamed
    reader = csv.reader(io.BytesIO(response.data))
    assert len([row for row in reader]) == 1
예제 #2
0
def load(patterns, full_reindex):
    '''
    Load one or more CADA CSV files matching patterns
    '''
    header('Loading CSV files')
    for pattern in patterns:
        for filename in iglob(pattern):
            echo('Loading {}'.format(white(filename)))
            with open(filename) as f:
                reader = csv.reader(f)
                # Skip header
                reader.next()
                for idx, row in enumerate(reader, 1):
                    try:
                        advice = csv.from_row(row)
                        skipped = False
                        if not full_reindex:
                            index(advice)
                        echo('.' if idx % 50 else white(idx), nl=False)
                    except Exception:
                        echo(cyan('s') if idx %
                             50 else white('{0}(s)'.format(idx)),
                             nl=False)
                        skipped = True
                if skipped:
                    echo(white('{}(s)'.format(idx)) if idx % 50 else '')
                else:
                    echo(white(idx) if idx % 50 else '')
                success('Processed {0} rows'.format(idx))
    if full_reindex:
        reindex()
예제 #3
0
파일: commands.py 프로젝트: etalab/cada
def fix(csvfile):
    '''Apply a fix (ie. remove plain names)'''
    header('Apply fixes from {}', csvfile.name)
    bads = []
    reader = csv.reader(csvfile)
    reader.next()  # Skip header

    for id, _, sources, dests in reader:
        advice = Advice.objects.get(id=id)
        sources = [s.strip() for s in sources.split(',') if s.strip()]
        dests = [d.strip() for d in dests.split(',') if d.strip()]

        if not len(sources) == len(dests):
            bads.append(id)
            continue

        for source, dest in zip(sources, dests):
            echo('{0}: Replace {1} with {2}', white(id), white(source), white(dest))
            advice.subject = advice.subject.replace(source, dest)
            advice.content = advice.content.replace(source, dest)

        advice.save()
        index(advice)

    for id in bads:
        echo('{0}: Replacements length not matching', white(id))

    success('Done')
예제 #4
0
파일: test_csv.py 프로젝트: etalab/cada
def test_export_csv_empty(client):
    response = client.get(url_for('site.export_csv'))
    assert response.status_code == 200
    assert response.mimetype == 'text/csv'
    assert response.is_streamed
    reader = csv.reader(io.BytesIO(response.data))
    assert len([row for row in reader]) == 1
예제 #5
0
파일: commands.py 프로젝트: psorianom/cada
def fix(csvfile):
    '''Apply a fix (ie. remove plain names)'''
    header('Apply fixes from {}', csvfile.name)
    bads = []
    reader = csv.reader(csvfile)
    reader.next()  # Skip header

    for id, _, sources, dests in reader:
        advice = Advice.objects.get(id=id)
        sources = [s.strip() for s in sources.split(',') if s.strip()]
        dests = [d.strip() for d in dests.split(',') if d.strip()]

        if not len(sources) == len(dests):
            bads.append(id)
            continue

        for source, dest in zip(sources, dests):
            echo('{0}: Replace {1} with {2}', white(id), white(source), white(dest))
            advice.subject = advice.subject.replace(source, dest)
            advice.content = advice.content.replace(source, dest)

        advice.save()
        index(advice)

    for id in bads:
        echo('{0}: Replacements length not matching', white(id))

    success('Done')
예제 #6
0
파일: commands.py 프로젝트: etalab/cada
def load(patterns, full_reindex):
    '''
    Load one or more CADA CSV files matching patterns
    '''
    header('Loading CSV files')
    for pattern in patterns:
        for filename in iglob(pattern):
            echo('Loading {}'.format(white(filename)))
            with open(filename) as f:
                reader = csv.reader(f)
                # Skip header
                reader.next()
                for idx, row in enumerate(reader, 1):
                    try:
                        advice = csv.from_row(row)
                        skipped = False
                        if not full_reindex:
                            index(advice)
                        echo('.' if idx % 50 else white(idx), nl=False)
                    except Exception:
                        echo(cyan('s') if idx % 50 else white('{0}(s)'.format(idx)), nl=False)
                        skipped = True
                if skipped:
                    echo(white('{}(s)'.format(idx)) if idx % 50 else '')
                else:
                    echo(white(idx) if idx % 50 else '')
                success('Processed {0} rows'.format(idx))
    if full_reindex:
        reindex()
예제 #7
0
def test_export_csv_with_content(client, advice_factory):
    nb_advices = 3
    advice_factory.create_batch(nb_advices)
    response = client.get(url_for('site.export_csv'))
    assert response.status_code == 200
    assert response.mimetype == 'text/csv'
    assert response.is_streamed
    reader = csv.reader(io.BytesIO(response.data))
    assert len([row for row in reader]) == 1 + nb_advices
예제 #8
0
def test_export_csv_with_content(client, advice, advice_factory):
    nb_advices = 3
    total_advices = nb_advices + 1  # One is already created to test all parts
    advice_factory.create_batch(nb_advices)
    response = client.get(url_for('site.export_csv'))
    assert response.status_code == 200
    assert response.mimetype == 'text/csv'
    assert response.is_streamed
    reader = csv.reader(io.BytesIO(response.data))
    assert len([row for row in reader]) == 1 + total_advices  # Include headers
예제 #9
0
파일: commands.py 프로젝트: pborreli/cada
def load(patterns):
    '''Load a CADA CSV file'''
    for pattern in patterns:
        for filename in iglob(pattern):
            print('Loading', filename)
            with open(filename) as f:
                reader = csv.reader(f)
                # Skip header
                reader.next()
                for idx, row in enumerate(reader, 1):
                    csv.from_row(row)
                    print('.' if idx % 50 else idx, end='')
                print('\nProcessed {0} rows'.format(idx))
    reindex()
예제 #10
0
def load(patterns, full_reindex):
    '''Load a CADA CSV file'''
    for pattern in patterns:
        for filename in iglob(pattern):
            print('Loading', filename)
            with open(filename) as f:
                reader = csv.reader(f)
                # Skip header
                reader.next()
                for idx, row in enumerate(reader, 1):
                    csv.from_row(row)
                    print('.' if idx % 50 else idx, end='')
                print('\nProcessed {0} rows'.format(idx))
    if full_reindex:
        reindex()
예제 #11
0
파일: commands.py 프로젝트: pborreli/cada
def fix(filename):
    bads = []
    with open(filename) as csvfile:
        reader = csv.reader(csvfile)
        reader.next()  # Skip header
        for id, _, sources, dests in reader:
            advice = Advice.objects.get(id=id)
            sources = [s.strip() for s in sources.split(',') if s.strip()]
            dests = [d.strip() for d in dests.split(',') if d.strip()]
            if not len(sources) == len(dests):
                bads.append(id)
                continue
            for source, dest in zip(sources, dests):
                print('{0}: Replace {1} with {2}'.format(id, source, dest))
                advice.subject = advice.subject.replace(source, dest)
                advice.content = advice.content.replace(source, dest)
            advice.save()
        print('')
    for id in bads:
        print('{0}: Replacements length not matching'.format(id))
    print('\nDone')
예제 #12
0
def fix(filename):
    bads = []
    with open(filename) as csvfile:
        reader = csv.reader(csvfile)
        reader.next()  # Skip header
        for id, _, sources, dests in reader:
            advice = Advice.objects.get(id=id)
            sources = [s.strip() for s in sources.split(',') if s.strip()]
            dests = [d.strip() for d in dests.split(',') if d.strip()]
            if not len(sources) == len(dests):
                bads.append(id)
                continue
            for source, dest in zip(sources, dests):
                print('{0}: Replace {1} with {2}'.format(id, source, dest))
                advice.subject = advice.subject.replace(source, dest)
                advice.content = advice.content.replace(source, dest)
            advice.save()
            index(advice)
        print('')
    for id in bads:
        print('{0}: Replacements length not matching'.format(id))
    print('\nDone')