def process_file(filename, input_dir, output_dir):
    input_file = os.path.join(input_dir, filename)
    output_file = os.path.join(output_dir, filename)
    logging.info('Processing file {}...'.format(filename))

    stats = Counter()

    with notempty(openall(output_file, 'wt')) as outf:
        header, it = headtail(parse_file(input_file, True))
        num_fields = len(header)
        do_wsafter = 'wsafter' not in header
        if do_wsafter:
            header.insert(1, 'wsafter')
            logging.debug('Adding the wsafter field...')
        print('\t'.join(header), file=outf)
        for document in it:
            stats['documents'] += 1
            try:
                stats['token_errors'] += fix_invalid_lines(document, num_fields)
                if do_wsafter:
                    add_wsafter(document)
            except ValueError:
                logging.exception(f'Error in file {input_file}')
                raise
            print(document, file=outf)
    return stats
def consumer(output_files: List[str], queue: Queue, header: str,
             documents: int, num_readers: Value, lock: Lock) -> int:
    """
    Reads :class:`Document`s from the shared queue and writes them to one of
    the output files at random.

    :param output_files: list of output file names.
    :param queue: the queue shared with all processes.
    :param header: the header of the tsv files. Written to all output files.
    :param documents: the number of documents to write to an output file.
    :param num_readers: a shared variable holding the number of readers that
                        are still active. This function exits if two conditions
                        are met: the queue is empty and *num_readers* is 0.
    :param lock: a lock that regulates access to *num_readers*.
    :returns: the number of documents written.
    """
    logging.info(f'Consumer started with {len(output_files)} files.')
    output_names = [os.path.basename(f) for f in output_files]
    outfs = [notempty(openall(f, 'wt')) for f in output_files]
    written = [0 for _ in outfs]
    docs_written = 0

    # Write the header
    for outf in outfs:
        print(header, file=outf)

    while outfs:
        i = random.randint(0, len(outfs) - 1)
        try:
            print(queue.get(timeout=5), file=outfs[i])
            written[i] += 1
            docs_written += 1
            if docs_written % 1000 == 0:
                logging.debug(
                    f'Consumer has written {docs_written} documents.')
            if written[i] == documents:
                logging.info(f'Written {documents} documents to '
                             f'{output_names[i]}; closing...')
                outfs[i].close()
                del outfs[i]
                del written[i]
                del output_names[i]
        except Empty:
            with lock:
                if num_readers.value == 0:
                    logging.info('Timeout waiting for queue; cleaning up...')
                    break
        except:
            logging.exception(f'Exception writing {output_names[i]}!')
            sys.exit(3)

    # Close any dangling output files
    for i in range(len(outfs)):
        logging.info(f'Written {written[i]} documents to '
                     f'{output_names[i]}; closing...')
        outfs[i].close()

    logging.info(f'Consumer finished; written {docs_written} documents.')
    return docs_written
def process_file(filename,
                 input_dir,
                 output_dir,
                 languages,
                 language_unit,
                 min_len_str,
                 keep_urls=None,
                 drop_urls=None):
    input_file = os.path.join(input_dir, filename)
    output_file = os.path.join(output_dir, filename)
    logging.info('Processing file {}...'.format(filename))

    stats = Counter()
    it = parse_file(input_file, True, True, True)
    it = each_doc(it, stats)
    if languages:
        if language_unit == 'doc':
            it = filter_languages_doc(it, languages, stats)
        else:
            it = filter_languages_p(it, languages, stats)
    if min_len_str:
        it = filter_length(it, min_len_str, stats)
    if drop_urls:
        # Get the right list: from the Manager or the local one
        url_list = drop_urls if drop_urls.__class__.__name__ == 'DictProxy' \
                             else urls_to_drop  # noqa
        it = filter_urls(it, url_list, stats)
    if keep_urls:
        # Get the right list: from the Manager or the local one
        url_list = keep_urls if keep_urls.__class__.__name__ == 'DictProxy' \
                             else urls_to_keep  # noqa
        it = retain_urls(it, url_list, stats)
    try:
        with notempty(openall(output_file, 'wt')) as outf:
            for doc in it:
                print(doc, file=outf)
    except:
        logging.exception('Got an error.')
    logging.info('Finished processing file {}...'.format(filename))
    return stats
def deduplicate_batch_documents(batch_prefix,
                                output_dir,
                                input_dir=None,
                                ignore_missing_files=False):
    """
    Filters documents not present in the batch and writes the filtered corpus
    files to output_dir. As above, input_dir can be specified if the location
    information in the batch files is outdated.

    Empty files will not be written.
    """
    batch_base = op.basename(batch_prefix)
    logging.info('Filtering batch {}...'.format(batch_base))

    kept, total = 0, 0
    num_files = 0
    for input_file, results in read_batch(batch_prefix):
        file_base = op.basename(input_file)
        url_set = set('_'.join(doc_id) for doc_id in results['id'])
        input_file = op.join(input_dir, file_base) if input_dir else input_file
        if os.path.isfile(input_file):
            with notempty(openall(op.join(output_dir, file_base),
                                  'wt')) as outf:
                for doc_no, doc in enumerate(parse_file(input_file), start=1):
                    if doc.attrs['url'] in url_set:
                        print(doc, file=outf)
                        kept += 1
                total += doc_no
            num_files += 1
        elif ignore_missing_files:
            logging.debug(
                'Input file {} was not found; ignoring...'.format(input_file))
        else:
            raise FileNotFoundError(
                'Input file {} not found.'.format(input_file))

    logging.info('Filtered batch {} of {} files; '
                 'kept {} documents out of {}.'.format(batch_base, num_files,
                                                       kept, total))
    return kept, total
def filter_file(input_file, output_file, uniqs, url_fn: UrlFn) -> FilterStats:
    """
    Filters an index file; i.e. drops all duplicate URLs.
    :param input_file: the input index file
    :param output_file: the output index file
    :param uniqs: the shared dictionary of unique URLs
    :param url_fn: the URL transformation function to apply to each URL. In the
                   scope of this program, this is either hashing or nothing.
    """
    logging.info('Filtering file {}...'.format(input_file))
    stats = FilterStats(old_files=1)
    with openall(input_file, 'rt') as inf, notempty(openall(output_file,
                                                            'wt')) as outf:
        line_no = lines_printed = 0
        for line_no, line in enumerate(map(str.strip, inf), start=1):
            try:
                url, warc, offset, length = line.split()[:7][-6:-2]
                record = IndexRecord(warc, offset, length)
                if record == uniqs.get(url_fn(url)):
                    print(line, file=outf)
                    lines_printed += 1
            except:
                logging.exception('Exception in file {}:{}'.format(
                    input_file, line_no))

    if line_no:
        logging.info('Kept {} URLs out of {} in {}.'.format(
            lines_printed, line_no, input_file))
        stats.old_urls = line_no
    else:
        logging.info('File {} was empty.'.format(input_file))

    if lines_printed:
        stats.new_files = 1
        stats.new_urls = lines_printed
    return stats