def write_to_dir(records, output_dir, max_records=1000, encoding='utf-8'): """Check if the output directory exists, and creates it if it does not. :param records: harvested records. :param output_dir: directory where the output should be sent. :param max_records: max number of records to be written in a single file. :return: paths to files created, total number of records """ if not records: return [], 0 output_path = check_or_create_dir(output_dir) files_created = [create_file_name(output_path)] total = 0 # total number of records processed f = codecs.open(files_created[0], 'w+', encoding=encoding) f.write('<ListRecords>') for record in records: total += 1 if total > 1 and total % max_records == 0: # we need a new file to write to f.close() files_created.append(create_file_name(output_path)) f = codecs.open(files_created[-1], 'w+', encoding=encoding) f.write(record.raw) f.write('</ListRecords>') f.close() return files_created, total
def write_to_dir(records, output_dir, max_records=1000, encoding='utf-8'): """Check if the output directory exists, and creates it if it does not. :param records: harvested records. :param output_dir: directory where the output should be sent. :param max_records: max number of records to be written in a single file. :return: paths to files created, total number of records """ if not records: return [], 0 output_path = check_or_create_dir(output_dir) files_created = [create_file_name(output_path)] total = 0 # total number of records processed f = codecs.open(files_created[0], 'w+', encoding=encoding) f.write('<ListRecords>') for record in records: total += 1 if total > 1 and total % max_records == 0: # we need a new file to write to f.close() files_created.append(create_file_name(output_path)) f = codecs.open(files_created[-1], 'w+', encoding=encoding) f.write(record.raw) f.write('</ListRecords>') f.close() return files_created, total
def test_check_or_create_dir(app, tmpdir): """oaiharvest - testing dir creation.""" with app.app_context(): check_or_create_dir(app.instance_path) check_or_create_dir(tmpdir.dirname + 'foo')
def test_check_or_create_dir(app, tmpdir): """oaiharvest - testing dir creation.""" with app.app_context(): check_or_create_dir(app.instance_path) check_or_create_dir(tmpdir.dirname + 'foo')