def write_to_dir(records, output_dir, max_records=1000, encoding='utf-8'):
"""Check if the output directory exists, and creates it if it does not.
:param records: harvested records.
:param output_dir: directory where the output should be sent.
:param max_records: max number of records to be written in a single file.
:return: paths to files created, total number of records
"""
if not records:
return [], 0
output_path = check_or_create_dir(output_dir)
files_created = [create_file_name(output_path)]
total = 0 # total number of records processed
f = codecs.open(files_created[0], 'w+', encoding=encoding)
f.write('<ListRecords>')
for record in records:
total += 1
if total > 1 and total % max_records == 0:
# we need a new file to write to
f.close()
files_created.append(create_file_name(output_path))
f = codecs.open(files_created[-1], 'w+', encoding=encoding)
f.write(record.raw)
f.write('</ListRecords>')
f.close()
return files_created, total
def write_to_dir(records, output_dir, max_records=1000, encoding='utf-8'):
"""Check if the output directory exists, and creates it if it does not.
:param records: harvested records.
:param output_dir: directory where the output should be sent.
:param max_records: max number of records to be written in a single file.
:return: paths to files created, total number of records
"""
if not records:
return [], 0
output_path = check_or_create_dir(output_dir)
files_created = [create_file_name(output_path)]
total = 0 # total number of records processed
f = codecs.open(files_created[0], 'w+', encoding=encoding)
f.write('<ListRecords>')
for record in records:
total += 1
if total > 1 and total % max_records == 0:
# we need a new file to write to
f.close()
files_created.append(create_file_name(output_path))
f = codecs.open(files_created[-1], 'w+', encoding=encoding)
f.write(record.raw)
f.write('</ListRecords>')
f.close()
return files_created, total
def test_create_file_name(tmpdir):
"""oaiharvest - testing dir creation."""
create_file_name(tmpdir.dirname + 'foo')
def test_create_file_name(tmpdir):
"""oaiharvest - testing dir creation."""
create_file_name(tmpdir.dirname + 'foo')