def __init__(self, config, use_shards=True):
self.student_filter = filter_factory(config)
self.batch_size = config.importer_batch_size
self.filename = config.import_filename
self.in_file = gcs.open(self.filename)
self.reader = csv.reader(self.in_file)
self.imported = 0
self.rejected = 0
if use_shards:
LOG.info("Checking student shards")
StudentShard.init_shards(config)
self.shard_gen = StudentShard.circular_shard_generator()
else:
self.shard_gen = None
def test_abstract_csv_importer(self):
dummy_count = self.config.estimated_student_count
self.write_stub_student_file(dummy_count)
# Basic row conversion
imp = StubCsvImporter(self.config, use_shards=False)
stu = imp.convert_row("dummy, row")
imp = StubCsvImporter(self.config, use_shards=True)
stu = imp.convert_row("dummy, row")
# Input file conversion, no shards
imp = StubCsvImporter(self.config, use_shards=False)
self.check_importer_count(dummy_count, imp, multi=True)
imp = StubCsvImporter(self.config, use_shards=False)
self.check_importer_count(dummy_count, imp, multi=False)
# Input file conversion, with shards
StudentShard.init_shards(self.config)
imp = StubCsvImporter(self.config, use_shards=True)
self.check_importer_count(dummy_count, imp, multi=True)
imp = StubCsvImporter(self.config, use_shards=True)
self.check_importer_count(dummy_count, imp, multi=False)
def setup_dummy_shards():
config = get_dummy_config()
StudentShard.init_shards(config)
