def __init__(self, config, use_shards=True): self.student_filter = filter_factory(config) self.batch_size = config.importer_batch_size self.filename = config.import_filename self.in_file = gcs.open(self.filename) self.reader = csv.reader(self.in_file) self.imported = 0 self.rejected = 0 if use_shards: LOG.info("Checking student shards") StudentShard.init_shards(config) self.shard_gen = StudentShard.circular_shard_generator() else: self.shard_gen = None
def test_abstract_csv_importer(self): dummy_count = self.config.estimated_student_count self.write_stub_student_file(dummy_count) # Basic row conversion imp = StubCsvImporter(self.config, use_shards=False) stu = imp.convert_row("dummy, row") imp = StubCsvImporter(self.config, use_shards=True) stu = imp.convert_row("dummy, row") # Input file conversion, no shards imp = StubCsvImporter(self.config, use_shards=False) self.check_importer_count(dummy_count, imp, multi=True) imp = StubCsvImporter(self.config, use_shards=False) self.check_importer_count(dummy_count, imp, multi=False) # Input file conversion, with shards StudentShard.init_shards(self.config) imp = StubCsvImporter(self.config, use_shards=True) self.check_importer_count(dummy_count, imp, multi=True) imp = StubCsvImporter(self.config, use_shards=True) self.check_importer_count(dummy_count, imp, multi=False)
def setup_dummy_shards(): config = get_dummy_config() StudentShard.init_shards(config)