def __init__(self, config, use_shards=True): self.student_filter = filter_factory(config) self.batch_size = config.importer_batch_size self.filename = config.import_filename self.in_file = gcs.open(self.filename) self.reader = csv.reader(self.in_file) self.imported = 0 self.rejected = 0 if use_shards: LOG.info("Checking student shards") StudentShard.init_shards(config) self.shard_gen = StudentShard.circular_shard_generator() else: self.shard_gen = None
def setup_dummy_students(sharded=None): shard_gen = None if not sharded: pass elif sharded == "circular": setup_dummy_shards() shard_gen = StudentShard.circular_shard_generator() else: # Most other test cases won't care what type of sharding to use, # they can simply pass in True to default to linear. setup_dummy_shards() shard_gen = StudentShard.linear_shard_generator() for i in get_dummy_ids(): instance = get_dummy_student(int(i), shard_gen) instance.put()