def test1(self): output = StringIO.StringIO() with JSONStreamWriter(output, type_=dict) as streamer: streamer.write('a', 1) streamer.write('b', 2) streamer.write('c', 3) correct = { "a": 1, "b": 2, "c": 3 } self.assertEqual(output.getvalue(), json.dumps(correct, sort_keys=True, indent=2, separators=(',', ': '))) output = StringIO.StringIO() with JSONStreamWriter(output, type_=dict) as streamer: streamer.write('a', 1) with streamer.write('data', type_=list) as streamer2: streamer2.write({0: 0, 1: 1, 2: 2}) streamer2.write({3: 3, 4: '4'}) streamer2.write(False) streamer.write('e', 2) correct = { "a": 1, "data": [ { 0: 0, 1: 1, 2: 2 }, { 3: 3, 4: "4" }, False ], "e": 2 } self.assertEqual( output.getvalue(), json.dumps(correct, sort_keys=True, indent=2, separators=(',', ': ')).replace(' [', '\n [') )
def test1(self): output = io.StringIO() with JSONStreamWriter(output, type_=dict) as streamer: streamer.write("a", 1) streamer.write("b", 2) streamer.write("c", 3) correct = {"a": 1, "b": 2, "c": 3} self.assertEqual( output.getvalue(), json.dumps(correct, sort_keys=True, indent=2, separators=(",", ": ")), ) output = io.StringIO() with JSONStreamWriter(output, type_=dict) as streamer: streamer.write("a", 1) with streamer.write("data", type_=list) as streamer2: streamer2.write({0: 0, 1: 1, 2: 2}) streamer2.write({3: 3, 4: "4"}) streamer2.write(False) streamer.write("e", 2) correct = { "a": 1, "data": [{ 0: 0, 1: 1, 2: 2 }, { 3: 3, 4: "4" }, False], "e": 2 } self.assertEqual( output.getvalue(), json.dumps(correct, sort_keys=True, indent=2, separators=(",", ": ")).replace(" [", "\n ["), )
def __init__(self, school, config_path, output_path, output_error_path, break_on_error=True, break_on_warning=False, display_progress_bar=True, skip_duplicates=True, validate=True, tracker=NullTracker()): """Construct ingestor object and resolve options. Args: school (string): The school code (e.g. jhu, gw, umich). config_path (str): Configuration file path. output_path (str): Output path. output_error_path (str): Error output path. break_on_error (bool, optional): Stop ingesting on error. break_on_warning (bool, optional): Stop ingesting on warning. display_progress_bar (bool, optional): display progress bar skip_duplicates (bool, optional): Skip ingesting courses that have already been seen. validate (bool, optional): Perform validation. tracker (library.tracker, optional): tracker object """ self.school = school self.validate = validate self.break_on_error = break_on_error self.break_on_warning = break_on_warning self.skip_duplicates = skip_duplicates self.tracker = tracker self.hoarder = Hoarder() self.tracker.add_viewer(self.hoarder) self.tracker.school = school # Initialize loggers for json and errors. self.json = JSONStreamWriter(output_path, type_=dict).enter() self.data_list = self.json.write('$data', type_=list).enter() self.logger = Logger(errorfile=output_error_path) if self.validate: self.validator = Validator(config_path, tracker=self.tracker) # Inherit dictionary functionality. super(Ingestor, self).__init__()
def __init__(self, output=None): self.json_logger = JSONStreamWriter(output) self.json_logger.enter() self.defaults = Vommit.get_model_defaults() super(Vommit, self).__init__() for name, model in Digestor.MODELS.items(): # if hasattr(self, 'digest_' + name): # continue def closure(name, model): def digest(self, model_params): obj = model.objects.filter( **Vommit.exclude(model_params)).first() self.diff(name, model_params, obj) return obj return digest setattr(self.__class__, 'digest_' + name, closure(name, model))
def __init__(self, output): self.defaults = Vommit.get_model_defaults() self.output = output self.json_streamer = JSONStreamWriter(self.output, type_=list).enter() super(Vommit, self).__init__() def exclude(dct): return {k: v for k, v in list(dct.items()) if k != "defaults"} for name, model in list(Digestor.MODELS.items()): # if hasattr(self, 'digest_' + name): # continue def closure(name, model): def digest(self, model_params): obj = model.objects.filter(**exclude(model_params)).first() self.diff(name, model_params, obj) return obj return digest setattr(self.__class__, "digest_" + name, closure(name, model))