def test1(self):
output = StringIO.StringIO()
with JSONStreamWriter(output, type_=dict) as streamer:
streamer.write('a', 1)
streamer.write('b', 2)
streamer.write('c', 3)
correct = {
"a": 1,
"b": 2,
"c": 3
}
self.assertEqual(output.getvalue(),
json.dumps(correct,
sort_keys=True,
indent=2,
separators=(',', ': ')))
output = StringIO.StringIO()
with JSONStreamWriter(output, type_=dict) as streamer:
streamer.write('a', 1)
with streamer.write('data', type_=list) as streamer2:
streamer2.write({0: 0, 1: 1, 2: 2})
streamer2.write({3: 3, 4: '4'})
streamer2.write(False)
streamer.write('e', 2)
correct = {
"a": 1,
"data":
[
{
0: 0,
1: 1,
2: 2
},
{
3: 3,
4: "4"
},
False
],
"e": 2
}
self.assertEqual(
output.getvalue(),
json.dumps(correct,
sort_keys=True,
indent=2,
separators=(',', ': ')).replace(' [', '\n [')
)
def test1(self):
output = io.StringIO()
with JSONStreamWriter(output, type_=dict) as streamer:
streamer.write("a", 1)
streamer.write("b", 2)
streamer.write("c", 3)
correct = {"a": 1, "b": 2, "c": 3}
self.assertEqual(
output.getvalue(),
json.dumps(correct,
sort_keys=True,
indent=2,
separators=(",", ": ")),
)
output = io.StringIO()
with JSONStreamWriter(output, type_=dict) as streamer:
streamer.write("a", 1)
with streamer.write("data", type_=list) as streamer2:
streamer2.write({0: 0, 1: 1, 2: 2})
streamer2.write({3: 3, 4: "4"})
streamer2.write(False)
streamer.write("e", 2)
correct = {
"a": 1,
"data": [{
0: 0,
1: 1,
2: 2
}, {
3: 3,
4: "4"
}, False],
"e": 2
}
self.assertEqual(
output.getvalue(),
json.dumps(correct,
sort_keys=True,
indent=2,
separators=(",", ": ")).replace(" [", "\n ["),
)
def __init__(self,
school,
config_path,
output_path,
output_error_path,
break_on_error=True,
break_on_warning=False,
display_progress_bar=True,
skip_duplicates=True,
validate=True,
tracker=NullTracker()):
"""Construct ingestor object and resolve options.
Args:
school (string): The school code (e.g. jhu, gw, umich).
config_path (str): Configuration file path.
output_path (str): Output path.
output_error_path (str): Error output path.
break_on_error (bool, optional): Stop ingesting on error.
break_on_warning (bool, optional): Stop ingesting on warning.
display_progress_bar (bool, optional): display progress bar
skip_duplicates (bool, optional): Skip ingesting courses
that have already been seen.
validate (bool, optional): Perform validation.
tracker (library.tracker, optional): tracker object
"""
self.school = school
self.validate = validate
self.break_on_error = break_on_error
self.break_on_warning = break_on_warning
self.skip_duplicates = skip_duplicates
self.tracker = tracker
self.hoarder = Hoarder()
self.tracker.add_viewer(self.hoarder)
self.tracker.school = school
# Initialize loggers for json and errors.
self.json = JSONStreamWriter(output_path, type_=dict).enter()
self.data_list = self.json.write('$data', type_=list).enter()
self.logger = Logger(errorfile=output_error_path)
if self.validate:
self.validator = Validator(config_path, tracker=self.tracker)
# Inherit dictionary functionality.
super(Ingestor, self).__init__()
def __init__(self, output=None):
self.json_logger = JSONStreamWriter(output)
self.json_logger.enter()
self.defaults = Vommit.get_model_defaults()
super(Vommit, self).__init__()
for name, model in Digestor.MODELS.items():
# if hasattr(self, 'digest_' + name):
# continue
def closure(name, model):
def digest(self, model_params):
obj = model.objects.filter(
**Vommit.exclude(model_params)).first()
self.diff(name, model_params, obj)
return obj
return digest
setattr(self.__class__, 'digest_' + name, closure(name, model))
def __init__(self, output):
self.defaults = Vommit.get_model_defaults()
self.output = output
self.json_streamer = JSONStreamWriter(self.output, type_=list).enter()
super(Vommit, self).__init__()
def exclude(dct):
return {k: v for k, v in list(dct.items()) if k != "defaults"}
for name, model in list(Digestor.MODELS.items()):
# if hasattr(self, 'digest_' + name):
# continue
def closure(name, model):
def digest(self, model_params):
obj = model.objects.filter(**exclude(model_params)).first()
self.diff(name, model_params, obj)
return obj
return digest
setattr(self.__class__, "digest_" + name, closure(name, model))