def test_object(self):
data = {
"help": 5,
"xxx": datetime.utcnow(),
"bla": None,
"foo": set(["a"]),
"yum": b"xxx",
}
json = dump_json(data)
out = load_json(json)
assert "bla" not in out, out
assert "xxx" in out, out
assert "a" in out["foo"], out
assert load_json("") is None
assert load_json(None) is None
assert dump_json(None) == ""
def test_object(self):
data = {
'help': 5,
'xxx': datetime.utcnow(),
'bla': None,
'foo': set(['a']),
'yum': b'xxx'
}
json = dump_json(data)
out = load_json(json)
assert 'bla' not in out, out
assert 'xxx' in out, out
assert 'a' in out['foo'], out
assert load_json('') is None
assert load_json(None) is None
assert dump_json(None) == ''
def save(cls, crawler, stage, level, run_id, error=None, message=None):
"""Create an event, possibly based on an exception."""
event = {
'stage': stage.name,
'level': level,
'timestamp': pack_now(),
'error': error,
'message': message
}
data = dump_json(event)
keys = [
make_key(crawler, "events"),
make_key(crawler, "events", level),
make_key(crawler, "events", stage),
make_key(crawler, "events", stage, level),
make_key(crawler, "events", run_id),
make_key(crawler, "events", run_id, level),
]
for key in keys:
conn.lpush(key, data)
conn.expire(key, REDIS_EXPIRE)
# Persist the counts for longer
count_keys = [
make_key(crawler, "events", "count", level),
make_key(crawler, "events", "count", stage, level),
make_key(crawler, "events", "count", run_id, level),
]
for key in count_keys:
conn.incr(key)
conn.expire(key, REDIS_LONG)
return event
def test_reporting_from_task(self):
task_data = {
"dataset": self.dataset,
"job": self.job.id,
"stage": self.job.get_stage(self.operation).stage,
"payload": {
"entities": ["a", "b", "c"]
},
}
task = Task.unpack(self.conn, dump_json(task_data))
serialized = task.serialize()
reporter = Reporter(task=task)
reporter.start()
task = self._get_task()
self.assertEqual(task.stage.stage, OP_REPORT)
data = task.payload
values = {
"status": Status.START,
"has_error": False,
"operation": self.operation,
"dataset": self.dataset,
"updated_at": None,
"start_at": None,
"job": self.job.id,
"task": serialized,
}
for key, value in values.items():
self.assertIn(key, data)
if value is not None:
self.assertEqual(value, data[key])
def serialize(self):
return dump_json({
'context': self.context or {},
'payload': self.payload,
'dataset': self.job.dataset.name,
'job': self.job.id,
'stage': self.stage.stage
})
def serialize(self):
return dump_json({
"context": self.context or {},
"payload": self.payload,
"dataset": self.job.dataset.name,
"job": self.job.id,
"stage": self.stage.stage,
})
def save(cls, crawler, stage, level, run_id, error=None, message=None):
"""Create an event, possibly based on an exception."""
event = {
'stage': stage.name,
'level': level,
'timestamp': pack_now(),
'error': error,
'message': message
}
data = dump_json(event)
conn.lpush(make_key(crawler, "events"), data)
conn.lpush(make_key(crawler, "events", level), data)
conn.lpush(make_key(crawler, "events", stage), data)
conn.lpush(make_key(crawler, "events", stage, level), data)
conn.lpush(make_key(crawler, "events", run_id), data)
conn.lpush(make_key(crawler, "events", run_id, level), data)
return event
def set_tag(self, key, value):
data = dump_json(value)
key = make_key(self.crawler, "tag", key)
return tags.set(key, data)
def set_tag(self, key, value):
data = dump_json(value)
key = make_key(self.crawler, "tag", key)
return conn.set(key, data, ex=self.crawler.expire)