def parse_and_store(db, incoming_data, bucket_name, logger):
incoming_records = records.parse_all(incoming_data)
logger.info("request contains %s documents" % len(incoming_records))
bucket = Bucket(db, bucket_name)
bucket.store(incoming_records)
def _write_to_bucket(bucket_config):
if bucket_config is None:
return jsonify(status="error",
message='Could not find bucket_config'), 404
g.bucket_name = bucket_config.name
try:
auth_header = request.headers['Authorization']
except KeyError:
return jsonify(status='error',
message='Authorization header missing.'), 403
if not bearer_token_is_valid(bucket_config, auth_header):
statsd.incr("write_api.bad_token", bucket=g.bucket_name)
return jsonify(status='error', message='Forbidden'), 403
try:
data = listify_json(request.json)
bucket = Bucket(db, bucket_config)
bucket.parse_and_store(data)
return jsonify(status='ok')
except (ParseError, ValidationError) as e:
return jsonify(status="error", message=str(e)), 400
def parse_and_store(db, incoming_data, bucket_name, logger):
incoming_records = records.parse_all(incoming_data)
logger.info(
"request contains %s documents" % len(incoming_records))
bucket = Bucket(db, bucket_name)
bucket.store(incoming_records)
def test_validation_error_if_auto_id_property_is_missing(self):
objects = [{
"postcode": "WC2B 6SE",
"name": "Aviation House"
}]
auto_id = ("postcode", "number")
bucket = Bucket(self.mock_database, "bucket", generate_id_from=auto_id)
bucket.parse_and_store(objects)
def test_validation_error_if_auto_id_property_is_missing(self):
objects = [{
"postcode": "WC2B 6SE",
"name": "Aviation House"
}]
config = BucketConfig("bucket", data_group="group", data_type="type", auto_ids=("postcode", "number"))
bucket = Bucket(self.mock_database, config)
bucket.parse_and_store(objects)
def test_no_id_generated_if_auto_id_is_none(self):
object = {
"postcode": "WC2B 6SE",
"number": "125",
"name": "Aviation House"
}
bucket = Bucket(self.mock_database, "bucket", generate_id_from=None)
bucket.parse_and_store([object])
self.mock_repository.save.assert_called_once_with(object)
def test_no_id_generated_if_auto_id_is_none(self):
object = {
"postcode": "WC2B 6SE",
"number": "125",
"name": "Aviation House"
}
config = BucketConfig("bucket", data_group="group", data_type="type")
bucket = Bucket(self.mock_database, config)
bucket.parse_and_store([object])
self.mock_repository.save.assert_called_once_with(object)
def test_auto_id_can_be_generated_from_a_timestamp(self):
objects = [{
"_timestamp": "2013-08-01T00:00:00+00:00",
"foo": "bar"
}]
auto_id = ["_timestamp", "foo"]
bucket = Bucket(self.mock_database, "bucket", generate_id_from=auto_id)
bucket.parse_and_store(objects)
saved_object = self.mock_repository.save.call_args[0][0]
assert_that(b64decode(saved_object['_id']),
is_("2013-08-01T00:00:00+00:00.bar"))
def test_auto_id_for_a_single_field(self):
objects = [{
"abc": "def"
}]
auto_id = ["abc"]
bucket = Bucket(self.mock_database, "bucket", generate_id_from=auto_id)
bucket.parse_and_store(objects)
self.mock_repository.save.assert_called_once_with({
"_id": b64encode("def"),
"abc": "def"
})
def test_auto_id_can_be_generated_from_a_timestamp(self):
objects = [{
"_timestamp": "2013-08-01T00:00:00+00:00",
"foo": "bar"
}]
config = BucketConfig("bucket", data_group="group", data_type="type", auto_ids=["_timestamp", "foo"])
bucket = Bucket(self.mock_database, config)
bucket.parse_and_store(objects)
saved_object = self.mock_repository.save.call_args[0][0]
assert_that(b64decode(saved_object['_id']),
is_("2013-08-01T00:00:00+00:00.bar"))
def test_auto_id_for_a_single_field(self):
objects = [{
"abc": "def"
}]
config = BucketConfig("bucket", data_group="group", data_type="type", auto_ids=["abc"])
bucket = Bucket(self.mock_database, config)
bucket.parse_and_store(objects)
self.mock_repository.save.assert_called_once_with({
"_id": b64encode("def"),
"abc": "def"
})
def test_auto_id_generation(self):
objects = [{
"postcode": "WC2B 6SE",
"number": "125",
"name": "Aviation House"
}]
auto_id = ("postcode", "number")
bucket = Bucket(self.mock_database, "bucket", generate_id_from=auto_id)
bucket.parse_and_store(objects)
self.mock_repository.save.assert_called_once_with({
"_id": b64encode("WC2B 6SE.125"),
"postcode": "WC2B 6SE",
"number": "125",
"name": "Aviation House"
})
def post_to_bucket(bucket_name):
g.bucket_name = bucket_name
tokens = app.config['TOKENS']
auth_header = request.headers.get('Authorization', None)
if not bearer_token_is_valid(tokens, auth_header, bucket_name):
statsd.incr("write_api.bad_token", bucket=g.bucket_name)
return jsonify(status='error', message='Forbidden'), 403
try:
data = load_json(request.json)
bucket = Bucket(db, bucket_name)
bucket.parse_and_store(data)
return jsonify(status='ok')
except (ParseError, ValidationError) as e:
return jsonify(status="error", message=str(e)), 400
def _store_data(bucket_name, parser):
file_stream = request.files["file"].stream
if not request.files["file"].filename:
return _invalid_upload("file is required")
try:
if request.content_length > MAX_UPLOAD_SIZE:
return _invalid_upload("file too large")
try:
data = parser(file_stream)
auto_id_keys = _auto_id_keys_for(bucket_name)
bucket = Bucket(db, bucket_name, generate_id_from=auto_id_keys)
bucket.parse_and_store(data)
return render_template("upload_ok.html")
except (ParseError, ValidationError) as e:
return _invalid_upload(e.message)
finally:
file_stream.close()
def _store_data(bucket_name, parser):
file_stream = request.files["file"].stream
if not request.files["file"].filename:
return _invalid_upload("file is required")
try:
if request.content_length > MAX_UPLOAD_SIZE:
return _invalid_upload("file too large")
try:
data = parser(file_stream)
auto_id_keys = _auto_id_keys_for(bucket_name)
bucket = Bucket(db, bucket_name, generate_id_from=auto_id_keys)
bucket.parse_and_store(data)
return render_template("upload_ok.html")
except (ParseError, ValidationError) as e:
return _invalid_upload(e.message)
finally:
file_stream.close()
def post_to_bucket(bucket_name):
g.bucket_name = bucket_name
tokens = app.config['TOKENS']
auth_header = request.headers.get('Authorization', None)
if not bearer_token_is_valid(tokens, auth_header, bucket_name):
statsd.incr("write_api.bad_token", bucket=g.bucket_name)
return jsonify(status='error', message='Forbidden'), 403
try:
data = load_json(request.json)
bucket = Bucket(db, bucket_name)
bucket.parse_and_store(data)
return jsonify(status='ok')
except (ParseError, ValidationError) as e:
return jsonify(status="error", message=str(e)), 400
def test_auto_id_generation(self):
objects = [{
"postcode": "WC2B 6SE",
"number": "125",
"name": "Aviation House"
}]
config = BucketConfig("bucket", data_group="group", data_type="type", auto_ids=("postcode", "number"))
bucket = Bucket(self.mock_database, config)
bucket.parse_and_store(objects)
self.mock_repository.save.assert_called_once_with({
"_id": b64encode("WC2B 6SE.125"),
"postcode": "WC2B 6SE",
"number": "125",
"name": "Aviation House"
})