def test_schema_invalid_json():
ModelValidator.BASE_PATH = './tests/res/'
try:
ModelValidator.validate(to_validate={}, field="title", model='bad')
assert False, f"test_model_validator [test_schema_invalid_json] failed"
except BaseException:
assert True
def test_schema_invalid_json():
ModelValidator.BASE_PATH = "./integration_tests/res/"
try:
ModelValidator.validate(to_validate={}, field="title", model="bad")
assert False, "test_model_validator [test_schema_invalid_json] failed"
except BaseException:
assert True
def validate_field_helper(field, value, model, field_type):
# TODO: need to write custom validator for these datetime fields as jsonschema
# validates datetime in format 2018-11-13T20:20:39+00:00, not a format we use
# also not totally necessary as these fields are created server side
if field in ("created_at", "updated_at"):
return value
# remove null characters from varchar and text fields
# Postgres does not support these well and it throws this error if you try to insert
# `Fatal error in main loop A string literal cannot contain NUL (0x00) characters`
# the fix is to replace those characters with empty with empty string
# https://stackoverflow.com/questions/1347646/postgres-error-on-insert-error-invalid-byte-sequence-for-encoding-utf8-0x0
if type(field_type) in (String, Text) and value:
value = value.encode("utf-8", "ignore").decode("utf-8", "ignore")
value = value.replace("\x00", "")
to_validate = {field: value}
try:
ModelValidator.validate(to_validate=to_validate,
model=model,
field=field)
except ValidationError as e:
value = get_default_value(field, value, model, e)
except BaseException as e:
logger.error(f"Validation failed: {e}")
return value
def test_one_field_schema_pass():
track = {"title": "ok"}
try:
ModelValidator.validate(to_validate=track,
field="title",
model="Track")
except ValidationError as e:
assert False, f"test_model_validator [test_one_field_schema_pass] failed: {e}"
def test_schema_missing():
try:
ModelValidator.validate(to_validate={},
field="title",
model="non-existant")
assert False, "test_model_validator [test_one_field_schema_empty_object] failed"
except BaseException:
assert True
def test_one_field_schema_empty_object():
track = {}
try:
ModelValidator.validate(to_validate=track,
field="title",
model="Track")
assert False, "test_model_validator [test_one_field_schema_empty_object] failed"
except BaseException:
assert True
def test_one_field_schema_bad_key():
track = {"wrong": "ok"}
try:
ModelValidator.validate(to_validate=track,
field="title",
model="Track")
assert False, "test_model_validator [test_one_field_schema_bad_key] failed"
except BaseException:
assert True
def test_one_field_schema_bad_value():
track = {"title": 1}
try:
ModelValidator.validate(to_validate=track,
field="title",
model="Track")
assert False, f"test_model_validator [test_one_field_schema_bad_value] failed"
except BaseException:
assert True
def test_one_field_schema_with_additional_properties():
track = {"title": "ok", "wrong": 1}
try:
ModelValidator.validate(to_validate=track,
field="title",
model="Track")
assert False, f"test_model_validator [test_one_field_schema_with_additional_properties] failed"
except BaseException:
assert True
def test_schema_missing_model_key():
ModelValidator.BASE_PATH = './tests/res/'
try:
ModelValidator.validate(to_validate={},
field="title",
model='user_bad')
assert False, f"test_model_validator [test_schema_missing_model_key] failed"
except BaseException:
assert True
class Playlist(Base):
__tablename__ = "playlists"
blockhash = Column(String, ForeignKey("blocks.blockhash"), nullable=True)
blocknumber = Column(Integer, ForeignKey("blocks.number"), nullable=True)
slot = Column(Integer, nullable=True)
txhash = Column(String, default="", nullable=False)
playlist_id = Column(Integer, nullable=False)
playlist_owner_id = Column(Integer, nullable=False)
is_album = Column(Boolean, nullable=False)
is_private = Column(Boolean, nullable=False)
playlist_name = Column(String)
playlist_contents = Column(JSONB, nullable=False)
playlist_image_multihash = Column(String)
playlist_image_sizes_multihash = Column(String)
description = Column(String)
upc = Column(String)
is_current = Column(Boolean, nullable=False)
is_delete = Column(Boolean, nullable=False)
last_added_to = Column(DateTime, nullable=True)
updated_at = Column(DateTime, nullable=False)
created_at = Column(DateTime, nullable=False)
PrimaryKeyConstraint(is_current, playlist_id, playlist_owner_id, txhash)
ModelValidator.init_model_schemas("Playlist")
fields = ["playlist_name", "description"]
# unpacking args into @validates
@validates(*fields)
def validate_field(self, field, value):
return validate_field_helper(field, value, "Playlist",
getattr(Playlist, field).type)
def __repr__(self):
return f"<Playlist(blockhash={self.blockhash},\
def get_default_value(field, value, model, e):
field_props = ModelValidator.get_properties_for_field(model, field)
# type field from the schema. this can either be a string or list
# required by JSONSchema, cannot be None
schema_type_field = field_props['type']
try:
default_value = field_props['default']
except KeyError:
default_value = None
# If the schema indicates this field is equal to object(if string) or contains object(if list) and
# the default value isn't set in the schema, set to SQL null, otherwise JSONB columns get
# set to string 'null'.
# Other fields can be set to their regular defaults or None.
if not default_value:
# if schema_type_field is defined as a list, need to check if 'object' is in list, else check string
if isinstance(schema_type_field,
list) and 'object' in schema_type_field:
default_value = null() # sql null
elif schema_type_field == 'object':
default_value = null() # sql null
logger.warning(f"Validation: Setting the default value {default_value} for field {field} " \
f"of type {schema_type_field} because of error: {e}")
return default_value
class User(Base):
__tablename__ = "users"
blockhash = Column(String, ForeignKey("blocks.blockhash"), nullable=True)
blocknumber = Column(Integer, ForeignKey("blocks.number"), nullable=True)
slot = Column(Integer, nullable=True)
user_storage_account = Column(String, nullable=True)
user_authority_account = Column(String, nullable=True)
txhash = Column(String, default="", nullable=False)
user_id = Column(Integer, nullable=False)
is_current = Column(Boolean, nullable=False)
handle = Column(String)
handle_lc = Column(String, index=True)
wallet = Column(String, index=True)
is_creator = Column(Boolean, nullable=False, default=False)
is_verified = Column(Boolean,
nullable=False,
default=False,
server_default="false")
name = Column(Text)
profile_picture = Column(String)
profile_picture_sizes = Column(String)
cover_photo = Column(String)
cover_photo_sizes = Column(String)
bio = Column(String)
location = Column(String)
metadata_multihash = Column(String)
creator_node_endpoint = Column(String)
updated_at = Column(DateTime, nullable=False)
created_at = Column(DateTime, nullable=False)
primary_id = Column(Integer, nullable=True)
secondary_ids = Column(postgresql.ARRAY(Integer), nullable=True)
replica_set_update_signer = Column(String, nullable=True)
has_collectibles = Column(Boolean,
nullable=False,
default=False,
server_default="false")
playlist_library = Column(JSONB, nullable=True)
is_deactivated = Column(Boolean,
nullable=False,
default=False,
server_default="false",
index=True)
# NOTE: There is no actualy primary key in the DB
PrimaryKeyConstraint(is_current, user_id, txhash)
ModelValidator.init_model_schemas("User")
fields = get_fields_to_validate("User")
# unpacking args into @validates
@validates(*fields)
def validate_field(self, field, value):
return validate_field_helper(field, value, "User",
getattr(User, field).type)
def __repr__(self):
return f"<User(blockhash={self.blockhash},\
def validate_field_helper(field, value, model):
# TODO: need to write custom validator for these datetime fields as jsonschema
# validates datetime in format 2018-11-13T20:20:39+00:00, not a format we use
# also not totally necessary as these fields are created server side
if field in ('created_at', 'updated_at'):
return value
to_validate = {field: value}
try:
ModelValidator.validate(to_validate=to_validate,
model=model,
field=field)
except ValidationError as e:
value = get_default_value(field, value, model, e)
except BaseException as e:
logger.error(f"Validation failed: {e}")
return value
class User(Base):
__tablename__ = "users"
blockhash = Column(String, ForeignKey("blocks.blockhash"), nullable=False)
blocknumber = Column(Integer, ForeignKey("blocks.number"), nullable=False)
txhash = Column(String, default='', nullable=False)
user_id = Column(Integer, nullable=False)
is_current = Column(Boolean, nullable=False)
handle = Column(String)
handle_lc = Column(String, index=True)
wallet = Column(String, index=True)
is_creator = Column(Boolean, nullable=False, default=False)
is_verified = Column(Boolean,
nullable=False,
default=False,
server_default='false')
name = Column(Text)
profile_picture = Column(String)
profile_picture_sizes = Column(String)
cover_photo = Column(String)
cover_photo_sizes = Column(String)
bio = Column(String)
location = Column(String)
metadata_multihash = Column(String)
creator_node_endpoint = Column(String)
updated_at = Column(DateTime, nullable=False)
created_at = Column(DateTime, nullable=False)
primary_id = Column(Integer, nullable=True)
secondary_ids = Column(postgresql.ARRAY(Integer), nullable=True)
replica_set_update_signer = Column(String, nullable=True)
has_collectibles = Column(Boolean,
nullable=False,
default=False,
server_default='false')
PrimaryKeyConstraint(is_current, user_id, blockhash, txhash)
ModelValidator.init_model_schemas('User')
fields = get_fields_to_validate('User')
# unpacking args into @validates
@validates(*fields)
def validate_field(self, field, value):
return validate_field_helper(field, value, 'User',
getattr(User, field).type)
def __repr__(self):
return f"<User(blockhash={self.blockhash},\
class Track(Base):
__tablename__ = "tracks"
blockhash = Column(String, ForeignKey("blocks.blockhash"), nullable=False)
blocknumber = Column(Integer, ForeignKey("blocks.number"), nullable=False)
track_id = Column(Integer, nullable=False)
is_current = Column(Boolean, nullable=False)
is_delete = Column(Boolean, nullable=False)
owner_id = Column(Integer, nullable=False)
route_id = Column(String, nullable=False)
title = Column(Text, nullable=True)
length = Column(Integer, nullable=True)
cover_art = Column(String, nullable=True)
cover_art_sizes = Column(String, nullable=True)
tags = Column(String, nullable=True)
genre = Column(String, nullable=True)
mood = Column(String, nullable=True)
credits_splits = Column(String, nullable=True)
remix_of = Column(postgresql.JSONB, nullable=True)
create_date = Column(String, nullable=True)
release_date = Column(String, nullable=True)
file_type = Column(String, nullable=True)
description = Column(String, nullable=True)
license = Column(String, nullable=True)
isrc = Column(String, nullable=True)
iswc = Column(String, nullable=True)
track_segments = Column(postgresql.JSONB, nullable=False)
metadata_multihash = Column(String, nullable=True)
download = Column(postgresql.JSONB, nullable=True)
updated_at = Column(DateTime, nullable=False)
created_at = Column(DateTime, nullable=False)
is_unlisted = Column(Boolean, nullable=False)
field_visibility = Column(postgresql.JSONB, nullable=True)
stem_of = Column(postgresql.JSONB, nullable=True)
# Primary key has to be combo of all 3 is_current/creator_id/blockhash
PrimaryKeyConstraint(is_current, track_id, blockhash)
ModelValidator.init_model_schemas('Track')
fields = get_fields_to_validate('Track')
# unpacking args into @validates
@validates(*fields)
def validate_field(self, field, value):
return validate_field_helper(field, value, 'Track')
def __repr__(self):
return (f"<Track("
f"blockhash={self.blockhash},"
f"blocknumber={self.blocknumber},"
f"track_id={self.track_id},"
f"is_current={self.is_current},"
f"is_delete={self.is_delete},"
f"owner_id={self.owner_id},"
f"route_id={self.route_id},"
f"title={self.title},"
f"length={self.length},"
f"cover_art={self.cover_art},"
f"cover_art_sizes={self.cover_art_sizes},"
f"tags={self.tags},"
f"genre={self.genre},"
f"mood={self.mood},"
f"credits_splits={self.credits_splits},"
f"remix_of={self.remix_of},"
f"create_date={self.create_date},"
f"release_date={self.release_date},"
f"file_type={self.file_type},"
f"description={self.description},"
f"license={self.license},"
f"isrc={self.isrc},"
f"iswc={self.iswc},"
f"track_segments={self.track_segments},"
f"metadata_multihash={self.metadata_multihash},"
f"download={self.download},"
f"updated_at={self.updated_at},"
f"created_at={self.created_at},"
f"stem_of={self.stem_of}"
")>")
class Track(Base):
__tablename__ = "tracks"
blockhash = Column(String, ForeignKey("blocks.blockhash"), nullable=True)
blocknumber = Column(Integer, ForeignKey("blocks.number"), nullable=True)
slot = Column(Integer, nullable=True)
txhash = Column(String, default="", nullable=False)
track_id = Column(Integer, nullable=False)
is_current = Column(Boolean, nullable=False)
is_delete = Column(Boolean, nullable=False)
owner_id = Column(Integer, nullable=False)
route_id = Column(String, nullable=False)
title = Column(Text, nullable=True)
length = Column(Integer, nullable=True)
cover_art = Column(String, nullable=True)
cover_art_sizes = Column(String, nullable=True)
tags = Column(String, nullable=True)
genre = Column(String, nullable=True)
mood = Column(String, nullable=True)
credits_splits = Column(String, nullable=True)
remix_of = Column(postgresql.JSONB, nullable=True)
create_date = Column(String, nullable=True)
release_date = Column(String, nullable=True)
file_type = Column(String, nullable=True)
description = Column(String, nullable=True)
license = Column(String, nullable=True)
isrc = Column(String, nullable=True)
iswc = Column(String, nullable=True)
track_segments = Column(postgresql.JSONB, nullable=False)
metadata_multihash = Column(String, nullable=True)
download = Column(postgresql.JSONB, nullable=True)
updated_at = Column(DateTime, nullable=False)
created_at = Column(DateTime, nullable=False)
is_unlisted = Column(Boolean, nullable=False)
field_visibility = Column(postgresql.JSONB, nullable=True)
stem_of = Column(postgresql.JSONB, nullable=True)
is_available = Column(Boolean, default=True, nullable=False)
_routes = relationship( # type: ignore
"TrackRoute",
primaryjoin="and_(\
remote(Track.track_id) == foreign(TrackRoute.track_id),\
TrackRoute.is_current)",
lazy="joined",
viewonly=True,
)
user = relationship(
"User",
primaryjoin="and_(\
remote(Track.owner_id) == foreign(User.user_id),\
User.is_current)",
lazy="joined",
viewonly=True,
)
@property
def _slug(self):
return self._routes[0].slug if self._routes else ""
@property
def permalink(self):
if self.user and self.user[0].handle and self._slug:
return f"/{self.user[0].handle}/{self._slug}"
return ""
PrimaryKeyConstraint(is_current, track_id, txhash)
ModelValidator.init_model_schemas("Track")
fields = get_fields_to_validate("Track")
# unpacking args into @validates
@validates(*fields)
def validate_field(self, field, value):
return validate_field_helper(field, value, "Track",
getattr(Track, field).type)
def __repr__(self):
return (f"<Track("
f"blockhash={self.blockhash},"
f"blocknumber={self.blocknumber},"
f"slot={self.slot},"
f"txhash={self.txhash},"
f"track_id={self.track_id},"
f"is_current={self.is_current},"
f"is_delete={self.is_delete},"
f"owner_id={self.owner_id},"
f"route_id={self.route_id},"
f"title={self.title},"
f"length={self.length},"
f"cover_art={self.cover_art},"
f"cover_art_sizes={self.cover_art_sizes},"
f"tags={self.tags},"
f"genre={self.genre},"
f"mood={self.mood},"
f"credits_splits={self.credits_splits},"
f"remix_of={self.remix_of},"
f"create_date={self.create_date},"
f"release_date={self.release_date},"
f"file_type={self.file_type},"
f"description={self.description},"
f"license={self.license},"
f"isrc={self.isrc},"
f"iswc={self.iswc},"
f"track_segments={self.track_segments},"
f"metadata_multihash={self.metadata_multihash},"
f"download={self.download},"
f"updated_at={self.updated_at},"
f"created_at={self.created_at},"
f"stem_of={self.stem_of},"
f"permalink={self.permalink},"
f"user={self.user}"
f"is_available={self.is_available}"
")>")