class HotelBookingWithHotelSchema(HotelBookingSchema):
hotel_booking = Nested(nested=HotelBookingWithNestedHotelSchema)
class EquipmentTypeSchema(mm.ModelSchema):
features = Nested(RoomFeatureSchema, many=True)
class Meta:
model = EquipmentType
fields = ('id', 'name', 'features')
class DocumentCreateSchema(DocumentUpdateSchema):
parent = Nested(DocumentParentSchema())
class PetFamilySchema(Schema):
pets_1 = Nested(PetSchema, many=True)
pets_2 = List(Nested(PetSchema))
class SchemaWithList(Schema):
list_field = List(Nested(PetSchema))
class MatchSchema(Schema, DatedSchema):
entity = Nested(EntitySchema, required=True)
match = Nested(EntitySchema, required=True)
score = Float(dump_only=True)
class MultiModifierSchema(Schema):
pet_unmodified = Nested(PetSchema)
pet_exclude = Nested(PetSchema, exclude=("name", ))
class FeatureExtractionOutput(Schema):
cell_features = Nested(CellFeatures)
sweep_features = Dict(keys=Str(), values=Nested(SweepFeatures))
cell_record = Nested(CellRecord)
sweep_records = List(Nested(SweepRecord))
cell_state = Nested(CellState)
class CommentSchema(ma.ModelSchema):
class Meta:
model = CommentModel
user = Nested(UserSchema, only=["username"])
class RampFeatures(Schema):
spiking_sweeps = Nested(SweepFeatures, many=True)
mean_spike_0 = Nested(FirstSpikeMeanFeatures)
class CellFeatures(Schema):
long_squares = Nested(LongSquareFeatures)
short_squares = Nested(ShortSquareFeatures)
ramps = Nested(RampFeatures)
class ShortSquareFeatures(Schema):
stimulus_amplitude = Float()
common_amp_sweeps = Nested(SweepFeatures, many=True)
mean_spike_0 = Nested(FirstSpikeMeanFeatures)
class SonificationJob(Job):
type = 'sonification'
description = 'Image Sonification'
file_id: int = Integer()
settings: SonificationSettings = Nested(SonificationSettings, default={})
format: str = String(default='wav')
def run(self):
settings = self.settings
pixels = get_subframe(self.user_id,
self.file_id,
x0=settings.x,
y0=settings.y,
w=settings.width,
h=settings.height)
x0 = settings.x - 1
y0 = settings.y - 1
df = get_data_file(self.user_id, self.file_id)
height, width = pixels.shape
if width != df.width or height != df.height:
# Sonifying a subimage; estimate background from the whole image
# first, then supply a cutout of background and RMS
# to sonify_image()
full_img = get_data_file_data(self.user_id, self.file_id)[0]
bkg, rms = estimate_background(full_img, size=settings.bkg_scale)
bkg = bkg[y0:y0 + height, x0:x0 + width]
rms = rms[y0:y0 + height, x0:x0 + width]
else:
# When sonifying the whole image, sonify_image() will estimate
# background automatically
bkg = rms = None
data = BytesIO()
sonify_image(
pixels,
data,
coord=settings.coord,
barycenter=settings.barycenter,
tempo=settings.tempo,
sampling_rate=settings.sampling_rate,
start_tone=settings.start_tone,
num_tones=settings.num_tones,
volume=settings.volume,
noise_volume=settings.noise_volume,
bkg_scale=settings.bkg_scale,
threshold=settings.threshold,
min_connected=settings.min_connected,
hi_clip=settings.hi_clip,
noise_lo=settings.noise_lo,
noise_hi=settings.noise_hi,
bkg=bkg,
rms=rms,
index_sounds=settings.index_sounds,
)
data = data.getvalue()
if self.format == 'mp3':
# Use ffmpeg to convert wav to mp3
temp_dir = tempfile.mkdtemp(prefix='ffmpeg-')
try:
wav_file = os.path.join(temp_dir, 'in.wav')
mp3_file = os.path.join(temp_dir, 'out.mp3')
with open(wav_file, 'wb') as f:
f.write(data)
subprocess.check_call(['ffmpeg', '-i', wav_file, mp3_file])
with open(mp3_file, 'rb') as f:
data = f.read()
finally:
shutil.rmtree(temp_dir)
mimetype = 'audio/x-mpeg-3'
else:
mimetype = 'audio/x-wav'
self.create_job_file('sonification', data, mimetype=mimetype)
class ProjectSchema(ma.ModelSchema):
class Meta:
model = ProjectModel
owner = Nested(UserSchema, only=["username"])
authorized = Nested(UserSchema, only=["username"], many=True)
class UserSchema(ma.ModelSchema):
class Meta:
model = Users
include_fk = True
dump_only = ("id", "oauth_id", "created")
load_only = ("password", )
name = field_for(
Users,
"name",
required=True,
allow_none=False,
validate=[
validate.Length(min=1,
max=128,
error="User names must not be empty")
],
)
email = field_for(
Users,
"email",
allow_none=False,
validate=[
validate.Email(
"Emails must be a properly formatted email address"),
validate.Length(min=1, max=128, error="Emails must not be empty"),
],
)
website = field_for(
Users,
"website",
validate=[
# This is a dirty hack to let website accept empty strings so you can remove your website
lambda website: validate.URL(
error=
"Websites must be a proper URL starting with http or https",
schemes={"http", "https"},
)(website) if website else True
],
)
country = field_for(Users, "country", validate=[validate_country_code])
password = field_for(Users, "password")
fields = Nested(UserFieldEntriesSchema,
partial=True,
many=True,
attribute="field_entries")
@pre_load
def validate_name(self, data):
name = data.get("name")
if name is None:
return
name = name.strip()
existing_user = Users.query.filter_by(name=name).first()
current_user = get_current_user()
if is_admin():
user_id = data.get("id")
if user_id:
if existing_user and existing_user.id != user_id:
raise ValidationError("User name has already been taken",
field_names=["name"])
else:
if existing_user:
if current_user:
if current_user.id != existing_user.id:
raise ValidationError(
"User name has already been taken",
field_names=["name"])
else:
raise ValidationError(
"User name has already been taken",
field_names=["name"])
else:
if name == current_user.name:
return data
else:
name_changes = get_config("name_changes", default=True)
if bool(name_changes) is False:
raise ValidationError("Name changes are disabled",
field_names=["name"])
if existing_user:
raise ValidationError("User name has already been taken",
field_names=["name"])
@pre_load
def validate_email(self, data):
email = data.get("email")
if email is None:
return
email = email.strip()
existing_user = Users.query.filter_by(email=email).first()
current_user = get_current_user()
if is_admin():
user_id = data.get("id")
if user_id:
if existing_user and existing_user.id != user_id:
raise ValidationError(
"Email address has already been used",
field_names=["email"])
else:
if existing_user:
if current_user:
if current_user.id != existing_user.id:
raise ValidationError(
"Email address has already been used",
field_names=["email"],
)
else:
raise ValidationError(
"Email address has already been used",
field_names=["email"])
else:
if email == current_user.email:
return data
else:
confirm = data.get("confirm")
if bool(confirm) is False:
raise ValidationError(
"Please confirm your current password",
field_names=["confirm"])
test = verify_password(plaintext=confirm,
ciphertext=current_user.password)
if test is False:
raise ValidationError(
"Your previous password is incorrect",
field_names=["confirm"])
if existing_user:
raise ValidationError(
"Email address has already been used",
field_names=["email"])
if check_email_is_whitelisted(email) is False:
raise ValidationError(
"Only email addresses under {domains} may register".
format(domains=get_config("domain_whitelist")),
field_names=["email"],
)
if get_config("verify_emails"):
current_user.verified = False
@pre_load
def validate_password_confirmation(self, data):
password = data.get("password")
confirm = data.get("confirm")
target_user = get_current_user()
if is_admin():
pass
else:
if password and (bool(confirm) is False):
raise ValidationError("Please confirm your current password",
field_names=["confirm"])
if password and confirm:
test = verify_password(plaintext=confirm,
ciphertext=target_user.password)
if test is True:
return data
else:
raise ValidationError(
"Your previous password is incorrect",
field_names=["confirm"])
else:
data.pop("password", None)
data.pop("confirm", None)
@pre_load
def validate_fields(self, data):
"""
This validator is used to only allow users to update the field entry for their user.
It's not possible to exclude it because without the PK Marshmallow cannot load the right instance
"""
fields = data.get("fields")
if fields is None:
return
current_user = get_current_user()
if is_admin():
user_id = data.get("id")
if user_id:
target_user = Users.query.filter_by(id=data["id"]).first()
else:
target_user = current_user
# We are editting an existing user
if self.view == "admin" and self.instance:
target_user = self.instance
provided_ids = []
for f in fields:
f.pop("id", None)
field_id = f.get("field_id")
# # Check that we have an existing field for this. May be unnecessary b/c the foriegn key should enforce
field = UserFields.query.filter_by(
id=field_id).first_or_404()
# Get the existing field entry if one exists
entry = UserFieldEntries.query.filter_by(
field_id=field.id, user_id=target_user.id).first()
if entry:
f["id"] = entry.id
provided_ids.append(entry.id)
# Extremely dirty hack to prevent deleting previously provided data.
# This needs a better soln.
entries = (UserFieldEntries.query.options(
load_only("id")).filter_by(user_id=target_user.id).all())
for entry in entries:
if entry.id not in provided_ids:
fields.append({"id": entry.id})
else:
provided_ids = []
for f in fields:
# Remove any existing set
f.pop("id", None)
field_id = f.get("field_id")
value = f.get("value")
# # Check that we have an existing field for this. May be unnecessary b/c the foriegn key should enforce
field = UserFields.query.filter_by(id=field_id).first_or_404()
if field.required is True and value.strip() == "":
raise ValidationError(f"Field '{field.name}' is required",
field_names=["fields"])
if field.editable is False:
raise ValidationError(
f"Field '{field.name}' cannot be editted",
field_names=["fields"],
)
# Get the existing field entry if one exists
entry = UserFieldEntries.query.filter_by(
field_id=field.id, user_id=current_user.id).first()
if entry:
f["id"] = entry.id
provided_ids.append(entry.id)
# Extremely dirty hack to prevent deleting previously provided data.
# This needs a better soln.
entries = (UserFieldEntries.query.options(
load_only("id")).filter_by(user_id=current_user.id).all())
for entry in entries:
if entry.id not in provided_ids:
fields.append({"id": entry.id})
@post_dump
def process_fields(self, data):
"""
Handle permissions levels for fields.
This is post_dump to manipulate JSON instead of the raw db object
Admins can see all fields.
Users (self) can see their edittable and public fields
Public (user) can only see public fields
"""
# Gather all possible fields
removed_field_ids = []
fields = UserFields.query.all()
# Select fields for removal based on current view and properties of the field
for field in fields:
if self.view == "user":
if field.public is False:
removed_field_ids.append(field.id)
elif self.view == "self":
if field.editable is False and field.public is False:
removed_field_ids.append(field.id)
# Rebuild fuilds
fields = data.get("fields")
if fields:
data["fields"] = [
field for field in fields
if field["field_id"] not in removed_field_ids
]
views = {
"user": [
"website",
"name",
"country",
"affiliation",
"bracket",
"id",
"oauth_id",
"fields",
],
"self": [
"website",
"name",
"email",
"country",
"affiliation",
"bracket",
"id",
"oauth_id",
"password",
"fields",
],
"admin": [
"website",
"name",
"created",
"country",
"banned",
"email",
"affiliation",
"secret",
"bracket",
"hidden",
"id",
"oauth_id",
"password",
"type",
"verified",
"fields",
],
}
def __init__(self, view=None, *args, **kwargs):
if view:
if isinstance(view, string_types):
kwargs["only"] = self.views[view]
elif isinstance(view, list):
kwargs["only"] = view
self.view = view
super(UserSchema, self).__init__(*args, **kwargs)
class ConvertResponseSchema(Schema):
data = Nested(ConvertSchema(), required=True)
class PermissionSchema(Schema, DatedSchema):
id = String(dump_only=True)
write = Boolean(required=True)
read = Boolean(required=True)
collection_id = String(dump_only=True, required=True)
role = Nested(RoleReferenceSchema)
class UserUpdateSchema(Schema):
full_name = Str()
email = Email()
cpf_cnpj = Str()
address = Nested(AddressSchema)
class FunctionTestingInputDtoSchema(Schema):
declarative_input = Nested(
DeclarativeFunctionInputSchema,
load_from='declarativeInput',
dump_to='declarativeInput') # type: DeclarativeFunctionInput
class StackingJob(Job):
type = 'stacking'
description = 'Stack Images'
result: StackingJobResult = Nested(StackingJobResult, default={})
file_ids: TList[int] = List(Integer(), default=[])
# alignment_settings: AlignmentSettings = Nested(
# AlignmentSettings, default={})
stacking_settings: StackingSettings = Nested(StackingSettings, default={})
def run(self):
settings = self.stacking_settings
if settings.mode not in ('average', 'sum', 'percentile', 'mode'):
raise ValueError(
'Stacking mode must be "average", "sum", "percentile", or '
'"mode"')
if settings.scaling is not None and \
settings.scaling.lower() not in ('none', 'average', 'median',
'mode'):
raise ValueError(
'Stacking mode must be "none", "average", "median", or "mode"')
if settings.scaling is not None:
settings.scaling = settings.scaling.lower()
if settings.scaling == 'none':
settings.scaling = None
if settings.rejection is not None and \
settings.rejection.lower() not in ('none', 'chauvenet', 'iraf',
'minmax', 'sigclip'):
raise ValueError(
'Rejection mode must be "none", "chauvenet", "iraf", '
'"minmax", or "sigclip"')
if settings.rejection is not None:
settings.rejection = settings.rejection.lower()
if settings.rejection == 'none':
settings.rejection = None
lo, hi = settings.lo, settings.hi
if settings.rejection == 'iraf':
if lo is not None:
if lo % 1:
raise ValueError(
'Number of lowest values to clip for rejection=iraf '
'must be integer')
lo = int(lo)
if hi is not None:
if hi % 1:
raise ValueError(
'Number of highest values to clip for rejection=iraf '
'must be integer')
hi = int(hi)
# Load data files
if not self.file_ids:
return
data_files = [
get_data_file_data(self.user_id, file_id)
for file_id in self.file_ids
]
# Check data dimensions
shape = data_files[0][0].shape
for i, data_file in enumerate(list(data_files[1:])):
if data_file[0].shape != shape:
self.add_error(
ValueError('Shape mismatch: expected {0[1]}x{0[0]}, got '
'{1[1]}x{1[0]}'.format(shape,
data_file[0].shape)),
{'file_id': self.file_ids[i + 1]})
data_files.remove(data_file)
# Combine using the given settings
data, header = combine(data_files,
mode=settings.mode,
scaling=settings.scaling,
rejection=settings.rejection,
percentile=settings.percentile,
lo=lo,
hi=hi,
max_mem_mb=app.config.get('JOB_MAX_RAM'),
callback=self.update_progress)[0]
# Create a new data file in the given session and return its ID
adb = get_data_file_db(self.user_id)
try:
self.result.file_id = create_data_file(
adb,
None,
get_root(self.user_id),
data,
header,
duplicates='append',
session_id=self.session_id).id
adb.commit()
except Exception:
adb.rollback()
raise
finally:
adb.remove()
class NameClashSchema(Schema):
pet_1 = Nested(PetSchema)
pet_2 = Nested(Pet)
class AbstractSchema(mm.ModelSchema):
submitter = Nested(UserSchema)
judge = Nested(UserSchema)
modified_by = Nested(UserSchema)
duplicate_of = Nested('self', only=_basic_abstract_fields)
merged_into = Nested('self', only=_basic_abstract_fields)
submitted_contrib_type = Nested(contribution_type_schema_basic)
accepted_contrib_type = Nested(contribution_type_schema_basic)
accepted_track = Nested(track_schema_basic)
submitted_for_tracks = Nested(track_schema_basic, many=True)
reviewed_for_tracks = Nested(track_schema_basic, many=True)
persons = Nested(AbstractPersonLinkSchema,
attribute='person_links',
many=True)
custom_fields = Nested(ContributionFieldValueSchema,
attribute='field_values',
many=True)
score = Float()
comments = Nested(AbstractCommentSchema, many=True)
reviews = Nested(AbstractReviewSchema, many=True)
class Meta:
model = Abstract
fields = ('id', 'friendly_id', 'title', 'content', 'submitted_dt',
'modified_dt', 'judgment_dt', 'state', 'submitter',
'modified_by', 'judge', 'submission_comment',
'judgment_comment', 'submitted_contrib_type',
'accepted_contrib_type', 'accepted_track',
'submitted_for_tracks', 'reviewed_for_tracks',
'duplicate_of', 'merged_into', 'persons', 'custom_fields',
'score', 'comments', 'reviews')
class SchemaWithDict(Schema):
dict_field = Dict(values=Nested(PetSchema))
class TestSchema2(DraftEnabledSchema):
nest = Nested(TestNestedSchema())
class Spider(Model):
# TODO: validate id against allowed file name
id = String(primary_key=True, validate=Length(min=1, max=243))
start_urls = List(Nested(StartUrl))
links_to_follow = String(default='all',
validate=OneOf(
['none', 'patterns', 'all', 'auto']))
allowed_domains = List(Domain)
respect_nofollow = Boolean(default=True)
follow_patterns = List(Regexp)
exclude_patterns = List(Regexp)
js_enabled = Boolean(default=False)
js_enable_patterns = List(Regexp)
js_disable_patterns = List(Regexp)
perform_login = Boolean(default=False)
login_url = String(default='', allow_none=True)
login_user = String(default='', allow_none=True)
login_password = String(default='', allow_none=True)
project = BelongsTo(Project,
related_name='spiders',
on_delete=CASCADE,
ignore_in_file=True)
samples = HasMany('Sample',
related_name='spider',
on_delete=CLEAR,
only='id')
class Meta:
path = u'spiders/{self.id}.json'
def __repr__(self):
return super(Spider, self).__repr__('id')
@classmethod
def load(cls, storage, instance=None, project=None, **kwargs):
if instance is None and project:
# Load Spiders collection from file listing
directories, files = storage.listdir('spiders')
return cls.collection(
cls(storage,
snapshots=('committed', ),
id=strip_json(filename)).with_snapshots()
for filename in files if filename.endswith('.json'))
return super(Spider, cls).load(storage,
instance,
project=project,
**kwargs)
@pre_load
def populate_id(self, data):
spider_id = data.get('id')
if spider_id and not _ID_RE.match(spider_id):
return data
path = self.context['path']
data['id'] = strip_json(path.split('/')[-1])
return data
@pre_load
def dump_templates(self, data):
if not data.get('templates'):
path = '/'.join(strip_json(self.context['path']).split('/')[:2])
storage = self.context['storage']
try:
names = OrderedDict((strip_json(fname), 1)
for fname in storage.listdir(path)[1])
data['samples'] = list(names)
return data
except OSError:
# Directory does not exist
data['samples'] = []
return data
templates = []
for template in data['templates']:
# Only migrate item templates
if template.get('page_type') != 'item':
continue
template['id'] = template.get('page_id') or template.get('name')
templates.append(template['id'])
path = self.context['path']
path = '/'.join((strip_json(path).strip('/'),
'{}.json'.format(template['id'])))
sample = json.dumps(template, sort_keys=True, indent=4)
self.context['storage'].save(path, ContentFile(sample, path))
data['samples'] = templates
path, storage = self.context['path'], self.context['storage']
spider = json.dumps(data, indent=4, sort_keys=True)
storage.save(path, ContentFile(spider, path))
return data
@pre_load
def normalize_start_urls(self, data):
if 'start_urls' in data or 'generated_urls' in data:
start_urls = data.get('start_urls', []) + data.get(
'generated_urls', [])
data['start_urls'] = StartUrlCollection(start_urls).normalize()
return data
@pre_load
def get_init_requests(self, data):
init_requests = data.pop('init_requests', [])
if init_requests:
login_request = init_requests[0]
if isinstance(login_request, dict):
data['login_url'] = login_request.get('loginurl', '')
data['login_user'] = login_request.get('username', '')
data['login_password'] = login_request.get('password', '')
data['perform_login'] = self._is_perform_login(data)
return data
@post_dump
def set_init_requests(self, data):
if data.pop('perform_login', None) and self._is_perform_login(data):
data['init_requests'] = [
OrderedDict([
('type', 'login'),
('loginurl', data['login_url']),
('username', data['login_user']),
('password', data['login_password']),
])
]
data.pop('login_url', None)
data.pop('login_user', None)
data.pop('login_password', None)
data.pop('samples', None)
return OrderedDict(sorted(iteritems(data)))
@staticmethod
def _is_perform_login(data):
return all(
data.get(field)
for field in ('login_url', 'login_user', 'login_password'))
class ListPaymentSchema(Schema):
"""Schema used to return a list of payments."""
payments = List(Nested(PaymentSchema))
class BadRequestSchema(Schema):
""" Schema for bad authorization """
message = Nested(ErrorFieldSchema, many=True)
class PhotometryJobResult(JobResult):
data: TList[PhotometryData] = List(Nested(PhotometryData), default=[])
class ShallowCombinedSchema(BaseSchema):
collection_id = String()
# Joint entity/document attributes
collection = Nested(CollectionSchema())
schema = SchemaName()
schemata = List(SchemaName())
names = List(String())
addresses = List(String())
phones = List(String())
emails = List(String())
identifiers = List(String())
countries = List(Country())
dates = List(PartialDate())
bulk = Boolean()
# Entity attributes
foreign_id = String()
name = String()
entities = List(String())
properties = Dict()
# Document attributes
status = String()
content_hash = String()
uploader_id = String()
uploader = Nested(RoleReferenceSchema())
error_message = String()
title = String()
summary = String()
languages = List(Language())
keywords = List(String())
date = PartialDate()
authored_at = PartialDate()
modified_at = PartialDate()
published_at = PartialDate()
retrieved_at = PartialDate()
file_name = String()
file_size = Integer()
author = String()
generator = String()
mime_type = String()
extension = String()
encoding = String()
source_url = String()
pdf_version = String()
columns = List(String())
headers = Dict()
children = Integer()
# TODO: is this a separate endpoint?
text = String()
html = String()
def document_links(self, data, pk, schemata):
links = {
'self': url_for('documents_api.view', document_id=pk),
'tags': url_for('entities_api.tags', id=pk),
'ui': document_url(pk)
}
if data.get('content_hash'):
links['file'] = url_for('documents_api.file', document_id=pk)
if schemata.intersection([Document.SCHEMA_PDF]):
links['pdf'] = url_for('documents_api.pdf', document_id=pk)
if schemata.intersection([Document.SCHEMA_PDF, Document.SCHEMA_TABLE]):
links['records'] = url_for('documents_api.records', document_id=pk)
if schemata.intersection([Document.SCHEMA_FOLDER]):
query = (('filter:parent.id', pk), )
links['children'] = url_for('documents_api.index', _query=query)
return links
def entity_links(self, data, pk, schemata):
return {
'self': url_for('entities_api.view', id=pk),
# 'similar': url_for('entities_api.similar', id=pk),
# 'documents': url_for('entities_api.documents', id=pk),
'references': url_for('entities_api.references', id=pk),
'tags': url_for('entities_api.tags', id=pk),
'ui': entity_url(pk)
}
@post_dump
def hypermedia(self, data):
pk = str(data.get('id'))
collection = data.get('collection', {})
collection_id = collection.get('id')
collection_id = collection_id or data.get('collection_id')
schemata = set(data.get('schemata', []))
if Document.SCHEMA in schemata:
data['links'] = self.document_links(data, pk, schemata)
else:
data['links'] = self.entity_links(data, pk, schemata)
if data.get('bulk'):
data['writeable'] = False
else:
data['writeable'] = request.authz.can_write(collection_id)
return data
class HotelBookingWithNestedHotelSchema(HotelBookingSchema):
hotel = Nested(nested=HotelSchema(many=False))