class FetchTableForm(NoCsrfForm, FormMixin):
table_name = SelectField('table name',
description='choose table',
choices=[(table_name, table_name)
for table_name in config['tables']],
validators=[InputRequired()])
proj_list = ListField(
'projections',
description='arguments to project',
validators=[Optional(),
JsonSerializableValidator(list)],
render_kw={'nullable': True})
proj_dict = DictField(
'renamed projections',
description='arguments to project and rename',
validators=[Optional(),
JsonSerializableValidator(dict)],
render_kw={'nullable': True})
def get_formatted(self):
formatted = super().get_formatted()
return {
formatted['table_name']: (([] if formatted['proj_list'] is None
else formatted['proj_list']),
({} if formatted['proj_dict'] is None
else formatted['proj_dict']))
}
class FuncForm(NoCsrfForm, FormMixin):
module = FormField(ModuleForm)
function = StringField('function',
description='function name in the module',
validators=[InputRequired()])
args = ListField(
'args',
description='list of arguments for init if function is class',
validators=[Optional(), JsonSerializableValidator(list)],
render_kw={'nullable': True})
kwargs = DictField(
'kwargs',
description='dict of keyword arguments for init if function is class',
validators=[Optional(), JsonSerializableValidator(dict)],
render_kw={'nullable': True})
def get_formatted(self):
formatted = super().get_formatted()
if formatted['args'] is None and formatted['kwargs'] is None:
return (formatted['module'], formatted['function'])
else:
return (formatted['module'], formatted['function'],
([] if formatted['args'] is None else formatted['args']),
({}
if formatted['kwargs'] is None else formatted['kwargs']))
def adapted_field(self, kwargs):
"""creates an adapted field type
"""
try:
attr_type = self.attr.adapter.attribute_type
except NotImplementedError:
attr_type = self.attr.sql_type
attr_type_name = self.attr.type.strip('<>')
adapter = config['custom_attributes'].get(attr_type_name, None)
if adapter is None:
pass
elif attr_type_name == 'liststring':
kwargs['validators'].append(JsonSerializableValidator(list))
return ListField(**kwargs)
elif attr_type_name == 'dictstring':
kwargs['validators'].append(JsonSerializableValidator(dict))
return DictField(**kwargs)
elif attr_type_name == 'tags':
return TagListField(**kwargs)
elif attr_type_name == 'link':
kwargs['validators'].append(URL(False))
elif attr_type_name == 'email':
kwargs['validators'].append(Email())
elif attr_type_name == 'lookupname':
kwargs['validators'].append(LookupNameValidator())
return self._create_field(attr_type, kwargs)
class FormConfig(NoCsrfForm, FormMixin):
name = StringField('name',
description=('name of the additional form'),
validators=[InputRequired()])
form_config = DictTextArea(
'form configuration',
description=('dictionary configuration of additional form'),
validators=[InputRequired(),
JsonSerializableValidator(dict)])
def _get_field(cls, key, value, required, default, description, iterate,
loc, folderpath):
"""get initialized field
"""
def post_process(x):
return x
if required:
kwargs = {
'validators': [InputRequired()],
'render_kw': {
'nullable': False
}
}
else:
kwargs = {
'validators': [Optional()],
'render_kw': {
'nullable': True
}
}
kwargs['default'] = default
kwargs['label'] = key.replace('_', ' ')
kwargs['description'] = (key if description is None else description)
if loc is None and not isinstance(value, dict):
if value == 'list':
kwargs['validators'].append(JsonSerializableValidator())
field = ListField(**kwargs)
elif value == 'dict':
kwargs['validators'].append(JsonSerializableValidator())
field = DictField(**kwargs)
elif value == 'str':
field = StringField(**kwargs)
elif value == 'set':
kwargs['validators'].append(JsonSerializableValidator())
post_process = set
field = ListField(**kwargs)
elif value == 'tuple':
kwargs['validators'].append(JsonSerializableValidator())
post_process = tuple
field = ListField(**kwargs)
elif value == 'int':
field = IntegerField(**kwargs)
elif value == 'float':
field = FloatField(**kwargs)
elif value == 'bool':
kwargs['validators'] = [Optional()]
field = BooleanField(**kwargs)
elif value == 'numpy.array':
kwargs['validators'].append(Extension())
post_process = cls.file_processing(value)
field = DynamicFileField(**kwargs)
elif value == 'numpy.recarray':
kwargs['validators'].append(Extension())
post_process = cls.file_processing(value)
field = DynamicFileField(**kwargs)
elif value == 'pandas.DataFrame':
kwargs['validators'].append(Extension())
post_process = cls.file_processing(value)
field = DynamicFileField(**kwargs)
elif value == 'pandas.Series':
kwargs['validators'].append(Extension())
post_process = cls.file_processing(value)
field = DynamicFileField(**kwargs)
elif value == 'json':
kwargs['validators'].append(Extension(['json']))
post_process = cls.file_processing(value)
field = DynamicFileField(**kwargs)
elif value == 'file':
kwargs['validators'].append(
Extension(config['attach_extensions']))
field = DynamicFileField(**kwargs)
elif isinstance(value, list):
choices = [
str(ele).strip().strip('"').strip("'") for ele in value
]
post_process = EnumReader(value, choices)
if default is None:
choices = ['NULL'] + choices
kwargs['choices'] = [(ele, ele) for ele in choices]
field = SelectField(**kwargs)
else:
raise LorisError(
f"field value {value} not accepted for {key}.")
elif loc is not None and isinstance(value, str):
loc = secure_filename(loc)
locpath = os.path.join(folderpath, loc)
# try up to three base directories down
if not os.path.exists(locpath):
# try main autoscript folder
locpath = os.path.join(os.path.dirname(folderpath), loc)
if not os.path.exists(locpath):
locpath = os.path.join(
os.path.dirname(os.path.dirname(folderpath)), loc)
if not os.path.exists(locpath):
raise LorisError(
f'Folder "{loc}" does not exist in '
f'autoscript folder '
f'"{os.path.basename(folderpath)}" '
f'and also not in the main autoscript folder.')
# get all files from folder
files = glob.glob(os.path.join(locpath, '*'))
# only match certain extensions
if (value == 'pandas.DataFrame') or (value == 'numpy.recarray'):
files = [
ifile for ifile in files
if (ifile.endswith('.pkl') or ifile.endswith('.npy')
or ifile.endswith('.csv') or ifile.endswith('.json'))
]
elif value == 'numpy.array':
files = [
ifile for ifile in files
if (ifile.endswith('.pkl') or ifile.endswith('.npy')
or ifile.endswith('.csv'))
]
elif (value == 'json') or (value == 'pandas.Series'):
files = [ifile for ifile in files if ifile.endswith('.json')]
else:
# skip file that start with two underscores e.g. __init__.py
files = [
ifile for ifile in files
if not os.path.basename(ifile).startswith('__')
]
# setup as choices
choices = [(str(ele), os.path.split(ele)[-1]) for ele in files]
# setup None choice
if default is None and not required:
choices = [('NULL', 'NULL')] + choices
kwargs['choices'] = choices
post_process = cls.file_processing(value)
field = SelectField(**kwargs)
elif isinstance(value, dict):
form, post_process = dynamic_autoscripted_form(
value, folderpath, NoCsrfForm)
field = FormField(form)
# TODO set number of fieldlists (startswith numeric)
else:
raise LorisError(f"field value {value} not accepted for {key}.")
# make iterator (can add multiple values together)
if iterate:
field = FieldList(
field,
min_entries=int(required) + 1 # have one required field if so
)
post_process = ListReader(post_process)
return field, post_process
class SettingstableForm(Form, FormMixin):
settings_name = StringField(
'settings name',
description='unique name for settings used to autopopulate',
validators=[InputRequired(), Length(max=63)])
description = TextAreaField(
'description',
description='longer description to describe settings',
validators=[Optional(), Length(max=4000)],
render_kw={'nullable': True})
func = FormField(FuncForm)
global_settings = DictField(
'global settings',
description=
'dict of keyword arguments to pass to function for every entry',
validators=[InputRequired(),
JsonSerializableValidator(dict)])
entry_settings = DictField(
'entry settings',
description=
'dict of keyword arguments to pass to function specific to each entry as defined by columns of the joined table',
validators=[InputRequired(),
JsonSerializableValidator(dict)])
fetch_method = SelectField(
'fetch method',
description='method used to fetch data',
choices=[('fetch1', 'fetch1'), ('fetch', 'fetch')],
)
fetch_tables = FieldList(FormField(FetchTableForm),
label='fetch tables',
min_entries=1,
render_kw={'nullable': True})
assign_output = SelectField(
'assign output',
description='assign the output of the function to a single column',
choices=[('NULL', 'NULL')] +
[(str(ele), str(ele))
for ele in table_class.child_table().heading.secondary_attributes
])
restrictions = RestrictionField(RESTRICTION_LABEL,
description=RESTRICTION_DESCRIPTION,
validators=[
Optional(),
OptionalJsonSerializableValidator(
(list, dict))
],
render_kw={'nullable': True})
parse_unique = ListField(
'parse as unique',
description=
'list of unique entries when using fetch - not wrapped into numpy.array',
validators=[Optional(),
JsonSerializableValidator(list)],
render_kw={'nullable': True})
def get_formatted(self):
formatted = super().get_formatted()
if formatted['fetch_tables'] is not None:
formatted['fetch_tables'] = {
key: value
for table_dict in formatted['fetch_tables']
for key, value in table_dict.items()
}
return formatted