def get_table(self, full_table_name):
"""get table from schemata
"""
schema, table_name = full_table_name.replace('`', '').split('.')
schema_module = self['schemata'].get(schema, None)
if schema_module is None:
raise LorisError(
f'schema {schema} not in database; refresh database')
table_name = table_name.strip('_#')
table_name_list = table_name.split('__')
if len(table_name_list) == 1:
table_name = to_camel_case(table_name)
try:
return getattr(schema_module, table_name)
except AttributeError:
raise LorisError(f'table {table_name} not in schema {schema}; '
'refresh database')
else:
assert len(table_name_list) == 2, \
f'invalid table name {table_name}.'
table_name = to_camel_case(table_name_list[0])
part_table_name = to_camel_case(table_name_list[1])
try:
return getattr(getattr(schema_module, table_name),
part_table_name)
except AttributeError:
raise LorisError(
f'table {table_name} not in schema {schema} '
f'or part table {part_table_name} not in table {table_name}'
'; refresh database')
def inserting_autoscript_stuff(attr, value, table_class, primary_dict):
"""inserting data/file from autoscript into database
"""
if attr.startswith('<') and attr.endswith('>'):
# assumes either data or filemixin was used
attr = attr.strip('<>')
part_table_name, attr = attr.split(':')
part_table = getattr(table_class, part_table_name)
if data_subclass(part_table, primary_dict):
to_insert = get_insert_part_mixin(attr,
value,
'data_lookup_name',
DataLookupName,
'a_datum',
primary_dict,
func=datareader)
elif file_subclass(part_table, primary_dict):
to_insert = get_insert_part_mixin(attr, value, 'file_lookup_name',
FileLookupName, 'a_file',
primary_dict)
else:
raise LorisError(f'part table {part_table.name} is not a '
'subclass of DataMixin or FilesMixin.')
part_table.insert1(to_insert)
elif attr in table_class.heading:
if table_class.heading[attr].is_blob:
value = datareader(value)
(table_class & primary_dict).save_update(attr, value)
else:
raise LorisError(f'attr {attr} does not exist in '
f'table {table_class.full_table_name}')
def get_table(self, full_table_name, as_string=False):
"""get table from schemata using full_table_name
"""
schema, table_name = full_table_name.replace('`', '').split('.')
schema_module = self['schemata'].get(schema, None)
if schema_module is None:
raise LorisError(
f'schema {schema} not in database; refresh database')
table_name = table_name.strip('_#')
table_name_list = table_name.split('__')
if as_string:
output = schema
else:
output = schema_module
for tname in table_name_list:
tname = to_camel_case(tname)
try:
if as_string:
output = '.'.join([output, tname])
else:
output = getattr(output, tname)
except AttributeError:
raise LorisError(f'table {table_name} not in schema {schema}; '
'refresh database')
return output
def __init__(self, key, value, folderpath):
self.key = key
self.folderpath = folderpath
if isinstance(value, (str, list)):
self.value = value
self.description = None
self.default = None
self.required = True
self.iterate = None
self.loc = None
elif isinstance(value, dict):
truth = set(value) - {
'type', 'comment', 'default', 'loc', 'iterate'
}
if truth:
raise LorisError('Key in dynamic auto-generated form contains '
f'illegal keywords: {truth}.')
if 'type' not in value:
raise LorisError(
'Must provide type key for dynamic auto-generated form; '
f'only provided these keys for "{key}": {set(value)}.')
self.value = value.get('type')
self.description = value.get('comment', None)
self.default = value.get('default', None)
self.required = ('default' not in value
or value.get('default', None) is not None)
self.loc = value.get('loc', None)
self.iterate = value.get('iterate', False)
else:
LorisError(f"value is wrong type {type(value)}")
self.get_field()
def dynamic_autoscripted_form(dictionary, folderpath, formclass=Form):
post_process_dict = {}
class DynamicForm(formclass, FormMixin):
pass
for key, value in dictionary.items():
# comments in the json or formatting guidelines start with _
if key.startswith('#'):
continue
if not key.isidentifier():
raise LorisError(
f"key {key} is not an identifier; i.e. alphanumeric "
"and underscore characters only. The key needs to be "
"an identifier if it is used as a keyword during function "
"calling.")
auto_field = AutoscriptedField(key, value, folderpath)
post_process_dict[key] = auto_field.post_process
setattr(DynamicForm, key, auto_field.field)
return DynamicForm, DictReader(post_process_dict)
def execute_slack_message(key_info, slack_info):
"""
Parameters
----------
key_info : dict
Information of a single entry of a specific table
slack_info : dict
Slack information passed to slack config
"""
token = slack_info['token']
channel = slack_info['channel']
text = slack_info.get('text', '').format(**key_info)
client = WebClient(token=token)
try:
response = client.chat_postMessage(channel=channel, text=text)
# assert response["message"]["text"] == text
except SlackApiError as e:
# You will get a SlackApiError if "ok" is False
# assert e.response["ok"] is False
# assert e.response["error"]
# str like 'invalid_auth', 'channel_not_found'
raise LorisError(f"Got a Slack error: {e}")
return response
def wait(self):
"""wait for the subprocess to finish
"""
if self.p is None:
raise LorisError('No subprocess is running.')
self.thread.join()
return self.rc, self.stdout, self.stderr
def get_formatted(self):
formatted_dict = super().get_formatted()
if formatted_dict['python_file'] is None and formatted_dict[
'python_module'] is None:
raise LorisError('No python file or module was given.')
elif formatted_dict['python_file']:
return formatted_dict['python_file']
else:
return formatted_dict['python_module']
def filereader(value):
"""
Copies filepath to tmp folder with uuid
"""
if not os.path.exists(value):
raise LorisError(f"Filepath {value} does not exist.")
basename = os.path.basename(value)
basename = f"{uuid.uuid4()}_{basename}"
dst = os.path.join(config['tmp_folder'], basename)
assert not os.path.exists(dst)
shutil.copyfile(value, dst)
return dst
def get_table_from_classname(self, class_name):
"""get table from schemata using table class name format
"""
table_info = class_name.split('.')
schema, table_classes = table_info[0], table_info[1:]
schema_module = self['schemata'].get(schema, None)
if schema_module is None:
raise LorisError(
f'schema {schema} not in database; refresh database')
table = schema_module
for table_ in table_classes:
try:
table = getattr(table, table_)
except AttributeError:
raise LorisError(f'table {table_} not in schema {schema}; '
'refresh database')
return table
def user_in_group(self, user, group):
"""is user in group
"""
table = self.assigned_table & {
self['user_name']: user,
self['group_name']: group
}
if len(table) == 1:
return True
elif len(table) == 0:
return False
else:
raise LorisError("assined user group table should only have "
"singular entries for given user and group.")
def __init__(self, user_name):
self.table = config.user_table
if not len(self.table()):
config.create_administrator()
self.user_name = user_name
self.restriction = {config['user_name']: self.user_name}
print(self.restriction)
self.restricted_table = self.table & self.restriction
if len(self.restricted_table) > 1:
raise LorisError('More than a single user entry')
self._entry = None
def __init__(self, user_name):
print(config['connection'])
self.table = config.user_table
# skip mysql error
try:
if not len(self.table()):
config.create_administrator()
except pymysql.err.InterfaceError:
warnings.warn('not creating administrator')
self.user_name = user_name
self.restriction = {config['user_name']: self.user_name}
print(self.restriction)
self.restricted_table = self.table & self.restriction
if len(self.restricted_table) > 1:
raise LorisError('More than a single user entry')
self._entry = None
while True:
length = len(process.lines)
if length > lnumbers:
print(''.join(process.lines[lnumbers:length]))
lnumbers = length
if process.p is not None and process.p.poll() is not None:
break
if process.thread.is_alive():
process.wait()
if process.stdout is not None:
print(process.stdout)
if process.rc != 0:
raise LorisError(f'automatic script error:\n{process.stderr}')
# update/insert fields with data from autoscript
if args.configattr != 'null' and args.configattr is not None:
inserting_autoscript_stuff(args.configattr, args.location,
table_class, primary_dict)
if args.outputattr != 'null' and args.outputattr is not None:
inserting_autoscript_stuff(args.outputattr,
os.path.join(cwd, args.outputfile),
table_class, primary_dict)
# field name or <part_table_name:data/file_lookupname>
# or just an attribute in the table
except Exception as e:
jobs.complete(table_class.full_table_name, primary_dict)
raise e
else:
def _get_field(cls, key, value, required, default, description, iterate,
loc, folderpath):
"""get initialized field
"""
def post_process(x):
return x
if required:
kwargs = {
'validators': [InputRequired()],
'render_kw': {
'nullable': False
}
}
else:
kwargs = {
'validators': [Optional()],
'render_kw': {
'nullable': True
}
}
kwargs['default'] = default
kwargs['label'] = key.replace('_', ' ')
kwargs['description'] = (key if description is None else description)
if loc is None and not isinstance(value, dict):
if value == 'list':
kwargs['validators'].append(JsonSerializableValidator())
field = ListField(**kwargs)
elif value == 'dict':
kwargs['validators'].append(JsonSerializableValidator())
field = DictField(**kwargs)
elif value == 'str':
field = StringField(**kwargs)
elif value == 'set':
kwargs['validators'].append(JsonSerializableValidator())
post_process = set
field = ListField(**kwargs)
elif value == 'tuple':
kwargs['validators'].append(JsonSerializableValidator())
post_process = tuple
field = ListField(**kwargs)
elif value == 'int':
field = IntegerField(**kwargs)
elif value == 'float':
field = FloatField(**kwargs)
elif value == 'bool':
kwargs['validators'] = [Optional()]
field = BooleanField(**kwargs)
elif value == 'numpy.array':
kwargs['validators'].append(Extension())
post_process = cls.file_processing(value)
field = DynamicFileField(**kwargs)
elif value == 'numpy.recarray':
kwargs['validators'].append(Extension())
post_process = cls.file_processing(value)
field = DynamicFileField(**kwargs)
elif value == 'pandas.DataFrame':
kwargs['validators'].append(Extension())
post_process = cls.file_processing(value)
field = DynamicFileField(**kwargs)
elif value == 'pandas.Series':
kwargs['validators'].append(Extension())
post_process = cls.file_processing(value)
field = DynamicFileField(**kwargs)
elif value == 'json':
kwargs['validators'].append(Extension(['json']))
post_process = cls.file_processing(value)
field = DynamicFileField(**kwargs)
elif value == 'file':
kwargs['validators'].append(
Extension(config['attach_extensions']))
field = DynamicFileField(**kwargs)
elif isinstance(value, list):
choices = [
str(ele).strip().strip('"').strip("'") for ele in value
]
post_process = EnumReader(value, choices)
if default is None:
choices = ['NULL'] + choices
kwargs['choices'] = [(ele, ele) for ele in choices]
field = SelectField(**kwargs)
else:
raise LorisError(
f"field value {value} not accepted for {key}.")
elif loc is not None and isinstance(value, str):
loc = secure_filename(loc)
locpath = os.path.join(folderpath, loc)
# try up to three base directories down
if not os.path.exists(locpath):
# try main autoscript folder
locpath = os.path.join(os.path.dirname(folderpath), loc)
if not os.path.exists(locpath):
locpath = os.path.join(
os.path.dirname(os.path.dirname(folderpath)), loc)
if not os.path.exists(locpath):
raise LorisError(
f'Folder "{loc}" does not exist in '
f'autoscript folder '
f'"{os.path.basename(folderpath)}" '
f'and also not in the main autoscript folder.')
# get all files from folder
files = glob.glob(os.path.join(locpath, '*'))
# only match certain extensions
if (value == 'pandas.DataFrame') or (value == 'numpy.recarray'):
files = [
ifile for ifile in files
if (ifile.endswith('.pkl') or ifile.endswith('.npy')
or ifile.endswith('.csv') or ifile.endswith('.json'))
]
elif value == 'numpy.array':
files = [
ifile for ifile in files
if (ifile.endswith('.pkl') or ifile.endswith('.npy')
or ifile.endswith('.csv'))
]
elif (value == 'json') or (value == 'pandas.Series'):
files = [ifile for ifile in files if ifile.endswith('.json')]
else:
# skip file that start with two underscores e.g. __init__.py
files = [
ifile for ifile in files
if not os.path.basename(ifile).startswith('__')
]
# setup as choices
choices = [(str(ele), os.path.split(ele)[-1]) for ele in files]
# setup None choice
if default is None and not required:
choices = [('NULL', 'NULL')] + choices
kwargs['choices'] = choices
post_process = cls.file_processing(value)
field = SelectField(**kwargs)
elif isinstance(value, dict):
form, post_process = dynamic_autoscripted_form(
value, folderpath, NoCsrfForm)
field = FormField(form)
# TODO set number of fieldlists (startswith numeric)
else:
raise LorisError(f"field value {value} not accepted for {key}.")
# make iterator (can add multiple values together)
if iterate:
field = FieldList(
field,
min_entries=int(required) + 1 # have one required field if so
)
post_process = ListReader(post_process)
return field, post_process
def insert(self, form, _id=None, **kwargs):
"""insert into datajoint table
Parameters
----------
form : wtf.form from dynamicform.formclass
_id : dict
restriction for single entry (for save updating)
kwargs : dict
arguments passed to datajoint Table.insert function
"""
if isinstance(form, FormMixin):
formatted_dict = form.get_formatted()
elif isinstance(form, dict):
formatted_dict = form
else:
raise LorisError(f'form is incorrect type (={type(form)}); '
'form must be dict or be a subclass of FormMixin')
# check if replace in kwargs affects order of main and part helper
replace = kwargs.get('replace', False)
def main_helper(override_update_truth=False):
primary_dict = self._insert(
formatted_dict,
_id,
override_update_truth=override_update_truth,
**kwargs)
return primary_dict
def part_helper(primary_dict):
for part_name, part_form in self.part_fields.items():
f_dicts = formatted_dict[part_name]
if f_dicts is None:
continue
for f_dict in f_dicts:
if _id is None:
_part_id = None
else:
# update with part entry that exist
_part_primary = {}
for key, value in f_dict.items():
if (key in part_form.table.primary_key
and key not in self.table.primary_key):
_part_primary.update(
part_form.fields[key].format_value(value))
_part_id = {**_id, **_part_primary}
# insert into part table
part_form._insert(f_dict,
_part_id,
primary_dict,
override_update_truth=True,
**kwargs)
if self.table.connection.in_transaction:
if replace and _id is not None:
part_helper(_id)
kwargs.pop('replace')
primary_dict = main_helper(True)
else:
primary_dict = main_helper()
part_helper(primary_dict)
else:
# perform operations within a transaction
with self.table.connection.transaction:
if replace and _id is not None:
part_helper(_id)
kwargs.pop('replace')
primary_dict = main_helper(True)
else:
primary_dict = main_helper()
part_helper(primary_dict)
return primary_dict
def get_autoscriptforms(self, autoscript_filepath, table_name,
form_creator, **kwargs):
"""get autoscript form and process_dict and buttons params
"""
foldername = os.path.basename(autoscript_filepath)
filepath = os.path.join(autoscript_filepath, AUTOSCRIPT_CONFIG)
with open(filepath, 'r') as f:
try:
config = json.load(f)
except Exception as e:
raise LorisError(f"Json config file for autoscript"
f" '{foldername}' badly "
f"formatted: {e}")
if 'buttons' not in config:
raise LorisError("In configuration file of autoscript "
f"{foldername}, no 'button' key "
"was provided.")
buttons = config['buttons']
config_forms = config.get('forms', {})
if not isinstance(buttons, dict):
raise LorisError(f'In configuration file of autoscript '
f'"{foldername}", '
'"scripts" keyword is incorrectly '
'formatted.')
for key, button in buttons.items():
base_message = (f'In configuration file of autoscript '
f'"{foldername}", '
'"buttons" keyword is incorrectly '
f'formatted for button "{key}":')
message = (f'{base_message} must be dictionary with '
'"script" key and "validate", "insert", "configattr", '
'"outputfile", and "outputattr" optionally defined.')
if not isinstance(button, dict):
raise LorisError(message)
elif not all([
isinstance(button.get('script', None), str),
isinstance(button.get('validate', []), list),
isinstance(button.get('insert', False), bool),
isinstance(button.get('configattr', ''), str),
isinstance(button.get('outputfile', ''), str),
isinstance(button.get('outputattr', ''), str),
]):
raise LorisError(message)
button['script'] = secure_filename(button['script'])
if not os.path.exists(
os.path.join(autoscript_filepath, button['script'])):
raise LorisError(f'{base_message} script "{button["script"]}" '
'does not exist in autoscript folder.')
if 'outputfile' in button:
button['outputfile'] = secure_filename(button['outputfile'])
if not isinstance(config_forms, dict):
raise LorisError(f'In configuration file of autoscript '
f'"{os.path.basename(autoscript_filepath)}", '
'"forms" keyword is incorrectly '
'formatted.')
forms = {}
post_process_dict = {}
for key, value in config_forms.items():
form, post_process = form_creator(value, autoscript_filepath,
**kwargs)
forms[key] = form
post_process_dict[key] = post_process
return forms, post_process_dict, buttons