def execute_patch(patchmodule, method=None, methodargs=None):
"""execute the patch"""
block_user(True)
dataent.db.begin()
start_time = time.time()
try:
log('Executing {patch} in {site} ({db})'.format(patch=patchmodule or str(methodargs),
site=dataent.local.site, db=dataent.db.cur_db_name))
if patchmodule:
if patchmodule.startswith("finally:"):
# run run patch at the end
dataent.flags.final_patches.append(patchmodule)
else:
if patchmodule.startswith("execute:"):
exec(patchmodule.split("execute:")[1],globals())
else:
dataent.get_attr(patchmodule.split()[0] + ".execute")()
update_patch_log(patchmodule)
elif method:
method(**methodargs)
except Exception:
dataent.db.rollback()
raise
else:
dataent.db.commit()
end_time = time.time()
block_user(False)
log('Success: Done in {time}s'.format(time = round(end_time - start_time, 3)))
return True
def scheduler_task(site, event, handler, now=False):
'''This is a wrapper function that runs a hooks.scheduler_events method'''
dataent.logger(__name__).info(
'running {handler} for {site} for event: {event}'.format(
handler=handler, site=site, event=event))
try:
if not now:
dataent.connect(site=site)
dataent.flags.in_scheduler = True
dataent.get_attr(handler)()
except Exception:
dataent.db.rollback()
traceback = log(
handler,
"Method: {event}, Handler: {handler}".format(event=event,
handler=handler))
dataent.logger(__name__).error(traceback)
raise
else:
dataent.db.commit()
dataent.logger(__name__).info(
'ran {handler} for {site} for event: {event}'.format(handler=handler,
site=site,
event=event))
def get_website_settings():
hooks = dataent.get_hooks()
context = dataent._dict({
'top_bar_items': get_items('top_bar_items'),
'footer_items': get_items('footer_items'),
"post_login": [
{"label": _("My Account"), "url": "/me"},
# {"class": "divider"},
{"label": _("Logout"), "url": "/?cmd=web_logout"}
]
})
settings = dataent.get_single("Website Settings")
for k in ["banner_html", "brand_html", "copyright", "twitter_share_via",
"facebook_share", "google_plus_one", "twitter_share", "linked_in_share",
"disable_signup", "hide_footer_signup", "head_html", "title_prefix",
"navbar_search"]:
if hasattr(settings, k):
context[k] = settings.get(k)
if settings.address:
context["footer_address"] = settings.address
for k in ["facebook_share", "google_plus_one", "twitter_share", "linked_in_share",
"disable_signup"]:
context[k] = int(context.get(k) or 0)
if dataent.request:
context.url = quote(str(get_request_site_address(full_address=True)), safe="/:")
context.encoded_title = quote(encode(context.title or ""), str(""))
for update_website_context in hooks.update_website_context or []:
dataent.get_attr(update_website_context)(context)
context.web_include_js = hooks.web_include_js or []
context.web_include_css = hooks.web_include_css or []
via_hooks = dataent.get_hooks("website_context")
for key in via_hooks:
context[key] = via_hooks[key]
if key not in ("top_bar_items", "footer_items", "post_login") \
and isinstance(context[key], (list, tuple)):
context[key] = context[key][-1]
add_website_theme(context)
if not context.get("favicon"):
context["favicon"] = "/assets/dataent/images/favicon.png"
if settings.favicon and settings.favicon != "attach_files:":
context["favicon"] = settings.favicon
return context
def main(app=None, module=None, doctype=None, verbose=False, tests=(),
force=False, profile=False, junit_xml_output=None, ui_tests=False,
doctype_list_path=None, skip_test_records=False, failfast=False):
global unittest_runner
if doctype_list_path:
app, doctype_list_path = doctype_list_path.split(os.path.sep, 1)
with open(dataent.get_app_path(app, doctype_list_path), 'r') as f:
doctype = f.read().strip().splitlines()
xmloutput_fh = None
if junit_xml_output:
xmloutput_fh = open(junit_xml_output, 'w')
unittest_runner = xmlrunner_wrapper(xmloutput_fh)
else:
unittest_runner = unittest.TextTestRunner
try:
dataent.flags.print_messages = verbose
dataent.flags.in_test = True
if not dataent.db:
dataent.connect()
# if not dataent.conf.get("db_name").startswith("test_"):
# raise Exception, 'db_name must start with "test_"'
# workaround! since there is no separate test db
dataent.clear_cache()
dataent.utils.scheduler.disable_scheduler()
set_test_email_config()
if not dataent.flags.skip_before_tests:
if verbose:
print('Running "before_tests" hooks')
for fn in dataent.get_hooks("before_tests", app_name=app):
dataent.get_attr(fn)()
if doctype:
ret = run_tests_for_doctype(doctype, verbose, tests, force, profile)
elif module:
ret = run_tests_for_module(module, verbose, tests, profile)
else:
ret = run_all_tests(app, verbose, profile, ui_tests, failfast=failfast)
dataent.db.commit()
# workaround! since there is no separate test db
dataent.clear_cache()
return ret
finally:
if xmloutput_fh:
xmloutput_fh.flush()
xmloutput_fh.close()
def get():
"""get session boot info"""
from dataent.desk.notifications import \
get_notification_info_for_boot, get_notifications
from dataent.boot import get_bootinfo, get_unseen_notes
from dataent.limits import get_limits, get_expiry_message
bootinfo = None
if not getattr(dataent.conf,'disable_session_cache', None):
# check if cache exists
bootinfo = dataent.cache().hget("bootinfo", dataent.session.user)
if bootinfo:
bootinfo['from_cache'] = 1
bootinfo["notification_info"].update(get_notifications())
bootinfo["user"]["recent"] = json.dumps(\
dataent.cache().hget("user_recent", dataent.session.user))
if not bootinfo:
# if not create it
bootinfo = get_bootinfo()
bootinfo["notification_info"] = get_notification_info_for_boot()
dataent.cache().hset("bootinfo", dataent.session.user, bootinfo)
try:
dataent.cache().ping()
except redis.exceptions.ConnectionError:
message = _("Redis cache server not running. Please contact Administrator / Tech support")
if 'messages' in bootinfo:
bootinfo['messages'].append(message)
else:
bootinfo['messages'] = [message]
# check only when clear cache is done, and don't cache this
if dataent.local.request:
bootinfo["change_log"] = get_change_log()
bootinfo["metadata_version"] = dataent.cache().get_value("metadata_version")
if not bootinfo["metadata_version"]:
bootinfo["metadata_version"] = dataent.reset_metadata_version()
bootinfo.notes = get_unseen_notes()
for hook in dataent.get_hooks("extend_bootinfo"):
dataent.get_attr(hook)(bootinfo=bootinfo)
bootinfo["lang"] = dataent.translate.get_user_lang()
bootinfo["disable_async"] = dataent.conf.disable_async
bootinfo["setup_complete"] = cint(dataent.db.get_single_value('System Settings', 'setup_complete'))
# limits
bootinfo.limits = get_limits()
bootinfo.expiry_message = get_expiry_message()
return bootinfo
def execute_postprocess(self, status):
# Execute post process
postprocess_method_path = self.get_plan().postprocess_method
if postprocess_method_path:
dataent.get_attr(postprocess_method_path)({
"status": status,
"stats": {
"push_insert": self.push_insert,
"push_update": self.push_update,
"push_delete": self.push_delete,
"pull_insert": self.pull_insert,
"pull_update": self.pull_update
}
})
def get_hook_method(hook_name, fallback=None):
method = (dataent.get_hooks().get(hook_name))
if method:
method = dataent.get_attr(method[0])
return method
if fallback:
return fallback
def get_versions():
"""Get versions of all installed apps.
Example:
{
"dataent": {
"title": "Dataent Framework",
"version": "5.0.0"
}
}"""
versions = {}
for app in dataent.get_installed_apps(sort=True):
app_hooks = dataent.get_hooks(app_name=app)
versions[app] = {
"title": app_hooks.get("app_title")[0],
"description": app_hooks.get("app_description")[0],
"branch": get_app_branch(app)
}
if versions[app]['branch'] != 'master':
branch_version = app_hooks.get('{0}_version'.format(
versions[app]['branch']))
if branch_version:
versions[app][
'branch_version'] = branch_version[0] + ' ({0})'.format(
get_app_last_commit_ref(app))
try:
versions[app]["version"] = dataent.get_attr(app + ".__version__")
except AttributeError:
versions[app]["version"] = '0.0.1'
return versions
def get_reply(self):
if self.startswith('open', 'show open', 'list open', 'get open'):
if self.tables:
doctype = self.get_doctype()
from dataent.desk.notifications import get_notification_config
filters = get_notification_config().get('for_doctype').get(
doctype, None)
if filters:
if isinstance(filters, dict):
data = dataent.get_list(doctype, filters=filters)
else:
data = [{
'name': d[0],
'title': d[1]
} for d in dataent.get_attr(filters)(as_list=True)]
return ", ".join(
'[{title}](#Form/{doctype}/{name})'.format(
doctype=doctype,
name=d.get('name'),
title=d.get('title') or d.get('name'))
for d in data)
else:
return _("Can't identify open {0}. Try something else."
).format(doctype)
def get_reply(self, query):
self.query = query.lower()
self.setup()
self.pre_process()
# basic replies
if self.query.split()[0] in ("hello", "hi"):
return _("Hello {0}").format(dataent.utils.get_fullname())
if self.query == "help":
return help_text.format(dataent.utils.get_fullname())
# build using parsers
replies = []
for parser in dataent.get_hooks('bot_parsers'):
reply = None
try:
reply = dataent.get_attr(parser)(self, query).get_reply()
except dataent.PermissionError:
reply = _("Oops, you are not allowed to know that")
if reply:
replies.append(reply)
if replies:
return '\n\n'.join(replies)
if not reply:
return _("Don't know, ask 'help'")
def get_all_nodes(doctype, parent, tree_method, **filters):
'''Recursively gets all data from tree nodes'''
if 'cmd' in filters:
del filters['cmd']
tree_method = dataent.get_attr(tree_method)
if not tree_method in dataent.whitelisted:
dataent.throw(_("Not Permitted"), dataent.PermissionError)
data = tree_method(doctype, parent, **filters)
out = [dict(parent=parent, data=data)]
if 'is_root' in filters:
del filters['is_root']
to_check = [d.get('value') for d in data if d.get('expandable')]
while to_check:
parent = to_check.pop()
data = tree_method(doctype, parent, is_root=False, **filters)
out.append(dict(parent=parent, data=data))
for d in data:
if d.get('expandable'):
to_check.append(d.get('value'))
return out
def _get_home_page():
home_page = None
get_website_user_home_page = dataent.get_hooks(
'get_website_user_home_page')
if get_website_user_home_page:
home_page = dataent.get_attr(get_website_user_home_page[-1])(
dataent.session.user)
if not home_page:
role_home_page = dataent.get_hooks("role_home_page")
if role_home_page:
for role in dataent.get_roles():
if role in role_home_page:
home_page = role_home_page[role][-1]
break
if not home_page:
home_page = dataent.get_hooks("home_page")
if home_page:
home_page = home_page[-1]
if not home_page:
home_page = dataent.db.get_value("Website Settings", None,
"home_page") or "login"
home_page = home_page.strip('/')
return home_page
def get_site_info():
from dataent.utils.user import get_system_managers
from dataent.core.doctype.user.user import STANDARD_USERS
from dataent.email.queue import get_emails_sent_this_month
# only get system users
users = dataent.get_all('User',
filters={
'user_type': 'System User',
'name': ('not in', STANDARD_USERS)
},
fields=[
'name', 'enabled', 'last_login', 'last_active',
'language', 'time_zone'
])
system_managers = get_system_managers(only_name=True)
for u in users:
# tag system managers
u.is_system_manager = 1 if u.name in system_managers else 0
u.full_name = get_fullname(u.name)
u.email = u.name
del u['name']
system_settings = dataent.db.get_singles_dict('System Settings')
space_usage = dataent._dict((dataent.local.conf.limits
or {}).get('space_usage', {}))
kwargs = {
"fields": ["user", "creation", "full_name"],
"filters": {
"Operation": "Login",
"Status": "Success"
},
"limit": "10"
}
site_info = {
'installed_apps': get_installed_apps_info(),
'users': users,
'country': system_settings.country,
'language': system_settings.language or 'english',
'time_zone': system_settings.time_zone,
'setup_complete': cint(system_settings.setup_complete),
'scheduler_enabled': system_settings.enable_scheduler,
# usage
'emails_sent': get_emails_sent_this_month(),
'space_used': flt((space_usage.total or 0) / 1024.0, 2),
'database_size': space_usage.database_size,
'backup_size': space_usage.backup_size,
'files_size': space_usage.files_size,
'last_logins': dataent.get_all("Activity Log", **kwargs)
}
# from other apps
for method_name in dataent.get_hooks('get_site_info'):
site_info.update(dataent.get_attr(method_name)(site_info) or {})
# dumps -> loads to prevent datatype conflicts
return json.loads(dataent.as_json(site_info))
def get_attr(cmd):
"""get method object from cmd"""
if '.' in cmd:
method = dataent.get_attr(cmd)
else:
method = globals()[cmd]
dataent.log("method:" + cmd)
return method
def update_and_get_user_progress():
'''
Return setup progress action states (called via `update_and_get_user_progress` hook)
'''
states = {}
for fn in dataent.get_hooks('update_and_get_user_progress'):
states.update(dataent.get_attr(fn)())
return states
def get_website_user_home_page(user):
home_page_method = dataent.get_hooks('get_website_user_home_page')
if home_page_method:
home_page = dataent.get_attr(home_page_method[-1])(user)
return '/' + home_page.strip('/')
elif dataent.get_hooks('website_user_home_page'):
return '/' + dataent.get_hooks('website_user_home_page')[-1].strip('/')
else:
return '/me'
def caller(*args, **kwargs):
region = get_region()
fn_name = inspect.getmodule(fn).__name__ + '.' + fn.__name__
if region in regional_overrides and fn_name in regional_overrides[
region]:
return dataent.get_attr(regional_overrides[region][fn_name])(
*args, **kwargs)
else:
return fn(*args, **kwargs)
def composer(self, *args, **kwargs):
hooks = []
method = f.__name__
doc_events = dataent.get_doc_hooks()
for handler in doc_events.get(self.doctype, {}).get(method, []) \
+ doc_events.get("*", {}).get(method, []):
hooks.append(dataent.get_attr(handler))
composed = compose(f, *hooks)
return composed(self, method, *args, **kwargs)
def get_user_progress_slides():
'''
Return user progress slides for the desktop (called via `get_user_progress_slides` hook)
'''
slides = []
if cint(dataent.db.get_single_value('System Settings', 'setup_complete')):
for fn in dataent.get_hooks('get_user_progress_slides'):
slides += dataent.get_attr(fn)()
return slides
def map_docs(method, source_names, target_doc): '''Returns the mapped document calling the given mapper method with each of the given source docs on the target doc''' method = dataent.get_attr(method) if method not in dataent.whitelisted: raise dataent.PermissionError for src in json.loads(source_names): target_doc = method(src, target_doc) return target_doc
def setup_domain(self): '''Setup domain icons, permissions, custom fields etc.''' self.setup_data() self.setup_roles() self.setup_properties() self.set_values() # always set the desktop icons while changing the domain settings self.setup_desktop_icons() if not int(dataent.defaults.get_defaults().setup_complete or 0): # if setup not complete, setup desktop etc. self.setup_sidebar_items() self.set_default_portal_role() if self.data.custom_fields: create_custom_fields(self.data.custom_fields) if self.data.on_setup: # custom on_setup method dataent.get_attr(self.data.on_setup)()
def get_dict_from_hooks(fortype, name):
translated_dict = {}
hooks = dataent.get_hooks("get_translated_dict")
for (hook_fortype, fortype_name) in hooks:
if hook_fortype == fortype and fortype_name == name:
for method in hooks[(hook_fortype, fortype_name)]:
translated_dict.update(dataent.get_attr(method)())
return translated_dict
def make(self):
"""build into msg_root"""
headers = {
"Subject": strip(self.subject),
"From": self.sender,
"To": ', '.join(self.recipients) if self.expose_recipients=="header" else "<!--recipient-->",
"Date": email.utils.formatdate(),
"Reply-To": self.reply_to if self.reply_to else None,
"CC": ', '.join(self.cc) if self.cc and self.expose_recipients=="header" else None,
'X-Dataent-Site': get_url(),
}
# reset headers as values may be changed.
for key, val in iteritems(headers):
self.set_header(key, val)
# call hook to enable apps to modify msg_root before sending
for hook in dataent.get_hooks("make_email_body_message"):
dataent.get_attr(hook)(self)
def load_assets(self):
from dataent.modules import get_module_path, scrub
import os
self.script = ''
page_name = scrub(self.name)
path = os.path.join(get_module_path(self.module), 'page', page_name)
# script
fpath = os.path.join(path, page_name + '.js')
if os.path.exists(fpath):
with open(fpath, 'r') as f:
self.script = render_include(f.read())
# css
fpath = os.path.join(path, page_name + '.css')
if os.path.exists(fpath):
with open(fpath, 'r') as f:
self.style = safe_decode(f.read())
# html as js template
for fname in os.listdir(path):
if fname.endswith(".html"):
with open(os.path.join(path, fname), 'r') as f:
template = f.read()
if "<!-- jinja -->" in template:
context = dataent._dict({})
try:
out = dataent.get_attr("{app}.{module}.page.{page}.{page}.get_context".format(
app = dataent.local.module_app[scrub(self.module)],
module = scrub(self.module),
page = page_name
))(context)
if out:
context = out
except (AttributeError, ImportError):
pass
template = dataent.render_template(template, context)
self.script = html_to_js_template(fname, template) + self.script
# flag for not caching this page
self._dynamic_page = True
if dataent.lang != 'en':
from dataent.translate import get_lang_js
self.script += get_lang_js("page", self.name)
for path in get_code_files_via_hooks("page_js", self.name):
js = get_js(path)
if js:
self.script += "\n\n" + js
def _get():
config = dataent._dict()
hooks = dataent.get_hooks()
if hooks:
for notification_config in hooks.notification_config:
nc = dataent.get_attr(notification_config)()
for key in ("for_doctype", "for_module", "for_other",
"targets"):
config.setdefault(key, {})
config[key].update(nc.get(key, {}))
return config
def get_permission_query_conditions(self):
condition_methods = dataent.get_hooks("permission_query_conditions",
{}).get(self.doctype, [])
if condition_methods:
conditions = []
for method in condition_methods:
c = dataent.call(dataent.get_attr(method), self.user)
if c:
conditions.append(c)
return " and ".join(conditions) if conditions else None
def test_patch_module_names(self):
dataent.flags.final_patches = []
dataent.flags.in_install = True
for patchmodule in patch_handler.get_all_patches():
if patchmodule.startswith("execute:"):
pass
else:
if patchmodule.startswith("finally:"):
patchmodule = patchmodule.split('finally:')[-1]
self.assertTrue(
dataent.get_attr(patchmodule.split()[0] + ".execute"))
dataent.flags.in_install = False
def clear_cache(path=None):
'''Clear website caches
:param path: (optional) for the given path'''
for key in ('website_generator_routes', 'website_pages',
'website_full_index'):
dataent.cache().delete_value(key)
dataent.cache().delete_value("website_404")
if path:
dataent.cache().hdel('website_redirects', path)
delete_page_cache(path)
else:
clear_sitemap()
dataent.clear_cache("Guest")
for key in ('portal_menu_items', 'home_page', 'website_route_rules',
'doctypes_with_web_view', 'website_redirects',
'page_context', 'website_page'):
dataent.cache().delete_value(key)
for method in dataent.get_hooks("website_clear_cache"):
dataent.get_attr(method)(path)
def execute_job(site, method, event, job_name, kwargs, user=None, is_async=True, retry=0):
'''Executes job in a worker, performs commit/rollback and logs if there is any error'''
from dataent.utils.scheduler import log
if is_async:
dataent.connect(site)
if os.environ.get('CI'):
dataent.flags.in_test = True
if user:
dataent.set_user(user)
if isinstance(method, string_types):
method_name = method
method = dataent.get_attr(method)
else:
method_name = cstr(method.__name__)
try:
method(**kwargs)
except (pymysql.InternalError, dataent.RetryBackgroundJobError) as e:
dataent.db.rollback()
if (retry < 5 and
(isinstance(e, dataent.RetryBackgroundJobError) or e.args[0] in (ER.LOCK_DEADLOCK, ER.LOCK_WAIT_TIMEOUT))):
# retry the job if
# 1213 = deadlock
# 1205 = lock wait timeout
# or RetryBackgroundJobError is explicitly raised
dataent.destroy()
time.sleep(retry+1)
return execute_job(site, method, event, job_name, kwargs,
is_async=is_async, retry=retry+1)
else:
log(method_name, message=repr(locals()))
raise
except:
dataent.db.rollback()
log(method_name, message=repr(locals()))
raise
else:
dataent.db.commit()
finally:
if is_async:
dataent.destroy()
def _for_module(m):
app_name = m.split(".")[0]
try:
docs_version = dataent.get_attr(app_name +
".config.docs.docs_version")
except AttributeError:
docs_version = None
if docs_version:
return docs_version
return getattr(importlib.import_module(m.split(".")[0]), "__version__",
"0.0.0")
