def create_new_parent(self, communication, email):
'''If no parent found, create a new reference document'''
# no parent found, but must be tagged
# insert parent type doc
parent = dataent.new_doc(self.append_to)
if self.subject_field:
parent.set(self.subject_field, dataent.as_unicode(email.subject)[:140])
if self.sender_field:
parent.set(self.sender_field, dataent.as_unicode(email.from_email))
if parent.meta.has_field("email_account"):
parent.email_account = self.name
parent.flags.ignore_mandatory = True
try:
parent.insert(ignore_permissions=True)
except dataent.DuplicateEntryError:
# try and find matching parent
parent_name = dataent.db.get_value(self.append_to, {self.sender_field: email.from_email})
if parent_name:
parent.name = parent_name
else:
parent = None
# NOTE if parent isn't found and there's no subject match, it is likely that it is a new conversation thread and hence is_first = True
communication.is_first = True
return parent
def execute():
ignore_doctypes = [
"Lead", "Opportunity", "POS Profile", "Tax Rule", "Pricing Rule"
]
customers = dataent.get_all('Customer', fields=["name", "customer_group"])
customer_group_fetch = get_fetch_fields('Customer', 'Customer Group',
ignore_doctypes)
batch_size = 1000
for i in range(0, len(customers), batch_size):
batch_customers = customers[i:i + batch_size]
for d in customer_group_fetch:
when_then = []
for customer in batch_customers:
value = dataent.db.escape(
dataent.as_unicode(customer.get("customer_group")))
when_then.append('''
WHEN `%s` = "%s" and %s != "%s"
THEN "%s"
''' % (d["master_fieldname"],
dataent.db.escape(dataent.as_unicode(
customer.name)), d["linked_to_fieldname"], value, value))
dataent.db.sql("""
update
`tab%s`
set
%s = CASE %s ELSE `%s` END
""" % (d['doctype'], d.linked_to_fieldname, " ".join(when_then),
d.linked_to_fieldname))
def login():
# LDAP LOGIN LOGIC
args = dataent.form_dict
ldap = dataent.get_doc("LDAP Settings")
user = ldap.authenticate(dataent.as_unicode(args.usr), dataent.as_unicode(args.pwd))
dataent.local.login_manager.user = user.name
dataent.local.login_manager.post_login()
# because of a GET request!
dataent.db.commit()
def escape(self, s, percent=True):
"""Excape quotes and percent in given string."""
# pymysql expects unicode argument to escape_string with Python 3
s = as_unicode(pymysql.escape_string(as_unicode(s)),
"utf-8").replace("`", "\\`")
# NOTE separating % escape, because % escape should only be done when using LIKE operator
# or when you use python format string to generate query that already has a %s
# for example: sql("select name from `tabUser` where name=%s and {0}".format(conditions), something)
# defaulting it to True, as this is the most frequent use case
# ideally we shouldn't have to use ESCAPE and strive to pass values via the values argument of sql
if percent:
s = s.replace("%", "%%")
return s
def get_charts_for_country(country, with_standard=False):
charts = []
def _get_chart_name(content):
if content:
content = json.loads(content)
if (content and content.get("disabled", "No") == "No") \
or dataent.local.flags.allow_unverified_charts:
charts.append(content["name"])
country_code = dataent.db.get_value("Country", country, "code")
if country_code:
folders = ("verified", )
if dataent.local.flags.allow_unverified_charts:
folders = ("verified", "unverified")
for folder in folders:
path = os.path.join(os.path.dirname(__file__), folder)
if not os.path.exists(path):
continue
for fname in os.listdir(path):
fname = dataent.as_unicode(fname)
if (fname.startswith(country_code) or
fname.startswith(country)) and fname.endswith(".json"):
with open(os.path.join(path, fname), "r") as f:
_get_chart_name(f.read())
# if more than one charts, returned then add the standard
if len(charts) != 1 or with_standard:
charts += ["Standard", "Standard with Numbers"]
return charts
def make_boilerplate(template, doc, opts=None):
target_path = get_doc_path(doc.module, doc.doctype, doc.name)
template_name = template.replace("controller", scrub(doc.name))
if template_name.endswith('._py'):
template_name = template_name[:-4] + '.py'
target_file_path = os.path.join(target_path, template_name)
if not doc: doc = {}
app_publisher = get_app_publisher(doc.module)
if not os.path.exists(target_file_path):
if not opts:
opts = {}
with open(target_file_path, 'w') as target:
with open(
os.path.join(get_module_path("core"), "doctype",
scrub(doc.doctype), "boilerplate", template),
'r') as source:
target.write(
dataent.as_unicode(
dataent.utils.cstr(source.read()).format(
app_publisher=app_publisher,
year=dataent.utils.nowdate()[:4],
classname=doc.name.replace(" ", ""),
doctype=doc.name,
**opts)))
def extract_messages_from_code(code, is_py=False):
"""Extracts translatable srings from a code file
:param code: code from which translatable files are to be extracted
:param is_py: include messages in triple quotes e.g. `_('''message''')`"""
try:
code = dataent.as_unicode(render_include(code))
except (TemplateError, ImportError, InvalidIncludePath):
# Exception will occur when it encounters John Resig's microtemplating code
pass
messages = []
messages += [(m.start(), m.groups()[0])
for m in re.compile('_\("([^"]*)"').finditer(code)]
messages += [(m.start(), m.groups()[0])
for m in re.compile("_\('([^']*)'").finditer(code)]
if is_py:
messages += [
(m.start(), m.groups()[0])
for m in re.compile('_\("{3}([^"]*)"{3}.*\)').finditer(code)
]
messages = [(pos, message) for pos, message in messages
if is_translatable(message)]
return pos_to_line_no(messages, code)
def update_app_details(self):
pkg_info_file = os.path.join('..', 'apps', self.app_name, '{app_name}.egg-info'.format(app_name=self.app_name), 'PKG-INFO')
if os.path.isfile(pkg_info_file):
app_data_path = pkg_info_file
with open(app_data_path, 'r') as f:
app_data = f.readlines()
app_data = dataent.as_unicode(''.join(app_data)).split('\n')
if '' in app_data:
app_data.remove('')
app_data = [x+'\n' for x in app_data]
for data in app_data:
if 'Version:' in data:
self.version = ''.join(re.findall('Version: (.*?)\\n', data))
elif 'Summary:' in data:
self.app_description = ''.join(re.findall('Summary: (.*?)\\n', data))
elif 'Author:' in data:
self.app_publisher = ''.join(re.findall('Author: (.*?)\\n', data))
elif 'Author-email:' in data:
self.app_email = ''.join(re.findall('Author-email: (.*?)\\n', data))
self.app_title = self.app_name
self.app_title = self.app_title.replace('-', ' ')
self.app_title = self.app_title.replace('_', ' ')
if os.path.isdir(os.path.join('..', 'apps', self.app_name, '.git')):
self.current_git_branch = safe_decode(check_output("git rev-parse --abbrev-ref HEAD".split(),
cwd=os.path.join('..', 'apps', self.app_name))).strip('\n')
self.is_git_repo = True
else:
self.current_git_branch = None
self.is_git_repo = False
else:
dataent.throw("Hey developer, the app you're trying to create an \
instance of doesn't actually exist. You could consider setting \
developer flag to 0 to actually create the app")
def get_chart(chart_template, existing_company=None):
chart = {}
if existing_company:
return get_account_tree_from_existing_company(existing_company)
elif chart_template == "Standard":
from epaas.accounts.doctype.account.chart_of_accounts.verified import standard_chart_of_accounts
return standard_chart_of_accounts.get()
elif chart_template == "Standard with Numbers":
from epaas.accounts.doctype.account.chart_of_accounts.verified \
import standard_chart_of_accounts_with_account_number
return standard_chart_of_accounts_with_account_number.get()
else:
folders = ("verified", )
if dataent.local.flags.allow_unverified_charts:
folders = ("verified", "unverified")
for folder in folders:
path = os.path.join(os.path.dirname(__file__), folder)
for fname in os.listdir(path):
fname = dataent.as_unicode(fname)
if fname.endswith(".json"):
with open(os.path.join(path, fname), "r") as f:
chart = f.read()
if chart and json.loads(chart).get(
"name") == chart_template:
return json.loads(chart).get("tree")
def find_parent_based_on_subject_and_sender(self, communication, email):
'''Find parent document based on subject and sender match'''
parent = None
if self.append_to and self.sender_field:
if self.subject_field:
# try and match by subject and sender
# if sent by same sender with same subject,
# append it to old coversation
subject = dataent.as_unicode(strip(re.sub("(^\s*(fw|fwd|wg)[^:]*:|\s*(re|aw)[^:]*:\s*)*",
"", email.subject, 0, flags=re.IGNORECASE)))
parent = dataent.db.get_all(self.append_to, filters={
self.sender_field: email.from_email,
self.subject_field: ("like", "%{0}%".format(subject)),
"creation": (">", (get_datetime() - relativedelta(days=10)).strftime(DATE_FORMAT))
}, fields="name")
# match only subject field
# when the from_email is of a user in the system
# and subject is atleast 10 chars long
if not parent and len(subject) > 10 and is_system_user(email.from_email):
parent = dataent.db.get_all(self.append_to, filters={
self.subject_field: ("like", "%{0}%".format(subject)),
"creation": (">", (get_datetime() - relativedelta(days=10)).strftime(DATE_FORMAT))
}, fields="name")
if parent:
parent = dataent._dict(doctype=self.append_to, name=parent[0].name)
return parent
def decode_email(self, email):
if not email: return
decoded = ""
for part, encoding in decode_header(
dataent.as_unicode(email).replace("\"",
" ").replace("\'", " ")):
if encoding:
decoded += part.decode(encoding)
else:
decoded += safe_decode(part)
return decoded
def validate_file(self):
"""Validates existence of public file
TODO: validate for private file
"""
if (self.file_url or "").startswith("/files/"):
if not self.file_name:
self.file_name = self.file_url.split("/files/")[-1]
if not os.path.exists(
get_files_path(
dataent.as_unicode(self.file_name.lstrip("/")))):
dataent.throw(
_("File {0} does not exist").format(self.file_url),
IOError)
def get_email_seen_status(self, uid, flag_string):
""" parse the email FLAGS response """
if not flag_string:
return None
flags = []
for flag in imaplib.ParseFlags(flag_string) or []:
pattern = re.compile("\w+")
match = re.search(pattern, dataent.as_unicode(flag))
flags.append(match.group(0))
if "Seen" in flags:
self.seen_status.update({uid: "SEEN"})
else:
self.seen_status.update({uid: "UNSEEN"})
def get_server_messages(app):
"""Extracts all translatable strings (tagged with :func:`dataent._`) from Python modules
inside an app"""
messages = []
file_extensions = ('.py', '.html', '.js', '.vue')
for basepath, folders, files in os.walk(dataent.get_pymodule_path(app)):
for dontwalk in (".git", "public", "locale"):
if dontwalk in folders: folders.remove(dontwalk)
for f in files:
f = dataent.as_unicode(f)
if f.endswith(file_extensions):
messages.extend(
get_messages_from_file(os.path.join(basepath, f)))
return messages
def _sanitize_content(self):
"""Sanitize HTML and Email in field values. Used to prevent XSS.
- Ignore if 'Ignore XSS Filter' is checked or fieldtype is 'Code'
"""
if dataent.flags.in_install:
return
for fieldname, value in self.get_valid_dict().items():
if not value or not isinstance(value, string_types):
continue
value = dataent.as_unicode(value)
if (u"<" not in value and u">" not in value):
# doesn't look like html so no need
continue
elif "<!-- markdown -->" in value and not ("<script" in value or
"javascript:" in value):
# should be handled separately via the markdown converter function
continue
df = self.meta.get_field(fieldname)
sanitized_value = value
if df and df.get("fieldtype") in (
"Data", "Code",
"Small Text") and df.get("options") == "Email":
sanitized_value = sanitize_email(value)
elif df and (
df.get("ignore_xss_filter") or
(df.get("fieldtype") == "Code"
and df.get("options") != "Email") or df.get("fieldtype")
in ("Attach", "Attach Image", "Barcode")
# cancelled and submit but not update after submit should be ignored
or self.docstatus == 2 or
(self.docstatus == 1 and not df.get("allow_on_submit"))):
continue
else:
sanitized_value = sanitize_html(
value, linkify=df.fieldtype == 'Text Editor')
self.set(fieldname, sanitized_value)
def get_page_info_from_template(path):
'''Return page_info from path'''
for app in dataent.get_installed_apps(dataent_last=True):
app_path = dataent.get_app_path(app)
folders = get_start_folders()
for start in folders:
search_path = os.path.join(app_path, start, path)
options = (search_path, search_path + '.html', search_path + '.md',
search_path + '/index.html', search_path + '/index.md')
for o in options:
option = dataent.as_unicode(o)
if os.path.exists(option) and not os.path.isdir(option):
return get_page_info(option, app, start, app_path=app_path)
return None
def add_missing_headers():
'''Walk and add missing headers in docs (to be called from bench execute)'''
path = dataent.get_app_path('epaas', 'docs')
for basepath, folders, files in os.walk(path):
for fname in files:
if fname.endswith('.md'):
with open(os.path.join(basepath, fname), 'r') as f:
content = dataent.as_unicode(f.read())
if not content.startswith('# ') and not '<h1>' in content:
with open(os.path.join(basepath, fname), 'w') as f:
if fname == 'index.md':
fname = os.path.basename(basepath)
else:
fname = fname[:-3]
h = fname.replace('_', ' ').replace('-', ' ').title()
content = '# {0}\n\n'.format(h) + content
f.write(content.encode('utf-8'))
def get_value(doctype,
fieldname,
filters=None,
as_dict=True,
debug=False,
parent=None):
'''Returns a value form a document
:param doctype: DocType to be queried
:param fieldname: Field to be returned (default `name`)
:param filters: dict or string for identifying the record'''
if dataent.is_table(doctype):
check_parent_permission(parent, doctype)
if not dataent.has_permission(doctype):
dataent.throw(_("No permission for {0}".format(doctype)),
dataent.PermissionError)
try:
filters = json.loads(filters)
if isinstance(filters, (integer_types, float)):
filters = dataent.as_unicode(filters)
except (TypeError, ValueError):
# filters are not passesd, not json
pass
try:
fieldname = json.loads(fieldname)
except (TypeError, ValueError):
# name passed, not json
pass
# check whether the used filters were really parseable and usable
# and did not just result in an empty string or dict
if not filters:
filters = None
return dataent.db.get_value(doctype,
filters,
fieldname,
as_dict=as_dict,
debug=debug)
def has_gravatar(email):
'''Returns gravatar url if user has set an avatar at gravatar.com'''
if (dataent.flags.in_import or dataent.flags.in_install
or dataent.flags.in_test):
# no gravatar if via upload
# since querying gravatar for every item will be slow
return ''
hexdigest = hashlib.md5(
dataent.as_unicode(email).encode('utf-8')).hexdigest()
gravatar_url = "https://secure.gravatar.com/avatar/{hash}?d=404&s=200".format(
hash=hexdigest)
try:
res = requests.get(gravatar_url)
if res.status_code == 200:
return gravatar_url
else:
return ''
except requests.exceptions.ConnectionError:
return ''
def make_boilerplate(dest, app_name):
if not os.path.exists(dest):
print("Destination directory does not exist")
return
# app_name should be in snake_case
app_name = dataent.scrub(app_name)
hooks = dataent._dict()
hooks.app_name = app_name
app_title = hooks.app_name.replace("_", " ").title()
for key in ("App Title (default: {0})".format(app_title),
"App Description", "App Publisher", "App Email",
"App Icon (default 'octicon octicon-file-directory')",
"App Color (default 'grey')", "App License (default 'MIT')"):
hook_key = key.split(" (")[0].lower().replace(" ", "_")
hook_val = None
while not hook_val:
hook_val = cstr(input(key + ": "))
if not hook_val:
defaults = {
"app_title": app_title,
"app_icon": "octicon octicon-file-directory",
"app_color": "grey",
"app_license": "MIT"
}
if hook_key in defaults:
hook_val = defaults[hook_key]
if hook_key == "app_name" and hook_val.lower().replace(
" ", "_") != hook_val:
print("App Name must be all lowercase and without spaces")
hook_val = ""
elif hook_key == "app_title" and not re.match(
"^(?![\W])[^\d_\s][\w -]+$", hook_val, re.UNICODE):
print(
"App Title should start with a letter and it can only consist of letters, numbers, spaces and underscores"
)
hook_val = ""
hooks[hook_key] = hook_val
dataent.create_folder(os.path.join(dest, hooks.app_name, hooks.app_name,
dataent.scrub(hooks.app_title)),
with_init=True)
dataent.create_folder(os.path.join(dest, hooks.app_name, hooks.app_name,
"templates"),
with_init=True)
dataent.create_folder(
os.path.join(dest, hooks.app_name, hooks.app_name, "www"))
dataent.create_folder(os.path.join(dest, hooks.app_name, hooks.app_name,
"templates", "pages"),
with_init=True)
dataent.create_folder(
os.path.join(dest, hooks.app_name, hooks.app_name, "templates",
"includes"))
dataent.create_folder(os.path.join(dest, hooks.app_name, hooks.app_name,
"config"),
with_init=True)
dataent.create_folder(
os.path.join(dest, hooks.app_name, hooks.app_name, "public", "css"))
dataent.create_folder(
os.path.join(dest, hooks.app_name, hooks.app_name, "public", "js"))
with open(
os.path.join(dest, hooks.app_name, hooks.app_name, "__init__.py"),
"w") as f:
f.write(dataent.as_unicode(init_template))
with open(os.path.join(dest, hooks.app_name, "MANIFEST.in"), "w") as f:
f.write(dataent.as_unicode(manifest_template.format(**hooks)))
with open(os.path.join(dest, hooks.app_name, ".gitignore"), "w") as f:
f.write(
dataent.as_unicode(
gitignore_template.format(app_name=hooks.app_name)))
with open(os.path.join(dest, hooks.app_name, "setup.py"), "w") as f:
f.write(dataent.as_unicode(setup_template.format(**hooks)))
with open(os.path.join(dest, hooks.app_name, "requirements.txt"),
"w") as f:
f.write("dataent")
with open(os.path.join(dest, hooks.app_name, "README.md"), "w") as f:
f.write(
dataent.as_unicode("## {0}\n\n{1}\n\n#### License\n\n{2}".format(
hooks.app_title, hooks.app_description, hooks.app_license)))
with open(os.path.join(dest, hooks.app_name, "license.txt"), "w") as f:
f.write(dataent.as_unicode("License: " + hooks.app_license))
with open(
os.path.join(dest, hooks.app_name, hooks.app_name, "modules.txt"),
"w") as f:
f.write(dataent.as_unicode(hooks.app_title))
with open(os.path.join(dest, hooks.app_name, hooks.app_name, "hooks.py"),
"w") as f:
f.write(dataent.as_unicode(hooks_template.format(**hooks)))
touch_file(
os.path.join(dest, hooks.app_name, hooks.app_name, "patches.txt"))
with open(
os.path.join(dest, hooks.app_name, hooks.app_name, "config",
"desktop.py"), "w") as f:
f.write(dataent.as_unicode(desktop_template.format(**hooks)))
with open(
os.path.join(dest, hooks.app_name, hooks.app_name, "config",
"docs.py"), "w") as f:
f.write(dataent.as_unicode(docs_template.format(**hooks)))
print("'{app}' created at {path}".format(app=app_name,
path=os.path.join(dest,
app_name)))
def clean_script_and_style(html):
# remove script and style
soup = BeautifulSoup(html, 'html5lib')
for s in soup(['script', 'style']):
s.decompose()
return dataent.as_unicode(soup)
def cstr(s, encoding='utf-8'):
return dataent.as_unicode(s, encoding)
def add_breadcrumbs_tag(path):
with open(path, 'r') as f:
content = dataent.as_unicode(f.read())
with open(path, 'wb') as f:
f.write(('<!-- add-breadcrumbs -->\n' + content).encode('utf-8'))
