def create_new_parent(self, communication, email): '''If no parent found, create a new reference document''' # no parent found, but must be tagged # insert parent type doc parent = dataent.new_doc(self.append_to) if self.subject_field: parent.set(self.subject_field, dataent.as_unicode(email.subject)[:140]) if self.sender_field: parent.set(self.sender_field, dataent.as_unicode(email.from_email)) if parent.meta.has_field("email_account"): parent.email_account = self.name parent.flags.ignore_mandatory = True try: parent.insert(ignore_permissions=True) except dataent.DuplicateEntryError: # try and find matching parent parent_name = dataent.db.get_value(self.append_to, {self.sender_field: email.from_email}) if parent_name: parent.name = parent_name else: parent = None # NOTE if parent isn't found and there's no subject match, it is likely that it is a new conversation thread and hence is_first = True communication.is_first = True return parent
def execute(): ignore_doctypes = [ "Lead", "Opportunity", "POS Profile", "Tax Rule", "Pricing Rule" ] customers = dataent.get_all('Customer', fields=["name", "customer_group"]) customer_group_fetch = get_fetch_fields('Customer', 'Customer Group', ignore_doctypes) batch_size = 1000 for i in range(0, len(customers), batch_size): batch_customers = customers[i:i + batch_size] for d in customer_group_fetch: when_then = [] for customer in batch_customers: value = dataent.db.escape( dataent.as_unicode(customer.get("customer_group"))) when_then.append(''' WHEN `%s` = "%s" and %s != "%s" THEN "%s" ''' % (d["master_fieldname"], dataent.db.escape(dataent.as_unicode( customer.name)), d["linked_to_fieldname"], value, value)) dataent.db.sql(""" update `tab%s` set %s = CASE %s ELSE `%s` END """ % (d['doctype'], d.linked_to_fieldname, " ".join(when_then), d.linked_to_fieldname))
def login(): # LDAP LOGIN LOGIC args = dataent.form_dict ldap = dataent.get_doc("LDAP Settings") user = ldap.authenticate(dataent.as_unicode(args.usr), dataent.as_unicode(args.pwd)) dataent.local.login_manager.user = user.name dataent.local.login_manager.post_login() # because of a GET request! dataent.db.commit()
def escape(self, s, percent=True): """Excape quotes and percent in given string.""" # pymysql expects unicode argument to escape_string with Python 3 s = as_unicode(pymysql.escape_string(as_unicode(s)), "utf-8").replace("`", "\\`") # NOTE separating % escape, because % escape should only be done when using LIKE operator # or when you use python format string to generate query that already has a %s # for example: sql("select name from `tabUser` where name=%s and {0}".format(conditions), something) # defaulting it to True, as this is the most frequent use case # ideally we shouldn't have to use ESCAPE and strive to pass values via the values argument of sql if percent: s = s.replace("%", "%%") return s
def get_charts_for_country(country, with_standard=False): charts = [] def _get_chart_name(content): if content: content = json.loads(content) if (content and content.get("disabled", "No") == "No") \ or dataent.local.flags.allow_unverified_charts: charts.append(content["name"]) country_code = dataent.db.get_value("Country", country, "code") if country_code: folders = ("verified", ) if dataent.local.flags.allow_unverified_charts: folders = ("verified", "unverified") for folder in folders: path = os.path.join(os.path.dirname(__file__), folder) if not os.path.exists(path): continue for fname in os.listdir(path): fname = dataent.as_unicode(fname) if (fname.startswith(country_code) or fname.startswith(country)) and fname.endswith(".json"): with open(os.path.join(path, fname), "r") as f: _get_chart_name(f.read()) # if more than one charts, returned then add the standard if len(charts) != 1 or with_standard: charts += ["Standard", "Standard with Numbers"] return charts
def make_boilerplate(template, doc, opts=None): target_path = get_doc_path(doc.module, doc.doctype, doc.name) template_name = template.replace("controller", scrub(doc.name)) if template_name.endswith('._py'): template_name = template_name[:-4] + '.py' target_file_path = os.path.join(target_path, template_name) if not doc: doc = {} app_publisher = get_app_publisher(doc.module) if not os.path.exists(target_file_path): if not opts: opts = {} with open(target_file_path, 'w') as target: with open( os.path.join(get_module_path("core"), "doctype", scrub(doc.doctype), "boilerplate", template), 'r') as source: target.write( dataent.as_unicode( dataent.utils.cstr(source.read()).format( app_publisher=app_publisher, year=dataent.utils.nowdate()[:4], classname=doc.name.replace(" ", ""), doctype=doc.name, **opts)))
def extract_messages_from_code(code, is_py=False): """Extracts translatable srings from a code file :param code: code from which translatable files are to be extracted :param is_py: include messages in triple quotes e.g. `_('''message''')`""" try: code = dataent.as_unicode(render_include(code)) except (TemplateError, ImportError, InvalidIncludePath): # Exception will occur when it encounters John Resig's microtemplating code pass messages = [] messages += [(m.start(), m.groups()[0]) for m in re.compile('_\("([^"]*)"').finditer(code)] messages += [(m.start(), m.groups()[0]) for m in re.compile("_\('([^']*)'").finditer(code)] if is_py: messages += [ (m.start(), m.groups()[0]) for m in re.compile('_\("{3}([^"]*)"{3}.*\)').finditer(code) ] messages = [(pos, message) for pos, message in messages if is_translatable(message)] return pos_to_line_no(messages, code)
def update_app_details(self): pkg_info_file = os.path.join('..', 'apps', self.app_name, '{app_name}.egg-info'.format(app_name=self.app_name), 'PKG-INFO') if os.path.isfile(pkg_info_file): app_data_path = pkg_info_file with open(app_data_path, 'r') as f: app_data = f.readlines() app_data = dataent.as_unicode(''.join(app_data)).split('\n') if '' in app_data: app_data.remove('') app_data = [x+'\n' for x in app_data] for data in app_data: if 'Version:' in data: self.version = ''.join(re.findall('Version: (.*?)\\n', data)) elif 'Summary:' in data: self.app_description = ''.join(re.findall('Summary: (.*?)\\n', data)) elif 'Author:' in data: self.app_publisher = ''.join(re.findall('Author: (.*?)\\n', data)) elif 'Author-email:' in data: self.app_email = ''.join(re.findall('Author-email: (.*?)\\n', data)) self.app_title = self.app_name self.app_title = self.app_title.replace('-', ' ') self.app_title = self.app_title.replace('_', ' ') if os.path.isdir(os.path.join('..', 'apps', self.app_name, '.git')): self.current_git_branch = safe_decode(check_output("git rev-parse --abbrev-ref HEAD".split(), cwd=os.path.join('..', 'apps', self.app_name))).strip('\n') self.is_git_repo = True else: self.current_git_branch = None self.is_git_repo = False else: dataent.throw("Hey developer, the app you're trying to create an \ instance of doesn't actually exist. You could consider setting \ developer flag to 0 to actually create the app")
def get_chart(chart_template, existing_company=None): chart = {} if existing_company: return get_account_tree_from_existing_company(existing_company) elif chart_template == "Standard": from epaas.accounts.doctype.account.chart_of_accounts.verified import standard_chart_of_accounts return standard_chart_of_accounts.get() elif chart_template == "Standard with Numbers": from epaas.accounts.doctype.account.chart_of_accounts.verified \ import standard_chart_of_accounts_with_account_number return standard_chart_of_accounts_with_account_number.get() else: folders = ("verified", ) if dataent.local.flags.allow_unverified_charts: folders = ("verified", "unverified") for folder in folders: path = os.path.join(os.path.dirname(__file__), folder) for fname in os.listdir(path): fname = dataent.as_unicode(fname) if fname.endswith(".json"): with open(os.path.join(path, fname), "r") as f: chart = f.read() if chart and json.loads(chart).get( "name") == chart_template: return json.loads(chart).get("tree")
def find_parent_based_on_subject_and_sender(self, communication, email): '''Find parent document based on subject and sender match''' parent = None if self.append_to and self.sender_field: if self.subject_field: # try and match by subject and sender # if sent by same sender with same subject, # append it to old coversation subject = dataent.as_unicode(strip(re.sub("(^\s*(fw|fwd|wg)[^:]*:|\s*(re|aw)[^:]*:\s*)*", "", email.subject, 0, flags=re.IGNORECASE))) parent = dataent.db.get_all(self.append_to, filters={ self.sender_field: email.from_email, self.subject_field: ("like", "%{0}%".format(subject)), "creation": (">", (get_datetime() - relativedelta(days=10)).strftime(DATE_FORMAT)) }, fields="name") # match only subject field # when the from_email is of a user in the system # and subject is atleast 10 chars long if not parent and len(subject) > 10 and is_system_user(email.from_email): parent = dataent.db.get_all(self.append_to, filters={ self.subject_field: ("like", "%{0}%".format(subject)), "creation": (">", (get_datetime() - relativedelta(days=10)).strftime(DATE_FORMAT)) }, fields="name") if parent: parent = dataent._dict(doctype=self.append_to, name=parent[0].name) return parent
def decode_email(self, email): if not email: return decoded = "" for part, encoding in decode_header( dataent.as_unicode(email).replace("\"", " ").replace("\'", " ")): if encoding: decoded += part.decode(encoding) else: decoded += safe_decode(part) return decoded
def validate_file(self): """Validates existence of public file TODO: validate for private file """ if (self.file_url or "").startswith("/files/"): if not self.file_name: self.file_name = self.file_url.split("/files/")[-1] if not os.path.exists( get_files_path( dataent.as_unicode(self.file_name.lstrip("/")))): dataent.throw( _("File {0} does not exist").format(self.file_url), IOError)
def get_email_seen_status(self, uid, flag_string): """ parse the email FLAGS response """ if not flag_string: return None flags = [] for flag in imaplib.ParseFlags(flag_string) or []: pattern = re.compile("\w+") match = re.search(pattern, dataent.as_unicode(flag)) flags.append(match.group(0)) if "Seen" in flags: self.seen_status.update({uid: "SEEN"}) else: self.seen_status.update({uid: "UNSEEN"})
def get_server_messages(app): """Extracts all translatable strings (tagged with :func:`dataent._`) from Python modules inside an app""" messages = [] file_extensions = ('.py', '.html', '.js', '.vue') for basepath, folders, files in os.walk(dataent.get_pymodule_path(app)): for dontwalk in (".git", "public", "locale"): if dontwalk in folders: folders.remove(dontwalk) for f in files: f = dataent.as_unicode(f) if f.endswith(file_extensions): messages.extend( get_messages_from_file(os.path.join(basepath, f))) return messages
def _sanitize_content(self): """Sanitize HTML and Email in field values. Used to prevent XSS. - Ignore if 'Ignore XSS Filter' is checked or fieldtype is 'Code' """ if dataent.flags.in_install: return for fieldname, value in self.get_valid_dict().items(): if not value or not isinstance(value, string_types): continue value = dataent.as_unicode(value) if (u"<" not in value and u">" not in value): # doesn't look like html so no need continue elif "<!-- markdown -->" in value and not ("<script" in value or "javascript:" in value): # should be handled separately via the markdown converter function continue df = self.meta.get_field(fieldname) sanitized_value = value if df and df.get("fieldtype") in ( "Data", "Code", "Small Text") and df.get("options") == "Email": sanitized_value = sanitize_email(value) elif df and ( df.get("ignore_xss_filter") or (df.get("fieldtype") == "Code" and df.get("options") != "Email") or df.get("fieldtype") in ("Attach", "Attach Image", "Barcode") # cancelled and submit but not update after submit should be ignored or self.docstatus == 2 or (self.docstatus == 1 and not df.get("allow_on_submit"))): continue else: sanitized_value = sanitize_html( value, linkify=df.fieldtype == 'Text Editor') self.set(fieldname, sanitized_value)
def get_page_info_from_template(path): '''Return page_info from path''' for app in dataent.get_installed_apps(dataent_last=True): app_path = dataent.get_app_path(app) folders = get_start_folders() for start in folders: search_path = os.path.join(app_path, start, path) options = (search_path, search_path + '.html', search_path + '.md', search_path + '/index.html', search_path + '/index.md') for o in options: option = dataent.as_unicode(o) if os.path.exists(option) and not os.path.isdir(option): return get_page_info(option, app, start, app_path=app_path) return None
def add_missing_headers(): '''Walk and add missing headers in docs (to be called from bench execute)''' path = dataent.get_app_path('epaas', 'docs') for basepath, folders, files in os.walk(path): for fname in files: if fname.endswith('.md'): with open(os.path.join(basepath, fname), 'r') as f: content = dataent.as_unicode(f.read()) if not content.startswith('# ') and not '<h1>' in content: with open(os.path.join(basepath, fname), 'w') as f: if fname == 'index.md': fname = os.path.basename(basepath) else: fname = fname[:-3] h = fname.replace('_', ' ').replace('-', ' ').title() content = '# {0}\n\n'.format(h) + content f.write(content.encode('utf-8'))
def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, parent=None): '''Returns a value form a document :param doctype: DocType to be queried :param fieldname: Field to be returned (default `name`) :param filters: dict or string for identifying the record''' if dataent.is_table(doctype): check_parent_permission(parent, doctype) if not dataent.has_permission(doctype): dataent.throw(_("No permission for {0}".format(doctype)), dataent.PermissionError) try: filters = json.loads(filters) if isinstance(filters, (integer_types, float)): filters = dataent.as_unicode(filters) except (TypeError, ValueError): # filters are not passesd, not json pass try: fieldname = json.loads(fieldname) except (TypeError, ValueError): # name passed, not json pass # check whether the used filters were really parseable and usable # and did not just result in an empty string or dict if not filters: filters = None return dataent.db.get_value(doctype, filters, fieldname, as_dict=as_dict, debug=debug)
def has_gravatar(email): '''Returns gravatar url if user has set an avatar at gravatar.com''' if (dataent.flags.in_import or dataent.flags.in_install or dataent.flags.in_test): # no gravatar if via upload # since querying gravatar for every item will be slow return '' hexdigest = hashlib.md5( dataent.as_unicode(email).encode('utf-8')).hexdigest() gravatar_url = "https://secure.gravatar.com/avatar/{hash}?d=404&s=200".format( hash=hexdigest) try: res = requests.get(gravatar_url) if res.status_code == 200: return gravatar_url else: return '' except requests.exceptions.ConnectionError: return ''
def make_boilerplate(dest, app_name): if not os.path.exists(dest): print("Destination directory does not exist") return # app_name should be in snake_case app_name = dataent.scrub(app_name) hooks = dataent._dict() hooks.app_name = app_name app_title = hooks.app_name.replace("_", " ").title() for key in ("App Title (default: {0})".format(app_title), "App Description", "App Publisher", "App Email", "App Icon (default 'octicon octicon-file-directory')", "App Color (default 'grey')", "App License (default 'MIT')"): hook_key = key.split(" (")[0].lower().replace(" ", "_") hook_val = None while not hook_val: hook_val = cstr(input(key + ": ")) if not hook_val: defaults = { "app_title": app_title, "app_icon": "octicon octicon-file-directory", "app_color": "grey", "app_license": "MIT" } if hook_key in defaults: hook_val = defaults[hook_key] if hook_key == "app_name" and hook_val.lower().replace( " ", "_") != hook_val: print("App Name must be all lowercase and without spaces") hook_val = "" elif hook_key == "app_title" and not re.match( "^(?![\W])[^\d_\s][\w -]+$", hook_val, re.UNICODE): print( "App Title should start with a letter and it can only consist of letters, numbers, spaces and underscores" ) hook_val = "" hooks[hook_key] = hook_val dataent.create_folder(os.path.join(dest, hooks.app_name, hooks.app_name, dataent.scrub(hooks.app_title)), with_init=True) dataent.create_folder(os.path.join(dest, hooks.app_name, hooks.app_name, "templates"), with_init=True) dataent.create_folder( os.path.join(dest, hooks.app_name, hooks.app_name, "www")) dataent.create_folder(os.path.join(dest, hooks.app_name, hooks.app_name, "templates", "pages"), with_init=True) dataent.create_folder( os.path.join(dest, hooks.app_name, hooks.app_name, "templates", "includes")) dataent.create_folder(os.path.join(dest, hooks.app_name, hooks.app_name, "config"), with_init=True) dataent.create_folder( os.path.join(dest, hooks.app_name, hooks.app_name, "public", "css")) dataent.create_folder( os.path.join(dest, hooks.app_name, hooks.app_name, "public", "js")) with open( os.path.join(dest, hooks.app_name, hooks.app_name, "__init__.py"), "w") as f: f.write(dataent.as_unicode(init_template)) with open(os.path.join(dest, hooks.app_name, "MANIFEST.in"), "w") as f: f.write(dataent.as_unicode(manifest_template.format(**hooks))) with open(os.path.join(dest, hooks.app_name, ".gitignore"), "w") as f: f.write( dataent.as_unicode( gitignore_template.format(app_name=hooks.app_name))) with open(os.path.join(dest, hooks.app_name, "setup.py"), "w") as f: f.write(dataent.as_unicode(setup_template.format(**hooks))) with open(os.path.join(dest, hooks.app_name, "requirements.txt"), "w") as f: f.write("dataent") with open(os.path.join(dest, hooks.app_name, "README.md"), "w") as f: f.write( dataent.as_unicode("## {0}\n\n{1}\n\n#### License\n\n{2}".format( hooks.app_title, hooks.app_description, hooks.app_license))) with open(os.path.join(dest, hooks.app_name, "license.txt"), "w") as f: f.write(dataent.as_unicode("License: " + hooks.app_license)) with open( os.path.join(dest, hooks.app_name, hooks.app_name, "modules.txt"), "w") as f: f.write(dataent.as_unicode(hooks.app_title)) with open(os.path.join(dest, hooks.app_name, hooks.app_name, "hooks.py"), "w") as f: f.write(dataent.as_unicode(hooks_template.format(**hooks))) touch_file( os.path.join(dest, hooks.app_name, hooks.app_name, "patches.txt")) with open( os.path.join(dest, hooks.app_name, hooks.app_name, "config", "desktop.py"), "w") as f: f.write(dataent.as_unicode(desktop_template.format(**hooks))) with open( os.path.join(dest, hooks.app_name, hooks.app_name, "config", "docs.py"), "w") as f: f.write(dataent.as_unicode(docs_template.format(**hooks))) print("'{app}' created at {path}".format(app=app_name, path=os.path.join(dest, app_name)))
def clean_script_and_style(html): # remove script and style soup = BeautifulSoup(html, 'html5lib') for s in soup(['script', 'style']): s.decompose() return dataent.as_unicode(soup)
def cstr(s, encoding='utf-8'): return dataent.as_unicode(s, encoding)
def add_breadcrumbs_tag(path): with open(path, 'r') as f: content = dataent.as_unicode(f.read()) with open(path, 'wb') as f: f.write(('<!-- add-breadcrumbs -->\n' + content).encode('utf-8'))