def run_all_tests(app=None, verbose=False, profile=False): import os apps = [app] if app else frappe.get_installed_apps() test_suite = unittest.TestSuite() for app in apps: for path, folders, files in os.walk(frappe.get_pymodule_path(app)): for dontwalk in ('locals', '.git', 'public'): if dontwalk in folders: folders.remove(dontwalk) # print path for filename in files: filename = cstr(filename) if filename.startswith("test_") and filename.endswith(".py"): # print filename[:-3] _add_test(app, path, filename, verbose, test_suite=test_suite) if profile: pr = cProfile.Profile() pr.enable() out = unittest.TextTestRunner(verbosity=1+(verbose and 1 or 0)).run(test_suite) if profile: pr.disable() s = StringIO.StringIO() ps = pstats.Stats(pr, stream=s).sort_stats('cumulative') ps.print_stats() print s.getvalue() return out
def _add_test(app, path, filename, verbose, test_suite=None): import os, imp if os.path.sep.join(["doctype", "doctype", "boilerplate"]) in path: # in /doctype/doctype/boilerplate/ return app_path = frappe.get_pymodule_path(app) relative_path = os.path.relpath(path, app_path) if relative_path=='.': module_name = app else: module_name = '{app}.{relative_path}.{module_name}'.format(app=app, relative_path=relative_path.replace('/', '.'), module_name=filename[:-3]) module = frappe.get_module(module_name) if getattr(module, "selenium_tests", False) and not frappe.conf.run_selenium_tests: return if not test_suite: test_suite = unittest.TestSuite() if os.path.basename(os.path.dirname(path))=="doctype": txt_file = os.path.join(path, filename[5:].replace(".py", ".json")) with open(txt_file, 'r') as f: doc = json.loads(f.read()) doctype = doc["name"] make_test_records(doctype, verbose) test_suite.addTest(unittest.TestLoader().loadTestsFromModule(module))
def get_build_maps(): """get all build.jsons with absolute paths""" # framework js and css files build_maps = {} for app_path in app_paths: path = os.path.join(app_path, 'public', 'build.json') if os.path.exists(path): with open(path) as f: try: for target, sources in iteritems(json.loads(f.read())): # update app path source_paths = [] for source in sources: if isinstance(source, list): s = frappe.get_pymodule_path(source[0], *source[1].split("/")) else: s = os.path.join(app_path, source) source_paths.append(s) build_maps[target] = source_paths except ValueError as e: print(path) print('JSON syntax error {0}'.format(str(e))) return build_maps
def get_build_maps(): """get all build.jsons with absolute paths""" # framework js and css files pymodules = [frappe.get_module(app) for app in frappe.get_all_apps(True)] app_paths = [os.path.dirname(pymodule.__file__) for pymodule in pymodules] build_maps = {} for app_path in app_paths: path = os.path.join(app_path, 'public', 'build.json') if os.path.exists(path): with open(path) as f: try: for target, sources in json.loads(f.read()).iteritems(): # update app path source_paths = [] for source in sources: if isinstance(source, list): s = frappe.get_pymodule_path(source[0], *source[1].split("/")) else: s = os.path.join(app_path, source) source_paths.append(s) build_maps[target] = source_paths except Exception, e: print path raise
def load_lang(lang, apps=None): """Combine all translations from `.csv` files in all `apps`""" out = {} for app in (apps or frappe.get_all_apps(True)): path = os.path.join(frappe.get_pymodule_path(app), "translations", lang + ".csv") out.update(get_translation_dict_from_file(path, lang, app)) return out
def load_lang(lang, apps=None): out = {} for app in (apps or frappe.get_all_apps(True)): path = os.path.join(frappe.get_pymodule_path(app), "translations", lang + ".csv") if os.path.exists(path): cleaned = dict([item for item in dict(read_csv_file(path)).iteritems() if item[1]]) out.update(cleaned) return out
def get_all_patches(): patches = [] for app in frappe.get_installed_apps(): # 3-to-4 fix if app == "webnotes": app = "frappe" patches.extend(frappe.get_file_items(frappe.get_pymodule_path(app, "patches.txt"))) return patches
def set_all_patches_as_completed(app): patch_path = os.path.join(frappe.get_pymodule_path(app), "patches.txt") if os.path.exists(patch_path): for patch in frappe.get_file_items(patch_path): frappe.get_doc({ "doctype": "Patch Log", "patch": patch }).insert(ignore_permissions=True) frappe.db.commit()
def get_server_messages(app): messages = [] for basepath, folders, files in os.walk(frappe.get_pymodule_path(app)): for dontwalk in (".git", "public", "locale"): if dontwalk in folders: folders.remove(dontwalk) for f in files: if f.endswith(".py") or f.endswith(".html") or f.endswith(".js"): messages.extend(get_messages_from_file(os.path.join(basepath, f))) return clean(messages)
def write_translations_file(app, lang, full_dict=None, app_messages=None): if not app_messages: app_messages = get_messages_for_app(app) if not app_messages: return tpath = frappe.get_pymodule_path(app, "translations") frappe.create_folder(tpath) write_csv_file(os.path.join(tpath, lang + ".csv"), app_messages, full_dict or get_full_dict(lang))
def get_server_messages(app): """Extracts all translatable strings (tagged with :func:`frappe._`) from Python modules inside an app""" messages = [] for basepath, folders, files in os.walk(frappe.get_pymodule_path(app)): for dontwalk in (".git", "public", "locale"): if dontwalk in folders: folders.remove(dontwalk) for f in files: if f.endswith(".py") or f.endswith(".html") or f.endswith(".js"): messages.extend(get_messages_from_file(os.path.join(basepath, f))) return messages
def load_lang(lang, apps=None): out = {} for app in (apps or frappe.get_all_apps(True)): path = os.path.join(frappe.get_pymodule_path(app), "translations", lang + ".csv") if os.path.exists(path): cleaned = dict([ item for item in dict(read_csv_file(path)).iteritems() if item[1] ]) out.update(cleaned) return out
def get_all_patches(): patches = [] for app in frappe.get_installed_apps(): if app == "shopping_cart": continue # 3-to-4 fix if app == "webnotes": app = "frappe" patches.extend( frappe.get_file_items(frappe.get_pymodule_path(app, "patches.txt"))) return patches
def run_all_tests(app=None, verbose=False, profile=False, ui_tests=False, failfast=False, junit_xml_output=False): import os apps = [app] if app else frappe.get_installed_apps() test_suite = unittest.TestSuite() for app in apps: for path, folders, files in os.walk(frappe.get_pymodule_path(app)): for dontwalk in ('locals', '.git', 'public', '__pycache__'): if dontwalk in folders: folders.remove(dontwalk) # for predictability folders.sort() files.sort() # print path for filename in files: if filename.startswith("test_") and filename.endswith(".py")\ and filename != 'test_runner.py': # print filename[:-3] _add_test(app, path, filename, verbose, test_suite, ui_tests) if junit_xml_output: runner = unittest_runner(verbosity=1 + (verbose and 1 or 0), failfast=failfast) else: runner = unittest_runner(resultclass=TimeLoggingTestResult, verbosity=1 + (verbose and 1 or 0), failfast=failfast) if profile: pr = cProfile.Profile() pr.enable() out = runner.run(test_suite) if profile: pr.disable() s = StringIO() ps = pstats.Stats(pr, stream=s).sort_stats('cumulative') ps.print_stats() print(s.getvalue()) return out
def load_lang(lang, apps=None): """Combine all translations from `.csv` files in all `apps`""" out = {} for app in (apps or frappe.get_all_apps(True)): path = os.path.join(frappe.get_pymodule_path(app), "translations", lang + ".csv") if os.path.exists(path): csv_content = read_csv_file(path) try: # with file and line numbers cleaned = dict([(item[1], item[2]) for item in csv_content if item[2]]) except IndexError: cleaned = dict([(item[0], item[1]) for item in csv_content if item[1]]) out.update(cleaned) return out
def get_server_messages(app): """Extracts all translatable strings (tagged with `_(`) from Python modules inside an app""" messages = [] file_extensions = (".py", ".html", ".js", ".vue", ".md") for basepath, folders, files in os.walk(frappe.get_pymodule_path(app)): for dontwalk in (".git", "public", "locale", "__pycache__", "translations"): if dontwalk in folders: folders.remove(dontwalk) for f in files: f = frappe.as_unicode(f) if f.endswith(file_extensions): messages.extend( get_messages_from_file(os.path.join(basepath, f))) return messages
def egd_load_lang(lang, apps=None): """Checks `en` too""" if is_app_for_actual_site(): import os from frappe.translate import get_translation_dict_from_file out = frappe.cache().hget("lang_full_dict", lang, shared=True) if not out: out = {} for app in (apps or frappe.get_all_apps(True)): path = os.path.join(frappe.get_pymodule_path(app), "translations", lang + ".csv") out.update(get_translation_dict_from_file(path, lang, app) or {}) frappe.cache().hset("lang_full_dict", lang, out, shared=True) return out or {} else: return frappe_load_lang(lang, apps)
def get_context(context): site_app = (frappe.get_hooks("multilanguage_app_site_name")[0] if frappe.get_hooks("multilanguage_app_site_name") else "") if not site_app: return context import os import io # from frappe.translate import get_messages_for_app from frappe.translate import deduplicate_messages from frappe.translate import get_all_messages_from_js_files from frappe.translate import get_server_messages from frappe.translate import get_translation_dict_from_file from csv import writer messages = [] # # full app messages # messages.extend(get_messages_for_app(site_app)) # app_include_files messages.extend(get_all_messages_from_js_files(site_app)) # server_messages messages.extend(get_server_messages(site_app)) messages = deduplicate_messages(messages) ctx_lang = frappe._dict({ "translated": frappe._dict(), "segments_as_txt": frappe._dict(), }) languages = frappe.get_hooks("translated_languages_for_website") for lang in languages: path = os.path.join(frappe.get_pymodule_path(site_app), "translations", lang + ".csv") data = get_translation_dict_from_file(path, lang, site_app) ctx_lang["translated"][lang] = data mem_file = io.StringIO() w = writer(mem_file, lineterminator="\n") for p, m in messages: translated = ctx_lang["translated"][lang][m] if m in ctx_lang[ "translated"][lang] else m w.writerow([p if p else '', m, translated]) ctx_lang["segments_as_txt"][lang] = mem_file.getvalue() context["languages_data"] = ctx_lang
def write_translations_file(app, lang, app_messages=None): """Write a translation file for a given language. :param app: `app` for which translations are to be written. :param lang: Language code. :param app_messages: Source strings (optional). """ if not app_messages: app_messages = get_messages_for_app(app) if not app_messages: return tpath = frappe.get_pymodule_path(app, "translations") frappe.create_folder(tpath) write_json_file(os.path.join(tpath, lang + ".json"), app_messages)
def write_translations_file(app, lang, full_dict=None, app_messages=None): """Write a translation file for a given language. :param app: `app` for which translations are to be written. :param lang: Language code. :param full_dict: Full translated language dict (optional). :param app_messages: Source strings (optional). """ if not app_messages: app_messages = get_messages_for_app(app) if not app_messages: return tpath = frappe.get_pymodule_path(app, "translations") frappe.create_folder(tpath) write_csv_file(os.path.join(tpath, lang + ".csv"), app_messages, full_dict or get_full_dict(lang))
def dfp_load_lang(lang, apps=None): """Checks `en` too""" from .hooks import multilanguage_app_site_hosts as multilanguage_site from .hooks import translated_languages_for_website as languages if frappe.local.site in multilanguage_site and languages: import os from frappe.translate import get_translation_dict_from_file out = frappe.cache().hget("lang_full_dict", lang, shared=True) if not out: out = {} for app in (apps or frappe.get_all_apps(True)): path = os.path.join(frappe.get_pymodule_path(app), "translations", lang + ".csv") out.update( get_translation_dict_from_file(path, lang, app) or {}) frappe.cache().hset("lang_full_dict", lang, out, shared=True) return out or {} return frappe_load_lang(lang, apps)
def run_all_tests(app=None, verbose=False): import os apps = [app] if app else frappe.get_installed_apps() test_suite = unittest.TestSuite() for app in apps: for path, folders, files in os.walk(frappe.get_pymodule_path(app)): for dontwalk in ('locals', '.git', 'public'): if dontwalk in folders: folders.remove(dontwalk) # print path for filename in files: filename = cstr(filename) if filename.startswith("test_") and filename.endswith(".py"): # print filename[:-3] _add_test(path, filename, verbose, test_suite=test_suite) return unittest.TextTestRunner(verbosity=1+(verbose and 1 or 0)).run(test_suite)
def process_lang(lang): path = os.path.join(frappe.get_pymodule_path(app), "translations", lang + ".csv") data = get_translation_dict_from_file(path, lang, app) ctx_lang["translated"][lang] = data mem_file = io.StringIO() w = writer(mem_file, lineterminator="\n") for p, m in messages: translated = m if (m in ctx_lang["translated"][lang] and ctx_lang["translated"][lang][m] not in [m, ""]): translated = ctx_lang["translated"][lang][m] # Set english as default language and use it for empty translation segments in other langs elif lang != "en" and m in ctx_lang["translated"]["en"]: translated = ctx_lang["translated"]["en"][m] w.writerow([p if p else '', m, translated]) raw = mem_file.getvalue() ctx_lang["segments_as_txt"][lang] = raw.replace("<", "<").replace( ">", ">")
def get_all_tests(app): test_file_list = [] for path, folders, files in os.walk(frappe.get_pymodule_path(app)): for dontwalk in ('locals', '.git', 'public', '__pycache__'): if dontwalk in folders: folders.remove(dontwalk) # for predictability folders.sort() files.sort() if os.path.sep.join(["doctype", "doctype", "boilerplate"]) in path: # in /doctype/doctype/boilerplate/ continue for filename in files: if filename.startswith("test_") and filename.endswith(".py") \ and filename != 'test_runner.py': test_file_list.append([path, filename]) return test_file_list
def _add_test(app, path, filename, verbose, test_suite=None, ui_tests=False): import os if os.path.sep.join(["doctype", "doctype", "boilerplate"]) in path: # in /doctype/doctype/boilerplate/ return app_path = frappe.get_pymodule_path(app) relative_path = os.path.relpath(path, app_path) if relative_path == '.': module_name = app else: module_name = '{app}.{relative_path}.{module_name}'.format( app=app, relative_path=relative_path.replace('/', '.'), module_name=filename[:-3]) module = importlib.import_module(module_name) if hasattr(module, "test_dependencies"): for doctype in module.test_dependencies: make_test_records(doctype, verbose=verbose) is_ui_test = True if hasattr(module, 'TestDriver') else False if is_ui_test != ui_tests: return if not test_suite: test_suite = unittest.TestSuite() if os.path.basename(os.path.dirname(path)) == "doctype": txt_file = os.path.join(path, filename[5:].replace(".py", ".json")) if os.path.exists(txt_file): with open(txt_file, 'r') as f: doc = json.loads(f.read()) doctype = doc["name"] make_test_records(doctype, verbose) test_suite.addTest(unittest.TestLoader().loadTestsFromModule(module))
def import_groups(): """rebuild all groups from groups.json""" frappe.conn.sql("""delete from `tabPost`""") frappe.conn.sql("""delete from `tabWebsite Group`""") frappe.conn.sql("""delete from `tabWebsite Sitemap` where ref_doctype='Website Group'""") frappe.conn.auto_commit_on_many_writes = True with open(frappe.get_pymodule_path("aapkamanch", "data", "groups.json")) as f: data = json.loads(f.read()) def create_groups(group_name, group_title, parent_website_sitemap): group = frappe.bean({ "doctype":"Website Group", "group_name": group_name, "group_title": group_title, "parent_website_sitemap": parent_website_sitemap, "public_read":1, "public_write":1, "group_type":"Forum" }).insert() return frappe.conn.get_value("Website Sitemap", {"ref_doctype":"Website Group", "docname": group.doc.name}) india = create_groups("Forum", "Forum", "") for state in data: state_groups = create_groups(state, state, india) for district in data[state]: district_groups = create_groups(district, district, state_groups) # for block in data[state][district] or []: # block_unit = create_groups(block, block, district_groups) frappe.conn.commit()
def run_all_tests(app=None, verbose=False, profile=False): import os apps = [app] if app else frappe.get_installed_apps() test_suite = unittest.TestSuite() for app in apps: for path, folders, files in os.walk(frappe.get_pymodule_path(app)): for dontwalk in ('locals', '.git', 'public'): if dontwalk in folders: folders.remove(dontwalk) # print path for filename in files: filename = cstr(filename) if filename.startswith("test_") and filename.endswith(".py"): # print filename[:-3] _add_test(app, path, filename, verbose, test_suite=test_suite) if profile: pr = cProfile.Profile() pr.enable() out = unittest.TextTestRunner(verbosity=1 + (verbose and 1 or 0)).run(test_suite) if profile: pr.disable() s = StringIO.StringIO() ps = pstats.Stats(pr, stream=s).sort_stats('cumulative') ps.print_stats() print s.getvalue() return out
def load_lang(lang, apps=None): """Combine all translations from `.csv` files in all `apps`. For derivative languages (es-GT), take translations from the base language (es) and then update translations from the child (es-GT)""" if lang == 'en': return {} out = frappe.cache().hget("lang_full_dict", lang) if not out: out = {} for app in (apps or frappe.get_all_apps(True)): path = os.path.join(frappe.get_pymodule_path(app), "translations", lang + ".csv") out.update(get_translation_dict_from_file(path, lang, app) or {}) if '-' in lang: parent = lang.split('-')[0] out = load_lang(parent).update(out) frappe.cache().hset("lang_full_dict", lang, out) return out or {}
def _add_test(app, path, filename, verbose, test_suite=None, ui_tests=False): import os if os.path.sep.join(["doctype", "doctype", "boilerplate"]) in path: # in /doctype/doctype/boilerplate/ return app_path = frappe.get_pymodule_path(app) relative_path = os.path.relpath(path, app_path) if relative_path=='.': module_name = app else: module_name = '{app}.{relative_path}.{module_name}'.format(app=app, relative_path=relative_path.replace('/', '.'), module_name=filename[:-3]) module = importlib.import_module(module_name) if hasattr(module, "test_dependencies"): for doctype in module.test_dependencies: make_test_records(doctype, verbose=verbose) is_ui_test = True if hasattr(module, 'TestDriver') else False if is_ui_test != ui_tests: return if not test_suite: test_suite = unittest.TestSuite() if os.path.basename(os.path.dirname(path))=="doctype": txt_file = os.path.join(path, filename[5:].replace(".py", ".json")) with open(txt_file, 'r') as f: doc = json.loads(f.read()) doctype = doc["name"] make_test_records(doctype, verbose) test_suite.addTest(unittest.TestLoader().loadTestsFromModule(module))
def load_lang(lang, apps=None): """Combine all translations from `.csv` files in all `apps`. For derivative languages (es-GT), take translations from the base language (es) and then update translations from the child (es-GT)""" if lang=='en': return {} out = frappe.cache().hget("lang_full_dict", lang, shared=True) if not out: out = {} for app in (apps or frappe.get_all_apps(True)): path = os.path.join(frappe.get_pymodule_path(app), "translations", lang + ".csv") out.update(get_translation_dict_from_file(path, lang, app) or {}) if '-' in lang: parent = lang.split('-')[0] parent_out = load_lang(parent) parent_out.update(out) out = parent_out frappe.cache().hset("lang_full_dict", lang, out, shared=True) return out or {}
def load_lang(lang, apps=None): """Combine all translations from `.csv` files in all `apps`""" out = {} for app in (apps or frappe.get_all_apps(True)): path = os.path.join(frappe.get_pymodule_path(app), "translations", lang + ".csv") if os.path.exists(path): csv_content = read_csv_file(path) cleaned = {} for item in csv_content: if len(item)==3: # with file and line numbers cleaned[item[1]] = strip(item[2]) elif len(item)==2: cleaned[item[0]] = strip(item[1]) else: raise Exception("Bad translation in '{app}' for language '{lang}': {values}".format( app=app, lang=lang, values=repr(item).encode("utf-8") )) out.update(cleaned) return out
def get_build_maps(): """get all build.jsons with absolute paths""" # framework js and css files build_maps = {} for app_path in app_paths: path = os.path.join(app_path, 'public', 'build.json') if os.path.exists(path): with open(path) as f: try: for target, sources in json.loads(f.read()).iteritems(): # update app path source_paths = [] for source in sources: if isinstance(source, list): s = frappe.get_pymodule_path(source[0], *source[1].split("/")) else: s = os.path.join(app_path, source) source_paths.append(s) build_maps[target] = source_paths except ValueError, e: print path print 'JSON syntax error {0}'.format(str(e))
def load_lang(lang, apps=None): """Combine all translations from `.csv` files in all `apps`""" out = {} for app in (apps or frappe.get_all_apps(True)): path = os.path.join(frappe.get_pymodule_path(app), "translations", lang + ".csv") if os.path.exists(path): csv_content = read_csv_file(path) cleaned = {} for item in csv_content: if len(item)==3: # with file and line numbers cleaned[item[1]] = item[2] elif len(item)==2: cleaned[item[0]] = item[1] else: raise Exception("Bad translation in '{app}' for language '{lang}': {values}".format( app=app, lang=lang, values=repr(item).encode("utf-8") )) out.update(cleaned) return out
def set_all_patches_as_completed(app): patch_path = os.path.join(frappe.get_pymodule_path(app), "patches.txt") if os.path.exists(patch_path): for patch in frappe.get_file_items(patch_path): frappe.get_doc({"doctype": "Patch Log", "patch": patch}).insert() frappe.db.commit()
def write_translations_file(app, lang, full_dict=None): tpath = frappe.get_pymodule_path(app, "translations") frappe.create_folder(tpath) write_csv_file(os.path.join(tpath, lang + ".csv"), get_messages_for_app(app), full_dict or get_full_dict(lang))
def __init__(self, zipcode): self.zipcode = int(str(zipcode).strip()) self.csv_input = pd.read_csv(frappe.get_pymodule_path("autofill_indiazipcode", "dataset", "all_india_zipcode.csv"), encoding ='latin1', sep=',', \ engine='python', quotechar = '"', skipinitialspace=True )