def make():
if ("windows" != base.host_platform()):
return
if not config.check_option("module", "mobile"):
return
print("[fetch & build]: glew")
base_dir = base.get_script_dir() + "/../../core/Common/3dParty/glew"
old_cur = os.getcwd()
os.chdir(base_dir)
base.common_check_version("glew", "1", clean)
if not base.is_dir("glew-2.1.0"):
base.download(
"https://deac-ams.dl.sourceforge.net/project/glew/glew/2.1.0/glew-2.1.0-win32.zip",
"./archive.zip")
base.extract("./archive.zip", "./")
base.delete_file("./archive.zip")
os.chdir(old_cur)
return
def make():
print("[fetch & build]: cef")
base_dir = base.get_script_dir() + "/../../core/Common/3dParty/cef"
old_cur = os.getcwd()
os.chdir(base_dir)
platforms = ["win_64", "win_32", "win_64_xp", "win_32_xp", "linux_64", "linux_32", "mac_64"]
url = "http://d2ettrnqo7v976.cloudfront.net/cef/3770/"
for platform in platforms:
if not config.check_option("platform", platform):
continue
url += (platform + "/cef_binary.7z")
if not base.is_dir(platform):
base.create_dir(platform)
os.chdir(platform)
data_url = base.get_file_last_modified_url(url)
old_data_url = base.readFile("./cef_binary.7z.data")
if (data_url != old_data_url):
if base.is_file("./cef_binary.7z"):
base.delete_file("./cef_binary.7z")
if base.is_dir("build"):
base.delete_dir("build")
if base.is_dir("build"):
continue
# download
if not base.is_file("./cef_binary.7z"):
base.download(url, "./cef_binary.7z")
# extract
base.extract("./cef_binary.7z", "./")
base.delete_file("./cef_binary.7z.data")
base.writeFile("./cef_binary.7z.data", data_url)
base.create_dir("./build")
# deploy
if ("mac_64" != platform):
base.copy_files("cef_binary/Release/*", "build/")
base.copy_files("cef_binary/Resources/*", "build/")
if (0 == platform.find("linux")):
base.cmd("chmod", ["a+xr", "build/locales"])
if ("mac_64" == platform):
base.cmd("mv", ["Chromium Embedded Framework.framework", "build/Chromium Embedded Framework.framework"])
os.chdir(base_dir)
os.chdir(old_cur)
return
def make():
git_dir = base.get_script_dir() + "/../.."
old_cur = os.getcwd()
work_dir = git_dir + "/server/FileConverter/bin"
if not base.is_dir(work_dir):
base.create_dir(work_dir)
os.chdir(work_dir)
arch = "x64"
arch2 = "_64"
if ("windows" == base.host_platform()) and not base.host_platform_is64():
arch = "x86"
arch2 = "_32"
url = "http://repo-doc-onlyoffice-com.s3.amazonaws.com/" + base.host_platform(
) + "/core/" + config.option("branch") + "/latest/" + arch + "/core.7z"
data_url = base.get_file_last_modified_url(url)
old_data_url = base.readFile("./core.7z.data")
if (old_data_url != data_url):
print("-----------------------------------------------------------")
print("Downloading core last version... --------------------------")
print("-----------------------------------------------------------")
if (base.is_file("./core.7z")):
base.delete_file("./core.7z")
if (base.is_dir("./core")):
base.delete_dir("./core")
if (base.is_dir("./HtmlFileInternal")):
base.delete_dir("./HtmlFileInternal")
base.download(url, "./core.7z")
print("-----------------------------------------------------------")
print("Extracting core last version... ---------------------------")
print("-----------------------------------------------------------")
base.extract("./core.7z", "./")
base.writeFile("./core.7z.data", data_url)
platform = ""
if ("windows" == base.host_platform()):
platform = "win" + arch2
else:
platform = base.host_platform() + arch2
base.copy_files("./core/*", "./")
base.generate_doctrenderer_config("./DoctRenderer.config",
"../../../sdkjs/deploy/", "server",
"../../../web-apps/vendor/")
base.support_old_versions_plugins(git_dir + "/sdkjs-plugins")
if base.is_dir(git_dir + "/fonts"):
base.delete_dir(git_dir + "/fonts")
base.create_dir(git_dir + "/fonts")
if ("mac" == base.host_platform()):
base.mac_correct_rpath_x2t("./")
print("-----------------------------------------------------------")
print("All fonts generation... -----------------------------------")
print("-----------------------------------------------------------")
base.cmd_exe("./allfontsgen", [
"--input=../../../core-fonts",
"--allfonts-web=../../../sdkjs/common/AllFonts.js",
"--allfonts=./AllFonts.js", "--images=../../../sdkjs/common/Images",
"--selection=./font_selection.bin", "--use-system=true",
"--output-web=../../../fonts"
])
print("All presentation themes generation... ---------------------")
print("-----------------------------------------------------------")
base.cmd_exe("./allthemesgen", [
"--converter-dir=\"" + git_dir + "/server/FileConverter/bin\"",
"--src=\"" + git_dir + "/sdkjs/slide/themes\"",
"--output=\"" + git_dir + "/sdkjs/common/Images\""
])
# add directories to open directories
data_local_devel = "{\n"
data_local_devel += "\"services\": {\n"
data_local_devel += "\"CoAuthoring\": {\n"
data_local_devel += "\"server\": {\n"
data_local_devel += "\"static_content\": {\n"
is_exist_addons = False
for addon in config.sdkjs_addons:
data_local_devel += ("\"/" + config.sdkjs_addons[addon] +
"\" : { \"path\": \"../../../" +
config.sdkjs_addons[addon] + "\" },\n")
is_exist_addons = True
for addon in config.web_apps_addons:
data_local_devel += ("\"/" + config.web_apps_addons[addon] +
"\" : { \"path\": \"../../../" +
config.web_apps_addons[addon] + "\" },\n")
is_exist_addons = True
if is_exist_addons:
data_local_devel = data_local_devel[:-2]
data_local_devel += "\n"
data_local_devel += "}\n"
data_local_devel += "}\n"
data_local_devel += "}\n"
data_local_devel += "}\n"
data_local_devel += "}\n"
base.writeFile(
git_dir + "/server/Common/config/local-development-" +
base.host_platform() + ".json", data_local_devel)
os.chdir(old_cur)
return
def make():
git_dir = base.get_script_dir() + "/../.."
old_cur = os.getcwd()
work_dir = git_dir + "/server/FileConverter/bin"
if not base.is_dir(work_dir):
base.create_dir(work_dir)
os.chdir(work_dir)
arch = "x64"
arch2 = "_64"
if ("windows" == base.host_platform()) and not base.host_platform_is64():
arch = "x86"
arch2 = "_32"
url = get_core_url(arch, config.option("branch"))
data_url = base.get_file_last_modified_url(url)
if (data_url == "" and config.option("branch") != "develop"):
url = get_core_url(arch, "develop")
data_url = base.get_file_last_modified_url(url)
old_data_url = base.readFile("./core.7z.data")
if (data_url != "" and old_data_url != data_url):
print("-----------------------------------------------------------")
print("Downloading core last version... --------------------------")
print("-----------------------------------------------------------")
if (base.is_file("./core.7z")):
base.delete_file("./core.7z")
if (base.is_dir("./core")):
base.delete_dir("./core")
base.download(url, "./core.7z")
print("-----------------------------------------------------------")
print("Extracting core last version... ---------------------------")
print("-----------------------------------------------------------")
base.extract("./core.7z", "./")
base.writeFile("./core.7z.data", data_url)
platform = ""
if ("windows" == base.host_platform()):
platform = "win" + arch2
else:
platform = base.host_platform() + arch2
base.copy_files("./core/*", "./")
else:
print("-----------------------------------------------------------")
print("Core is up to date. ---------------------------------------")
print("-----------------------------------------------------------")
base.generate_doctrenderer_config("./DoctRenderer.config",
"../../../sdkjs/deploy/", "server",
"../../../web-apps/vendor/")
base.support_old_versions_plugins(git_dir + "/sdkjs-plugins")
if base.is_dir(git_dir + "/fonts"):
base.delete_dir(git_dir + "/fonts")
base.create_dir(git_dir + "/fonts")
if ("mac" == base.host_platform()):
base.mac_correct_rpath_x2t("./")
print("-----------------------------------------------------------")
print("All fonts generation... -----------------------------------")
print("-----------------------------------------------------------")
base.cmd_exe("./allfontsgen", [
"--input=../../../core-fonts",
"--allfonts-web=../../../sdkjs/common/AllFonts.js",
"--allfonts=./AllFonts.js", "--images=../../../sdkjs/common/Images",
"--selection=./font_selection.bin", "--use-system=true",
"--output-web=../../../fonts"
])
print("All presentation themes generation... ---------------------")
print("-----------------------------------------------------------")
base.cmd_exe("./allthemesgen", [
"--converter-dir=\"" + git_dir + "/server/FileConverter/bin\"",
"--src=\"" + git_dir + "/sdkjs/slide/themes\"",
"--output=\"" + git_dir + "/sdkjs/common/Images\""
])
#print("- allthemesgen (ios) --------------------------------------")
#base.cmd_exe("./allthemesgen", ["--converter-dir=\"" + git_dir + "/server/FileConverter/bin\"", "--src=\"" + git_dir + "/sdkjs/slide/themes\"", "--output=\"" + git_dir + "/sdkjs/common/Images\"", "--postfix=ios", "--params=280,224"])
# android
#print("- allthemesgen (android) ----------------------------------")
#base.cmd_exe("./allthemesgen", ["--converter-dir=\"" + git_dir + "/server/FileConverter/bin\"", "--src=\"" + git_dir + "/sdkjs/slide/themes\"", "--output=\"" + git_dir + "/sdkjs/common/Images\"", "--postfix=android", "--params=280,224"])
# add directories to open directories
addon_base_path = "../../../"
server_config = {}
static_content = {}
sql = {}
server_addons = []
if (config.option("server-addons") != ""):
server_addons = config.option("server-addons").rsplit(", ")
if ("server-lockstorage" in server_addons):
server_config["editorDataStorage"] = "editorDataRedis"
sdkjs_addons = []
if (config.option("sdkjs-addons") != ""):
sdkjs_addons = config.option("sdkjs-addons").rsplit(", ")
for addon in sdkjs_addons:
static_content["/" + addon] = {"path": addon_base_path + addon}
web_apps_addons = []
if (config.option("web-apps-addons") != ""):
web_apps_addons = config.option("web-apps-addons").rsplit(", ")
for addon in web_apps_addons:
static_content["/" + addon] = {"path": addon_base_path + addon}
if (config.option("external-folder") != ""):
external_folder = config.option("external-folder")
static_content["/sdkjs"] = {
"path": addon_base_path + external_folder + "/sdkjs"
}
static_content["/web-apps"] = {
"path": addon_base_path + external_folder + "/web-apps"
}
if (config.option("sql-type") != ""):
sql["type"] = config.option("sql-type")
if (config.option("db-port") != ""):
sql["dbPort"] = config.option("db-port")
if (config.option("db-user") != ""):
sql["dbUser"] = config.option("db-user")
if (config.option("db-pass") != ""):
sql["dbPass"] = config.option("db-pass")
server_config["static_content"] = static_content
json_file = git_dir + "/server/Common/config/local-development-" + base.host_platform(
) + ".json"
base.writeFile(
json_file,
json.dumps(
{
"services": {
"CoAuthoring": {
"server": server_config,
"sql": sql
}
}
},
indent=2))
os.chdir(old_cur)
return
def results(algo=None):
print("algorithm:", algo if algo in ALGOS else None)
samples = 500 # up to 2000
print("sample size:", samples)
keys = glob.glob('Inspec/keys/*.key')
res = [0] * samples
if algo == 'textrank':
# load a spaCy model, depending on language, scale, etc.
nlp = spacy.load("en_core_web_sm")
# add PyTextRank to the spaCy pipeline
tr = pytextrank.TextRank()
nlp.add_pipe(tr.PipelineComponent, name="textrank", last=True)
elif algo == 'sentiment_pos' or algo == 'sentiment_pos_tfidf':
sid = SentimentIntensityAnalyzer()
for i, key in enumerate(keys[:samples]):
# get actual keywords
key_file = open(key)
whitespace = re.compile(r"\s+")
# remove whitespace and convert to lowercase
actual = [
whitespace.sub(" ", w).strip().lower()
for w in key_file.readlines()
]
# get text document corresponding to current key
num = re.findall(r'\d+', key)[0]
doc = 'Inspec/docsutf8/{}.txt'.format(num)
# get extracted keywords
if algo == 'rake':
extracted = rake(doc)
elif algo == 'textrank':
extracted = textrank(doc, nlp)
elif algo == 'window':
extracted = window(doc)
elif algo == 'window_w_tf_idf':
extracted = window_w_tf_idf(doc)
elif algo == 'tf_idf':
extracted = tf_idf(doc)
elif algo == 'sentiment_pos':
extracted = sentiment_pos(doc, sid)
elif algo == 'sentiment_pos_tfidf':
extracted = sentiment_pos_tfidf(doc, sid)
else:
extracted = extract(doc)
# calculate results
tp = len(set(extracted).intersection(
set(actual))) # number of true positives
precision = tp / len(extracted)
recall = tp / len(actual)
f_measure = (2 * precision * recall) / (
precision + recall) if precision + recall else 0
res[i] = (precision, recall, f_measure)
# calculate average results
avg_res = [sum(x) / len(x) for x in zip(*res)]
print("precision: {}, recall: {}, F-measure: {}".format(*avg_res))
import sys
sys.path.append('./../../../../../build_tools/scripts')
import base
import os
if ("linux" == base.host_platform()):
if not base.is_dir("./build"):
base.bash("./build")
if ("mac" == base.host_platform()):
if not base.is_file("./build.zip"):
base.bash("./run_docker")
if not base.is_dir("./build"):
base.configure_common_apps(
"./../../../../../build_tools/scripts/base.py")
base.extract("./build.zip", "./")
if not base.is_dir("v8"):
if not base.is_dir("depot_tools"):
base.cmd("git", [
"clone",
"https://chromium.googlesource.com/chromium/tools/depot_tools.git"
])
old_cur = os.getcwd()
os.environ[
"PATH"] = old_cur + "/depot_tools" + os.pathsep + os.environ["PATH"]
base.cmd("./depot_tools/fetch", ["v8"], True)
os.chdir(old_cur + "/v8")
base.cmd("git", ["checkout", "-b", "6.0", "branch-heads/6.0"], True)
os.chdir(old_cur)
base.cmd("gclient", ["sync"])
# Main code
from sklearn.ensemble import RandomForestClassifier
from data.consts import *
from base import extract
if __name__ == "__main__":
dt = extract(RandomForestClassifier,
CONFLICTS_PATH,
CONFLICTS_DATA_TYPES,
hasHeaders=CONFLICTS_HAS_HEADER,
headers=CONFLICTS_HEADERS)
print(str(dt))
freetype_version = "2.10.1"
freetype_name = "freetype-" + freetype_version
# fetch emsdk
if not base.is_dir("emsdk"):
base.cmd("git", ["clone", "https://github.com/emscripten-core/emsdk.git"])
os.chdir("emsdk")
base.cmd("emsdk", ["install", "latest"])
base.cmd("emsdk", ["activate", "latest"])
os.chdir("../")
# fetch freetype
if not base.is_dir(freetype_name):
base.download("https://netcologne.dl.sourceforge.net/project/freetype/freetype2/2.10.1/ft" + freetype_version.replace(".", "") + ".zip", freetype_name + ".zip")
base.extract(freetype_name + ".zip", "./")
# compile
compiler_flags = ["-o fonts.js",
"-O3",
"-fno-exceptions",
"-fno-rtti",
"-s WASM=1",
"-s ALLOW_MEMORY_GROWTH=1",
"-s FILESYSTEM=0",
"-s ENVIRONMENT='web'"]
exported_functions = ["_malloc",
"_free",
"_ASC_FT_Malloc",
