def server_addons_checkout():
if ("" == config.option("server-addons")):
return
addons_list = config.option("server-addons").rsplit(", ")
for name in addons_list:
git_update(name, True)
return
def web_apps_addons_checkout():
if ("" == config.option("web-apps-addons")):
return
addons_list = config.option("web-apps-addons").rsplit(", ")
for name in addons_list:
git_update(name, True)
return
def sdkjs_addons_checkout():
if ("" == config.option("sdkjs-addons")):
return
addons_list = config.option("sdkjs-addons").rsplit(", ")
for name in addons_list:
if name in config.sdkjs_addons:
git_update(config.sdkjs_addons[name], True)
return
def get_web_apps_addons():
result = {}
if ("" == config.option("web-apps-addons")):
return result
addons_list = config.option("web-apps-addons").rsplit(", ")
for name in addons_list:
result[name] = [True, False]
return result
def sdkjs_addons_desktop_param():
if ("" == config.option("sdkjs-addons-desktop")):
return []
params = []
addons_list = config.option("sdkjs-addons-desktop").rsplit(", ")
for name in addons_list:
params.append("--addon=" + name)
return params
def server_addons_checkout():
if ("" == config.option("server-addons")):
return
addons_list = config.option("server-addons").rsplit(", ")
for name in addons_list:
if name in config.server_addons:
git_update(config.server_addons[name])
return
def get_server_addons():
result = {}
if ("" == config.option("server-addons")):
return result
addons_list = config.option("server-addons").rsplit(", ")
for name in addons_list:
result[name] = [True, False]
return result
def web_apps_addons_param():
if ("" == config.option("web-apps-addons")):
return []
params = []
addons_list = config.option("web-apps-addons").rsplit(", ")
for name in addons_list:
params.append("--addon=" + name)
return params
def __init__(self, **kwargs):
self.dtype = get_or_default(kwargs, "dtype", theano.config.floatX)
self.scope = get_or_default(kwargs, "scope", "linear")
self.concat = get_or_default(kwargs, "concat", False)
self.multibias = get_or_default(kwargs, "multibias", False)
self.bias = option(use=True, initializer=zeros_initializer)
self.weight = option(output_major=False,
initializer=uniform_initializer)
def server_addons_param():
if ("" == config.option("server-addons")):
return []
params = []
addons_list = config.option("server-addons").rsplit(", ")
for name in addons_list:
if name in config.server_addons:
params.append("--addon=" + config.server_addons[name])
return params
def get_client():
"""
Returns an xmpp client object that connects to the xmpp server
specified in harvest's configuration.
"""
if config.option("debug"):
return xmpp.Client(server=config.option("server"))
else:
return xmpp.Client(server=config.option("server"), debug=[])
def qt_setup(platform):
compiler = config.check_compiler(platform)
qt_dir = config.option("qt-dir") if (
-1 == platform.find("_xp")) else config.option("qt-dir-xp")
qt_dir = (qt_dir + "/" +
compiler["compiler"]) if platform_is_32(platform) else (
qt_dir + "/" + compiler["compiler_64"])
set_env("QT_DEPLOY", qt_dir + "/bin")
return qt_dir
def __init__(self, **kwargs):
self.dtype = get_or_default(kwargs, "dtype", theano.config.floatX)
self.scope = get_or_default(kwargs, "scope", "feedforward")
self.activation = get_or_default(kwargs, "activation",
theano.tensor.nnet.sigmoid)
self.concat = get_or_default(kwargs, "concat", False)
self.multibias = get_or_default(kwargs, "multibias", False)
self.bias = option(use=True, initializer=zeros_initializer)
self.weight = option(output_major=False,
initializer=uniform_initializer)
def make():
if ("1" == base.get_env("OO_NO_BUILD_JS")):
return
if ("core" == config.option("module")):
return
base_dir = base.get_script_dir() + "/.."
out_dir = base_dir + "/out/js/"
branding = config.option("branding-name")
if ("" == branding):
branding = "onlyoffice"
out_dir += branding
base.create_dir(out_dir)
# builder
build_interface(base_dir + "/../web-apps/build")
build_sdk_builder(base_dir + "/../sdkjs/build")
base.create_dir(out_dir + "/builder")
base.copy_dir(base_dir + "/../web-apps/deploy/web-apps",
out_dir + "/builder/web-apps")
base.copy_dir(base_dir + "/../sdkjs/deploy/sdkjs",
out_dir + "/builder/sdkjs")
# desktop
build_sdk_desktop(base_dir + "/../sdkjs/build")
if config.check_option("module", "desktop"):
build_sdk_desktop(base_dir + "/../sdkjs/build")
base.create_dir(out_dir + "/desktop")
base.copy_dir(base_dir + "/../sdkjs/deploy/sdkjs",
out_dir + "/desktop/sdkjs")
base.copy_dir(base_dir + "/../web-apps/deploy/web-apps",
out_dir + "/desktop/web-apps")
base.delete_dir(out_dir +
"/desktop/web-apps/apps/documenteditor/embed")
base.delete_dir(out_dir +
"/desktop/web-apps/apps/documenteditor/mobile")
base.delete_dir(out_dir +
"/desktop/web-apps/apps/presentationeditor/embed")
base.delete_dir(out_dir +
"/desktop/web-apps/apps/presentationeditor/mobile")
base.delete_dir(out_dir +
"/desktop/web-apps/apps/spreadsheeteditor/embed")
base.delete_dir(out_dir +
"/desktop/web-apps/apps/spreadsheeteditor/mobile")
base.copy_file(
base_dir + "/../web-apps/apps/api/documents/index.html.desktop",
out_dir + "/desktop/web-apps/apps/api/documents/index.html")
build_interface(base_dir + "/../desktop-apps/common/loginpage/build")
base.copy_file(
base_dir + "/../desktop-apps/common/loginpage/deploy/index.html",
out_dir + "/desktop/index.html")
return
def sdkjs_addons_checkout():
if ("" == config.option("sdkjs-addons")):
return
addons_list = config.option("sdkjs-addons").rsplit(", ")
for name in addons_list:
git_update(name, True)
if ("" != config.option("sdkjs-addons-desktop")):
addons_list = config.option("sdkjs-addons-desktop").rsplit(", ")
for name in addons_list:
git_update(name, True)
return
def qt_config(platform):
config_param = config.option("module") + " " + config.option("config")
config_param_lower = config_param.lower()
if (-1 != platform.find("xp")):
config_param += " build_xp"
if ("ios" == platform):
set_env("BITCODE_GENERATION_MODE", "bitcode")
set_env("ENABLE_BITCODE", "YES")
config_param = config_param.replace("desktop", "")
config_param += " iphoneos device"
if (-1 == config_param_lower.find("debug")):
config_param += " release"
return config_param
def get_sdkjs_addons():
result = {}
if ("" == config.option("sdkjs-addons")):
return result
addons_list = config.option("sdkjs-addons").rsplit(", ")
for name in addons_list:
result[name] = [True, False]
if ("" != config.option("sdkjs-addons-desktop")):
addons_list = config.option("sdkjs-addons-desktop").rsplit(", ")
for name in addons_list:
result[name] = [True, False]
return result
def make():
if ("1" != config.option("develop")):
return
if not dependence.check_dependencies():
exit(1)
build_server.build_server_develop()
build_js.build_js_develop(base_dir + "/../../..")
develop_config_server.make()
if ("" != config.option("branding")):
branding_develop_script_dir = base_dir + "/../../../" + config.option(
"branding") + "/build_tools/scripts"
if base.is_file(branding_develop_script_dir + "/develop.py"):
base.cmd_in_dir(branding_develop_script_dir, "python",
["develop.py"], True)
exit(0)
def _call_vcvarsall_and_return_env(arch):
vcvarsall = config.option("vs-path") + "/vcvarsall.bat"
interesting = set(("INCLUDE", "LIB", "LIBPATH", "PATH"))
result = {}
keys = ""
popen = subprocess.Popen('"%s" %s & set' % (vcvarsall, arch), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
stdout, stderr = popen.communicate()
popen.wait()
lines = stdout.split("\n")
for line in lines:
if '=' not in line:
continue
line = line.strip()
key, value = line.split('=', 1)
key = key.upper()
if key in interesting:
if value.endswith(os.pathsep):
value = value[:-1]
result[key] = value
finally:
popen.stdout.close()
popen.stderr.close()
return result
def descr(name):
option = config.option(name)
descr = option.description()
doc = option.documentation()
if doc:
descr += "\n" + doc
return descr
def build_js_develop(root_dir):
#_run_npm_cli(root_dir + "/sdkjs/build")
external_folder = config.option("--external-folder")
if (external_folder != ""):
external_folder = "/" + external_folder
_run_npm(root_dir + external_folder + "/sdkjs/build")
_run_grunt(root_dir + external_folder + "/sdkjs/build",
get_build_param(False) + base.sdkjs_addons_param())
_run_grunt(root_dir + external_folder + "/sdkjs/build",
["develop"] + base.sdkjs_addons_param())
_run_npm(root_dir + external_folder + "/web-apps/build")
_run_npm(root_dir + external_folder + "/web-apps/build/sprites")
_run_grunt(root_dir + external_folder + "/web-apps/build/sprites", [])
old_cur = os.getcwd()
old_product_version = base.get_env("PRODUCT_VERSION")
base.set_env("PRODUCT_VERSION", old_product_version + "d")
os.chdir(root_dir + external_folder + "/web-apps/vendor/framework7-react")
base.cmd("npm", ["install"])
base.cmd("npm", ["run", "deploy-word"])
base.cmd("npm", ["run", "deploy-cell"])
base.cmd("npm", ["run", "deploy-slide"])
base.set_env("PRODUCT_VERSION", old_product_version)
os.chdir(old_cur)
return
def fetchone(self, direction=pytis.data.FORWARD, transaction=None):
if direction != pytis.data.FORWARD or not self._giveone:
return None
self._giveone = False
row_data = [(o, pytis.data.Value(config.option(o).type(), getattr(config, o)))
for o in [c.id() for c in self.columns()]]
return pytis.data.Row(row_data)
def descr(name):
option = config.option(name)
descr = option.description()
doc = option.documentation()
if doc:
descr += "\n" + doc
return descr
def fetchone(self, direction=pytis.data.FORWARD, transaction=None):
if direction != pytis.data.FORWARD or not self._giveone:
return None
self._giveone = False
row_data = [(o, pytis.data.Value(config.option(o).type(), getattr(config, o)))
for o in [c.id() for c in self.columns()]]
return pytis.data.Row(row_data)
def make():
path = base.get_script_dir() + "/../../core/Common/3dParty/openssl"
old_cur = os.getcwd()
os.chdir(path)
base.set_env("ANDROID_HOME", base.get_android_sdk_home())
if (-1 != config.option("platform").find("android")
and not base.is_dir("./build/android")):
subprocess.call(["./build-android-openssl.sh"])
if (-1 != config.option("platform").find("ios")
and not base.is_dir("./build/ios")):
subprocess.call(["./build-ios-openssl.sh"])
os.chdir(old_cur)
return
def check_dependencies():
if (host_platform == 'windows' and not check_vc_components()):
return False
if (host_platform == 'mac'):
return True
checksResult = CDependencies()
checksResult.append(check_git())
checksResult.append(check_nodejs())
if (host_platform == 'linux'):
checksResult.append(check_npm())
checksResult.append(check_curl())
checksResult.append(check_7z())
checksResult.append(check_java())
checksResult.append(check_erlang())
checksResult.append(check_rabbitmq())
checksResult.append(check_gruntcli())
if (host_platform == 'windows'):
checksResult.append(check_buildTools())
if (config.option("sql-type") == 'mysql'):
checksResult.append(check_mysqlServer())
else:
checksResult.append(check_postgreSQL())
server_addons = []
if (config.option("server-addons") != ""):
server_addons = config.option("server-addons").rsplit(", ")
if ("server-lockstorage" in server_addons):
checksResult.append(check_redis())
if (len(checksResult.install) > 0):
install_args = ['install.py']
install_args += checksResult.get_uninstall()
install_args += checksResult.get_removepath()
install_args += checksResult.get_install()
if (host_platform == 'windows'):
install_args[0] = './scripts/develop/' + install_args[0]
code = libwindows.sudo(unicode(sys.executable), install_args)
elif (host_platform == 'linux'):
base.cmd_in_dir('./scripts/develop/', 'python', install_args)
get_updates()
check_npmPath()
if (config.option("sql-type") == 'mysql'):
return check_MySQLConfig(checksResult.sqlPath)
return check_postgreConfig(checksResult.sqlPath)
def qt_config(platform):
config_param = config.option("module") + " " + config.option(
"config") + " " + config.option("features")
config_param_lower = config_param.lower()
if (-1 != platform.find("xp")):
config_param += " build_xp"
if ("ios" == platform):
set_env("BITCODE_GENERATION_MODE", "bitcode")
set_env("ENABLE_BITCODE", "YES")
config_param = config_param.replace("desktop", "")
config_param += " iphoneos device"
if (-1 == config_param_lower.find("debug")):
config_param += " release"
if ("mac_arm64" == platform):
config_param += " apple_silicon use_javascript_core"
if config.check_option("module", "mobile"):
config_param += " support_web_socket"
return config_param
def make():
if ("1" != config.option("develop")):
return
if not dependence.check_dependencies():
exit(1)
build_server.build_server_develop()
build_js.build_js_develop(base_dir + "/../../..")
develop_config_server.make()
exit(0)
def copy_sdkjs_plugins_server(dst_dir, is_name_as_guid=False, is_desktop_local=False):
plugins_dir = get_script_dir() + "/../../sdkjs-plugins"
plugins_list_config = config.option("sdkjs-plugin-server")
if ("" == plugins_list_config):
return
plugins_list = plugins_list_config.rsplit(", ")
for name in plugins_list:
copy_sdkjs_plugin(plugins_dir, dst_dir, name, is_name_as_guid, is_desktop_local)
return
def make():
path = base.get_script_dir() + "/../../core/Common/3dParty/curl"
old_cur = os.getcwd()
os.chdir(path)
if (-1 != config.option("platform").find("android")):
if base.is_dir(path + "/build/android"):
os.chdir(old_cur)
return
subprocess.call(["./build-android-curl.sh"])
elif (-1 != config.option("platform").find("ios")):
if base.is_dir(path + "/build/ios"):
os.chdir(old_cur)
return
subprocess.call(["./build-ios-curl.sh"])
os.chdir(old_cur)
return
def git_update(repo):
print("[git] update: " + repo)
url = "https://github.com/ONLYOFFICE/" + repo + ".git"
if config.option("git-protocol") == "ssh":
url = "[email protected]:ONLYOFFICE/" + repo + ".git"
folder = get_script_dir() + "/../../" + repo
is_not_exit = False
if not is_dir(folder):
cmd("git", ["clone", url, folder])
is_not_exit = True
old_cur = os.getcwd()
os.chdir(folder)
cmd("git", ["fetch"], False if ("1" != config.option("update-light")) else True)
if is_not_exit or ("1" != config.option("update-light")):
cmd("git", ["checkout", "-f", config.option("branch")])
cmd("git", ["pull"], False if ("1" != config.option("update-light")) else True)
os.chdir(old_cur)
return
def create_pull_request(branches_to,
repo,
is_no_errors=False,
is_current_dir=False):
print("[git] create pull request: " + repo)
url = "https://github.com/ONLYOFFICE/" + repo + ".git"
if config.option("git-protocol") == "ssh":
url = "[email protected]:ONLYOFFICE/" + repo + ".git"
folder = get_script_dir() + "/../../" + repo
if is_current_dir:
folder = repo
is_not_exit = False
if not is_dir(folder):
retClone = cmd("git", ["clone", url, folder], is_no_errors)
if retClone != 0:
return
is_not_exit = True
old_cur = os.getcwd()
os.chdir(folder)
branch_from = config.option("branch")
cmd("git", ["checkout", "-f", branch_from], is_no_errors)
cmd("git", ["pull"], is_no_errors)
for branch_to in branches_to:
if "" != run_command("git log origin/" + branch_to + "..origin/" +
branch_from)["stdout"]:
cmd("git", ["checkout", "-f", branch_to], is_no_errors)
cmd("git", ["pull"], is_no_errors)
cmd("hub", [
"pull-request", "--force", "--base", branch_to, "--head",
branch_from, "--message",
"Merge from " + branch_from + " to " + branch_to
], is_no_errors)
if 0 != cmd("git", [
"merge", "origin/" + branch_from, "--no-ff", "--no-edit"
], is_no_errors):
print_error("[git] Conflicts merge " + "origin/" +
branch_from + " to " + branch_to + " in repo " +
url)
cmd("git", ["merge", "--abort"], is_no_errors)
else:
cmd("git", ["push"], is_no_errors)
os.chdir(old_cur)
return
def connect_client(client):
"""
Attempts to connect the given xmpp client object. Logs an error
and raises some sort of xmpp error if connection fails.
"""
try:
client.connect()
except Exception, e:
logging.info('Connecting to %s failed.', config.option("server"))
raise e
def make():
if (-1 == config.option("platform").find("mac")
and -1 == config.option("platform").find("ios")):
return
current_dir_old = os.getcwd()
print("[build]: socketrocket")
os.chdir(current_dir)
if (-1 != config.option("platform").find("mac")):
if not base.is_dir(current_dir + "/build/mac_64") or not base.is_dir(
current_dir + "/build/mac_arm_64"):
buildMacOS()
elif (-1 != config.option("platform").find("ios")):
if not base.is_dir(current_dir + "/build/ios"):
buildIOS()
os.chdir(current_dir_old)
return
def setup_xmpp(handler):
"""
Sets up harvest's xmpp connection, and returns a client object to
use it. If the username specified in harvest's configuration has
not been registered yet, it's automatically registered with the
config-specified password.
The returned client object has its 'presence' and 'iq' events
handled by the given handler object.
"""
client = get_client()
connect_client(client)
if not auth_client(client):
logging.warning('Unrecognized XMPP account: %s! Attempting registration...',
config.option("username"))
ident = {"username": config.option("username"),
"password": config.option("password")}
if not xmpp.features.register(config.option("client"),
config.option("server"), ident):
raise Exception('XMPP New Account registration failed. Bailing out!')
else:
logging.info('Registered %s. Restarting setup...',
config.option("username"))
# At this point, we have created a new XMPP account, and need to restart
# the setup process.
return SetupXmpp()
register_handlers(client, handler)
return client
def main():
LOGFORMAT = '%(asctime)s %(levelname)s %(filename)s %(lineno)s %(message)s'
logging.basicConfig(level=logging.DEBUG, filemode='a', format=LOGFORMAT,
filename=config.option('log_file'))
logging.debug('Starting main thread')
# Delayed import because the path is not set before this point.
sys.path.insert(0, config.option("path"))
from handler import DDSHandler
from harvest import Combine
client = setup_xmpp(DDSHandler())
logging.debug('Connection started')
combine = Combine(client, timeout=10)
combine.daemon = True
combine.start() # wrrrrrrr
try:
process_client(client)
except KeyboardInterrupt: # SIGINT
logging.debug('Shutting down.')
combine.die()
combine.join()
def auth_client(client):
"""
Attempts to authenticate the given xmpp client object with the
username and password credentials specified in harvest's
config. Returns True if the client was successfully authed, and
False otherwise
"""
try:
logging.info('Authenticating %s...', config.option("username"))
auth = client.auth(config.option("username"),
config.option("password"),
config.option("resource"), sasl=False)
if not auth:
logging.info('Auth may have failed for %s.',
config.option("username"))
return auth
except:
logging.info('Authentication failed for %s.', config.option("username"))
return False
return corr_x
def reconstruct(corr_x,W,b,b_rec,input_layer):
hidden = T.nnet.sigmoid(T.dot(corr_x,W) + b)
recon = T.dot(hidden,W.T) + b_rec
if not input_layer:
recon = T.nnet.sigmoid(recon)
return recon
def cost(x,recon_x,kl_divergence):
if not kl_divergence:
return T.mean(T.sum((x - recon_x)**2,axis=1))
else:
return -T.mean(T.sum(x * T.log(recon_x) + (1 - x) * T.log(1 - recon_x), axis=1))
output_file = config.option("output_file","Output of pretraining.")
@output_file
def save(P,output_file):
logging.info("Saving model.")
P.save(output_file)
@output_file
def load(P,output_file):
logging.info("Loading model.")
P.load(output_file)
def build_validation_callback(P):
def validation_callback(prev_score,curr_score):
if curr_score < prev_score: save(P)
return validation_callback
if __name__ == "__main__":
def __init__(self, req):
"""Initialize the global wiking handler instance.
The argument 'req' is the initial request which triggered the Handler
creation, however the handler will normally exist much longer than for
this single request and its method 'handle()' will be called to handle
also other requests (including this one). The constructor only needs
the request instance to gather some global information to be able to
initialize the configuration etc.
"""
# Initialize the global configuration stored in 'wiking.cfg'.
config_file = req.option("config_file")
if config_file:
# Read the configuration file first, so that the request options have a higher priority.
wiking.cfg.user_config_file = config_file
for option in wiking.cfg.options():
name = option.name()
value = req.option(name)
if value and name != "config_file":
if name in ("translation_path", "resource_path", "modules"):
separator = value.find(":") != -1 and ":" or ","
value = tuple([d.strip() for d in value.split(separator)])
if name == "translation_path":
value = tuple(wiking.cfg.translation_path) + value
elif name == "resource_path":
value += tuple(wiking.cfg.resource_path)
elif isinstance(option, wiking.cfg.NumericOption):
if value.isdigit():
value = int(value)
else:
log(OPERATIONAL, "Invalid numeric value for '%s':" % name, value)
continue
elif isinstance(option, wiking.cfg.BooleanOption):
if value.lower() in ("yes", "true", "on"):
value = True
elif value.lower() in ("no", "false", "off"):
value = False
else:
log(OPERATIONAL, "Invalid boolean value for '%s':" % name, value)
continue
elif not isinstance(option, wiking.cfg.StringOption):
log(OPERATIONAL, "Unable to set '%s' through request options." % name)
continue
setattr(wiking.cfg, name, value)
# Apply default values which depend on the request.
server_hostname = wiking.cfg.server_hostname
if server_hostname is None:
# TODO: The name returned by req.server_hostname() works for simple
# cases where there are no server aliases, but we can not guarantee
# that it is really unique. Thus might be safer to raise an error
# when req.primary_server_hostname() returns None and require
# configuring server_hostname explicitly.
server_hostname = req.primary_server_hostname() or req.server_hostname()
wiking.cfg.server_hostname = server_hostname
domain = server_hostname
if domain.startswith("www."):
domain = domain[4:]
if wiking.cfg.webmaster_address is None:
wiking.cfg.webmaster_address = "webmaster@" + domain
if wiking.cfg.default_sender_address is None:
wiking.cfg.default_sender_address = "wiking@" + domain
if wiking.cfg.dbname is None:
wiking.cfg.dbname = server_hostname
if wiking.cfg.resolver is None:
wiking.cfg.resolver = wiking.WikingResolver(wiking.cfg.modules)
# Modify pytis configuration.
import pytis.util
import config
config.dblisten = False
config.log_exclude = [pytis.util.ACTION, pytis.util.EVENT, pytis.util.DEBUG]
for option in ("dbname", "dbhost", "dbport", "dbuser", "dbpass", "dbsslm", "dbschemas"):
setattr(config, option, getattr(wiking.cfg, option))
config.dbconnections = wiking.cfg.connections
config.dbconnection = config.option("dbconnection").default()
config.resolver = wiking.cfg.resolver
del config
self._application = application = wiking.module.Application
self._exporter = wiking.cfg.exporter(translations=wiking.cfg.translation_path)
application.initialize(req)
# Save the current handler instance for profiling purposes.
Handler._instance = self
import data_io
import cPickle as pickle
from pprint import pprint
from itertools import izip, chain
from theano_toolkit import updates
from theano_toolkit.parameters import Parameters
import config
import chunk
import frame_label_data
import validator
import model
import epoch_train_loop
learning_file = config.option("learning_file","Parameters used during updates (e.g. momentum..).")
temporary_file = config.option("temporary_file","File to save temporary parameters while training.")
@learning_file
@temporary_file
def save_state(P,update_vars,temporary_file,learning_file):
logging.info("Saving model and state.")
P.save(temporary_file)
update_vars.save(learning_file)
@learning_file
@temporary_file
def load_state(P,update_vars,learning_file,temporary_file):
logging.info("Loading previous model and state.")
P.load(temporary_file)
update_vars.load(learning_file)
import config
import math
from itertools import izip
import numpy as np
import theano
import theano.tensor as T
batch_size = config.option("batch_size","Size of the batches.",type=config.int)
def create_shared_variable(data_variable):
shared_var = theano.shared(np.zeros((1,) * data_variable.ndim,
dtype=data_variable.dtype))
shared_var.name = "%s_shared"%data_variable
return shared_var
def create_shared_variables(inputs):
return { var:create_shared_variable(var)
for var in inputs }
@batch_size
def build_trainer(inputs,updates,outputs=None,batch_size=256,mapping=None):
"""
Creates a shared variables and a function to load chunk into shared variables and train
"""
if mapping is None:
mapping = create_shared_variables(inputs)
idx = T.iscalar('idx')
train = theano.function(
inputs = [idx],
outputs = outputs,
updates = updates,
def run():
parser = argparse.ArgumentParser(
description="Dump Pytis application specifications to STDOUT",
epilog=(
"Additionally, you can pass other valid Pytis command line arguments, "
"such as --dbhost or --dbname to override certain configuration file "
"options."
),
)
parser.add_argument(
"-w", "--wiking", action="store_true", help="Dump a wiking application (Pytis wx application is the default)"
)
parser.add_argument(
"-e",
"--exit-on-error",
action="store_true",
help=(
"Print traceback and exit when exception occurs. "
"By default, the error message is printed to the "
"dump output and the program continues."
),
)
parser.add_argument(
"-x", "--exclude", nargs="+", metavar="NAME", default=(), help="Name(s) of specification(s) to skip"
)
parser.add_argument(
"-i", "--include", nargs="+", metavar="NAME", default=(), help="Dump only given specification name(s)"
)
parser.add_argument("--config", required=True, metavar="PATH", help="Configuration file path")
args, argv = parser.parse_known_args()
import config
try:
config.add_command_line_options([sys.argv[0], "--config", args.config] + argv)
except getopt.GetoptError as e:
parser.print_help()
sys.exit(1)
config.log_logger = (NullLogger, (), {})
import lcg
lcg.log = lambda *args, **kwargs: None
if args.wiking:
import wiking
wiking.cfg.user_config_file = config.config_file
config.resolver = wiking.cfg.resolver = wiking.WikingResolver(wiking.cfg.modules)
config.dbconnections = wiking.cfg.connections
config.dbconnection = config.option("dbconnection").default()
resolver = config.resolver
processed, errors = 0, 0
if args.include:
names = args.include
else:
if args.wiking:
base_class = wiking.Module
else:
base_class = pytis.presentation.Specification
names = sorted(name for name, cls in resolver.walk(cls=base_class))
for name in names:
if name not in args.exclude:
try:
if args.wiking:
cls = resolver.wiking_module_cls(name)
if not hasattr(cls, "Spec"):
continue
module = resolver.wiking_module(name)
spec = module.Spec(cls)
data = module._data
else:
spec = resolver.specification(name)
if not spec.public:
continue
data_spec = spec.data_spec()
data = data_spec.create(dbconnection_spec=config.dbconnection)
# print ' ', spec.table
view_spec = spec.view_spec()
record = pytis.presentation.PresentedRow(view_spec.fields(), data, None)
for fid in sorted(record.keys()):
t = record.type(fid)
f = view_spec.field(fid)
attr = (
# ('label', f.label()),
("not_null", t.not_null()),
# We care only about the resulting not_null value which
# is printed above, not about type's not_null argument.
("type", (str(t).replace(" not_null=True", "").replace(" not_null=False", ""))),
("editable", f.editable()),
("computer", f.computer()),
)
attributes = " ".join("%s=%s" % (k, v) for k, v in attr if v is not None)
print OBJID_REGEX.sub("", "%s.%s %s" % (name, fid, attributes)).encode("utf-8")
print
except Exception as e:
if args.exit_on_error:
try:
import cgitb
tb = cgitb.text(sys.exc_info())
except:
import traceback
tb = "".join(traceback.format_exception(*sys.exc_info())) + "\n"
sys.stderr.write(tb)
sys.stderr.write("Failed on specification: %s\n\n" % name)
sys.exit(1)
else:
print "ERROR: %s: %s: %r" % (name, e.__class__.__name__, e)
errors += 1
processed += 1
sys.stderr.write(
"Processed %d specifications with %d errors%s.\n"
% (processed, errors, " (grep output for '^ERROR:' or use --exit-on-error)" if errors else "")
)
def _create_data_object(self):
columns = [pytis.data.ColumnSpec(option, config.option(option).type())
for option in self._layout().order()]
return pytis.data.DataFactory(_ConfigData, columns).create()