def __init__(self, locales, timezone, queue=None):
threading.Thread.__init__(self)
self.locales = locales
self.default_locale = ''
for loc in locales:
if loc[3]:
self.default_locale = loc[1]
break
if self.default_locale == '':
self.default_locale = DEFAULTLOCALE
self.timezone = timezone.strip()
self.queue = queue
self.user = getoutput("logname")[0]
self.user_dir = "/home/%s" % self.user
self.current_default = getoutput("awk -F'[=.]' '/UTF-8/{ print $2 }' /etc/default/locale")[0]
self.scriptDir = abspath(dirname(__file__))
self.edition = 'all'
# Get configuration settings
self.debian_version = get_debian_version()
config = get_config_dict(join(self.scriptDir, "solydxk-system.conf"))
self.debian_frontend = "DEBIAN_FRONTEND=%s" % config.get('DEBIAN_FRONTEND', 'noninteractive')
self.apt_options = config.get('APT_OPTIONS_8', '')
if self.debian_version == 0 or self.debian_version >= 9:
self.apt_options = config.get('APT_OPTIONS_9', '')
self.info = config.get('INFO', '/usr/share/solydxk/info')
if exists(self.info):
config = get_config_dict(self.info)
self.edition = config.get('EDITION', 'all').replace(' ', '').lower()
# Steps
self.max_steps = 10
self.current_step = 0
def __init__(self, language, our_total, our_current):
self.cache = apt.Cache()
self.language = language
self.our_current = our_current
self.our_total = our_total
self.locale = self.language.lower().split("_")
self.scriptDir = os.path.dirname(os.path.realpath(__file__))
self.edition = "all"
config = get_config_dict(CONFIG_FILE)
self.info = config.get("info", "/etc/linuxmint/info")
if os.path.exists(self.info):
config = get_config_dict(self.info)
self.edition = config.get("EDITION", "all").replace(" ", "").lower()
def __init__(self, setup, target_dir):
self.setup = setup
self.language = setup.language
self.username = setup.username
self.target_dir = target_dir
self.locale = self.language.lower().split("_")
self.scriptDir = os.path.dirname(os.path.realpath(__file__))
self.edition = 'all'
self.force = get_apt_force()
config = get_config_dict(CONFIG_FILE)
self.info = config.get('info', '/etc/linuxmint/info')
if os.path.exists(self.info):
config = get_config_dict(self.info)
self.edition = config.get('EDITION', 'all').replace(' ', '').lower()
def __init__(self, setup, target_dir):
self.setup = setup
self.language = setup.language
self.username = setup.username
self.target_dir = target_dir
self.locale = self.language.lower().split("_")
self.scriptDir = os.path.dirname(os.path.realpath(__file__))
self.edition = 'all'
self.force = get_apt_force()
config = get_config_dict(CONFIG_FILE)
self.info = config.get('info', '/etc/linuxmint/info')
if os.path.exists(self.info):
config = get_config_dict(self.info)
self.edition = config.get('EDITION', 'all').replace(' ', '').lower()
def __init__(self):
# Set distribution name and version
for f, n, v in (('/etc/os-release', 'PRETTY_NAME', 'VERSION'),
('/etc/lsb-release', 'DISTRIB_DESCRIPTION', 'DISTRIB_RELEASE'),
(CONFIG_FILE, 'DISTRIBUTION_NAME', 'DISTRIBUTION_VERSION')):
try:
config = get_config_dict(f)
name, version = config[n], config[v]
except (IOError, KeyError): continue
else: break
else: name, version = 'Unknown GNU/Linux', '1.0'
self.distribution_name, self.distribution_version = name, version
config = get_config_dict('/etc/os-release')
self.distribution_id = config.get('ID', 'distro')
# Set other configuration
config = get_config_dict(CONFIG_FILE)
self.live_user = config.get('live_user', 'user')
self.media = config.get('live_media_source', '/lib/live/mount/medium/live/filesystem.squashfs')
self.media_type = config.get('live_media_type', 'squashfs')
# Flush print when it's called
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
def run(self):
httpCode = -1
dlSpeed = 0
mirror_index = 0
for mirrorData in self.mirrors:
mirror_index += 1
try:
mirror = mirrorData[3].strip()
if mirror == "URL":
continue
if mirror.endswith('/'):
mirror = mirror[:-1]
# Only check Debian repository: SolydXK is on the same server
httpCode = -1
dlSpeed = 0
config = get_config_dict(
join(self.scriptDir, "solydxk-system.conf"))
dl_file = config.get('DLTEST', 'dltest')
url = os.path.join(mirror, dl_file)
http = "http://"
if url[0:4] == "http":
http = ""
cmd = "curl --connect-timeout 5 -m 5 -w '%%{http_code}\n%%{speed_download}\n' -o /dev/null -s --location %s%s" % (
http, url)
lst = getoutput(cmd)
if len(lst) == 2:
httpCode = int(lst[0])
dlSpeed = lst[1]
# Download speed returns as string with decimal separator
# On non-US systems converting to float throws an error
# Split on the separator, and use the left part only
if ',' in dlSpeed:
dlSpeed = dlSpeed.split(',')[0]
elif '.' in dlSpeed:
dlSpeed = dlSpeed.split('.')[0]
dlSpeed = int(dlSpeed) / 1024
self.queue.put([
mirror,
"%d kb/s" % dlSpeed, mirror_index,
len(self.mirrors)
])
print(("Server {0} - {1} kb/s ({2})".format(
mirror, dlSpeed,
self.get_human_readable_http_code(httpCode))))
except Exception as detail:
# This is a best-effort attempt, fail graciously
print(("Error: http code = {} / error = {}".format(
self.get_human_readable_http_code(httpCode), detail)))
def language_specific(self):
localizeConf = join(self.scriptDir, "localize/%s" % self.default_locale)
if exists(localizeConf):
try:
print((" --> Localizing %s" % self.edition))
config = get_config_dict(localizeConf)
packages = config.get(self.edition, '').strip()
if packages != "":
self.queue_progress()
shell_exec("%s apt-get install %s %s" % (self.debian_frontend, self.apt_options, packages))
except Exception as detail:
msg = "ERROR: %s" % detail
print(msg)
def languageSpecific(self):
localizeConf = os.path.join(self.scriptDir, "localize/%s" % self.language)
if os.path.exists(localizeConf):
try:
print((" --> Localizing %s" % self.edition))
config = get_config_dict(localizeConf)
packages = config.get(self.edition, '').strip()
if packages != "":
self.update_progress(message=_("Install additional localized packages"))
self.exec_cmd("apt-get --yes %s install %s" % (self.force, packages))
except Exception as detail:
msg = "ERROR: %s" % detail
print(msg)
self.update_progress(message=msg)
def languageSpecific(self):
localizeConf = os.path.join(self.scriptDir, "localize/%s" % self.language)
if os.path.exists(localizeConf):
try:
print((" --> Localizing %s" % self.edition))
config = get_config_dict(localizeConf)
packages = config.get(self.edition, '').strip()
if packages != "":
self.update_progress(message=_("Install additional localized packages"))
self.exec_cmd("apt-get --yes %s install %s" % (self.force, packages))
except Exception as detail:
msg = "ERROR: %s" % detail
print(msg)
self.update_progress(message=msg)
def main():
signal(SIGINT, sig_handler)
args = parse_args()
default_branch = args.branch
username = utils.get_username(args)
if args.gerrit == "submit":
gerrit.submit(default_branch, username, args.owner)
sys.exit(0)
elif args.gerrit == "vote":
gerrit.vote(default_branch, username, args.owner)
sys.exit(0)
base_path = utils.get_base_path(default_branch)
config_dict = utils.get_config_dict(args.config, default_branch)
if args.path_to_crowdin == "crowdin" and not utils.check_dependencies():
sys.exit(1)
if args.upload_sources:
upload.upload_sources_crowdin(default_branch, config_dict,
args.path_to_crowdin)
elif args.upload_translations:
upload.upload_translations_crowdin(default_branch, config_dict,
args.path_to_crowdin)
elif args.download:
xml_files = utils.get_xml_files(base_path, default_branch)
download.download_crowdin(
base_path,
default_branch,
xml_files,
username,
config_dict,
args.path_to_crowdin,
)
elif args.unzip:
xml_files = utils.get_xml_files(base_path, default_branch)
from_zip.unzip(args.unzip, base_path, default_branch, xml_files,
username)
if download.has_created_commits() or upload.has_uploaded():
print("\nDone!")
sys.exit(0)
else:
print("\nNothing to commit")
sys.exit(2)
def get_mirror_data(excludeMirrors=[], getDeadMirrors=False):
mirrorData = []
scriptDir = abspath(dirname(__file__))
config = get_config_dict(join(scriptDir, "solydxk-system.conf"))
mirrors_url = config.get(
'MIRRORSLIST', 'http://repository.solydxk.com/umfiles/mirrors.list')
print(mirrors_url)
mirrorsList = join(scriptDir, basename(mirrors_url))
if getDeadMirrors:
mirrorsList = "%s.dead" % mirrorsList
try:
# Download the mirrors list from the server
url = mirrors_url
if getDeadMirrors:
url = "%s.dead" % url
txt = urlopen(url).read().decode('utf-8')
if txt != '':
# Save to a file
with open(mirrorsList, 'w') as f:
f.write(txt)
except:
pass
if exists(mirrorsList):
with open(mirrorsList, 'r') as f:
lines = f.readlines()
for line in lines:
data = line.strip().split(',')
if len(data) > 2:
if getDeadMirrors:
blnAdd = False
for repo in get_local_repos():
if data[2] in repo:
blnAdd = True
break
else:
blnAdd = True
for excl in excludeMirrors:
if excl in data[2]:
blnAdd = False
break
if blnAdd:
mirrorData.append(data)
return mirrorData
def languageSpecific(self):
self.our_current += 1
localizeConf = os.path.join(self.scriptDir, "localize/%s" % self.language)
if os.path.exists(localizeConf):
try:
print " --> Localizing %s" % self.edition
config = get_config_dict(localizeConf)
packages = config.get(self.edition, "").strip()
if packages != "":
self.update_progress(
total=self.our_total,
current=self.our_current,
message=_("Install additional localized packages"),
)
chroot_exec("apt-get --yes --force-yes install %s" % packages)
except Exception as detail:
msg = "ERROR: %s" % detail
print msg
self.update_progress(total=self.our_total, current=self.our_current, message=msg)
def create_lexicon(language='english'):
"""Create Alphabet, lexicon, and dictionary of wordform entries."""
## First, read in processed lexicon dataframe.
config_dict = get_config_dict(config, language=language)
n = config_dict['n'] ## Just for accessing the saved file
PROCESSED_PATH = "data/processed/{lan1}/reals/{lan2}_with_mps_{n}phone_holdout.csv".format(lan1=language, lan2=language, n=n)
print(PROCESSED_PATH)
df_processed = pd.read_csv(PROCESSED_PATH)
## Then construct alphabet.
alphabet = Alphabet()
PHON = config_dict['phon_column']
WORD = config_dict['word_column']
# Then create lexicon and dictionary.
print("Creating lexicon object...")
lexicon = Lexicon(df_processed, alphabet, PHON=PHON, WORD=WORD)
word_info = lexicon.process_data()
return alphabet, lexicon, word_info, df_processed
print("Creating lexicon object...")
lexicon = Lexicon(df_processed, alphabet, PHON=PHON, WORD=WORD)
word_info = lexicon.process_data()
return alphabet, lexicon, word_info, df_processed
### Main
random.seed(42)
#### Parameters
batch_size = 32
language = 'english' ### Change this parameter to determine which language to run it for
num_splits = 10
## Config dict
config_dict = get_config_dict(config, language=language)
PHON = config_dict['phon_column']
WORD = config_dict['word_column']
## Create objects
alphabet, lexicon, word_info, df_processed = create_lexicon(language=language)
trainloader, devloader = get_data_loaders(
lexicon, word_info, batch_size, num_splits)
############################ Code for creating dataloader
########################### Code for training model
from flask import Flask, request, send_from_directory, render_template
from flask_socketio import SocketIO, send, emit
from flask_cors import CORS
import utils as u
import trans as trans
from rio import Rio
config = u.get_config_dict()
rio = Rio(config)
app = Flask(__name__)
socketio = SocketIO(app)
CORS(app)
@app.route('/<mp_id>/<struct>/<idx>', methods=['GET'])
def state(mp_id, struct, idx):
app.logger.debug('hit state.html')
c = trans.container(rio, mp_id)
d = trans.definitions(rio, mp_id)
t = trans.title(rio, mp_id, struct, idx)
upper_page = render_template('html/upper.html',
container=c,
definitions=d,
title=t)
lower_page = render_template('html/lower.html',
container=c,
definitions=d,