def prepare_build(dir_path):
control_file = os.path.join(dir_path, 'debian/control')
rules_file = os.path.join(dir_path, 'debian/rules')
install_file = os.path.join(dir_path, 'debian/install')
make_file = os.path.join(dir_path, 'Makefile')
if not os.path.exists(control_file) or not os.path.exists(rules_file):
return
remove_deps = ['libraspberrypi-dev', 'openbox (>=3.5.2-4~kano.1)', 'chromium']
remove_installs = ['eglsaver']
control_file_str = read_file_contents(control_file)
for dep in remove_deps:
control_file_str = control_file_str.replace(dep + ',', '').replace(dep, '')
write_file_contents(control_file, control_file_str)
# custom build target
if os.path.exists(make_file):
make_file_str = read_file_contents(make_file)
if 'kano-debber:' in make_file_str:
rules_file_str = read_file_contents(rules_file)
insert_start = '#!/usr/bin/make -f\n'
insert_str = 'override_dh_auto_build:\n\tdh_auto_build -- kano-debber\n'
rules_file_str = rules_file_str.replace(insert_start, insert_start + insert_str)
write_file_contents(rules_file, rules_file_str)
install_lines = read_file_contents_as_lines(install_file)
if install_lines:
for remove_install in remove_installs:
install_lines = [l for l in install_lines if remove_install not in l]
write_file_contents(install_file, '\n'.join(install_lines))
def read_snippet(self, file):
"""
Locate the appropriate snippet for the current system and profile and
read it's contents.
This file could be located in a remote location.
This will first check for a per-system snippet, a per-profile snippet,
a distro snippet, and a general snippet. If no snippet is located, it
returns None.
"""
for snipclass in ('system', 'profile', 'distro'):
if self.varExists('%s_name' % snipclass):
fullpath = '%s/per_%s/%s/%s' % (self.getVar('snippetsdir'),
snipclass, file, self.getVar('%s_name' % snipclass))
try:
contents = utils.read_file_contents(fullpath, fetch_if_remote=True)
return contents
except FileNotFoundException:
pass
try:
return "#errorCatcher ListErrors\n" + utils.read_file_contents('%s/%s' % (self.getVar('snippetsdir'), file), fetch_if_remote=True)
except FileNotFoundException:
return None
def generate_kickstart(self, profile=None, system=None):
obj = system
if system is None:
obj = profile
meta = utils.blender(self.api, False, obj)
kickstart_path = utils.find_kickstart(meta["kickstart"])
if not kickstart_path:
return "# kickstart is missing or invalid: %s" % meta["kickstart"]
ksmeta = meta["ks_meta"]
del meta["ks_meta"]
meta.update(ksmeta) # make available at top level
meta["yum_repo_stanza"] = self.generate_repo_stanza(obj, (system is None))
meta["yum_config_stanza"] = self.generate_config_stanza(obj, (system is None))
meta["kernel_options"] = utils.hash_to_string(meta["kernel_options"])
# meta["config_template_files"] = self.generate_template_files_stanza(g, False)
try:
raw_data = utils.read_file_contents(kickstart_path, self.api.logger,
self.settings.template_remote_kickstarts)
if raw_data is None:
return "# kickstart is sourced externally: %s" % meta["kickstart"]
data = self.templar.render(raw_data, meta, None, obj)
return data
except FileNotFoundException:
self.api.logger.warning("kickstart not found: %s" % meta["kickstart"])
return "# kickstart not found: %s" % meta["kickstart"]
def generate_kickstart(self, profile=None, system=None):
obj = system
if system is None:
obj = profile
meta = utils.blender(self.api, False, obj)
kickstart_path = utils.find_kickstart(meta["kickstart"])
if not kickstart_path:
return "# kickstart is missing or invalid: %s" % meta["kickstart"]
ksmeta = meta["ks_meta"]
del meta["ks_meta"]
meta.update(ksmeta) # make available at top level
meta["yum_repo_stanza"] = self.generate_repo_stanza(
obj, (system is None))
meta["yum_config_stanza"] = self.generate_config_stanza(
obj, (system is None))
meta["kernel_options"] = utils.hash_to_string(meta["kernel_options"])
# meta["config_template_files"] = self.generate_template_files_stanza(g, False)
try:
raw_data = utils.read_file_contents(
kickstart_path, self.api.logger,
self.settings.template_remote_kickstarts)
if raw_data is None:
return "# kickstart is sourced externally: %s" % meta[
"kickstart"]
data = self.templar.render(raw_data, meta, None, obj)
return data
except FileNotFoundException:
self.api.logger.warning("kickstart not found: %s" %
meta["kickstart"])
return "# kickstart not found: %s" % meta["kickstart"]
def handle_icon_request(self, user, request):
iconid = request.get('iconid')
if iconid == None or type(iconid) != str:
return None
debug('Icon request from %s: %s\n' % (user.get('nick'), iconid))
if iconid == 'user':
icon = read_file_contents(seek_face_name(self.myself))
version = self.myself.get('faceversion')
limiter = self.iconfetchlimiters['user']
elif iconid.startswith('c:'):
cname = iconid[2:]
if not valid_community(cname):
return None
if cname not in self.myself.get('communities'):
return None
com = self.get_ordinary_community(cname)
if com == None:
return None
if com.get('myiconversion') != com.get('iconversion'):
# Do not reply with a old version of the icon!
return
icon = read_file_contents(seek_community_icon_name(com))
version = com.get('iconversion')
limiter = self.iconfetchlimiters.get(iconid)
if limiter == None:
limiter = Rate_Limiter(ICON_PUSH_INTERVAL)
self.iconfetchlimiters[iconid] = limiter
else:
return None
if icon == None:
icon = ''
if version == None:
version = 0
request = {'t': 'iconpush', 'iconid': iconid, 'icon': icon, 'version': version}
if normal_traffic_mode():
self.fetcher.fetch(user, PLUGIN_TYPE_COMMUNITY, request, None, ack=False)
elif limiter == None or limiter.check():
self.fetcher.fetch_community(self.get_default_community(), PLUGIN_TYPE_COMMUNITY, request, None, ack=False)
return {}
def test_read_file_contents(self):
# txt
expected_contents = ('hello\nworld\n123', 'default')
with open(self.test_txt, 'w') as f:
f.write(expected_contents[0])
actual_contents = utils.read_file_contents(self.test_txt)
self.assertEqual(expected_contents, actual_contents)
# json
expected_contents = ({
'hello': 'world',
'1': {
'2': '3',
'4': '5'
}
}, 'default')
with open(self.test_txt, 'w') as f:
json.dump(expected_contents[0], f)
actual_contents = utils.read_file_contents(self.test_txt,
read_json=True)
self.assertEqual(expected_contents, actual_contents)
def prepare_build(dir_path):
control_file = os.path.join(dir_path, 'debian/control')
rules_file = os.path.join(dir_path, 'debian/rules')
install_file = os.path.join(dir_path, 'debian/install')
make_file = os.path.join(dir_path, 'Makefile')
if not os.path.exists(control_file) or not os.path.exists(rules_file):
return
remove_deps = [
'libraspberrypi-dev', 'openbox (>=3.5.2-4~kano.1)', 'chromium'
]
remove_installs = ['eglsaver']
control_file_str = read_file_contents(control_file)
for dep in remove_deps:
control_file_str = control_file_str.replace(dep + ',',
'').replace(dep, '')
write_file_contents(control_file, control_file_str)
# custom build target
if os.path.exists(make_file):
make_file_str = read_file_contents(make_file)
if 'kano-debber:' in make_file_str:
rules_file_str = read_file_contents(rules_file)
insert_start = '#!/usr/bin/make -f\n'
insert_str = 'override_dh_auto_build:\n\tdh_auto_build -- kano-debber\n'
rules_file_str = rules_file_str.replace(insert_start,
insert_start + insert_str)
write_file_contents(rules_file, rules_file_str)
install_lines = read_file_contents_as_lines(install_file)
if install_lines:
for remove_install in remove_installs:
install_lines = [
l for l in install_lines if remove_install not in l
]
write_file_contents(install_file, '\n'.join(install_lines))
def extract_lyrics_from_file(lyrics_filepath):
"""
Extract lyrics from provided file path
Args:
lyrics_filepath: str, path to lyrics file
Returns: str, lyrics or '' if path does not exist
"""
# read in the lyrics of each song
lyrics = ''
if os.path.exists(lyrics_filepath):
lyrics = read_file_contents(lyrics_filepath)[0]
return lyrics
def generate_kickstart(self, profile=None, system=None):
obj = system
if system is None:
obj = profile
meta = utils.blender(self.api, False, obj)
kickstart_path = utils.find_kickstart(meta["kickstart"])
if not kickstart_path:
return "# kickstart is missing or invalid: %s" % meta["kickstart"]
ksmeta = meta["ks_meta"]
del meta["ks_meta"]
meta.update(ksmeta) # make available at top level
meta["yum_repo_stanza"] = self.generate_repo_stanza(
obj, (system is None))
meta["yum_config_stanza"] = self.generate_config_stanza(
obj, (system is None))
meta["kernel_options"] = utils.hash_to_string(meta["kernel_options"])
# meta["config_template_files"] = self.generate_template_files_stanza(g, False)
# add extra variables for other distro types
if "tree" in meta:
urlparts = urlparse.urlsplit(meta["tree"])
meta["install_source_directory"] = urlparts[2]
try:
raw_data = utils.read_file_contents(
kickstart_path, self.api.logger,
self.settings.template_remote_kickstarts)
if raw_data is None:
return "# kickstart is sourced externally: %s" % meta[
"kickstart"]
distro = profile.get_conceptual_parent()
if system is not None:
distro = system.get_conceptual_parent().get_conceptual_parent()
data = self.templar.render(raw_data, meta, None, obj)
if distro.breed == "suse":
# AutoYaST profile
data = self.generate_autoyast(profile, system, data)
return data
except FileNotFoundException:
self.api.logger.warning("kickstart not found: %s" %
meta["kickstart"])
return "# kickstart not found: %s" % meta["kickstart"]
def generate_kickstart(self, profile=None, system=None):
obj = system
if system is None:
obj = profile
meta = utils.blender(self.api, False, obj)
kickstart_path = utils.find_kickstart(meta["kickstart"])
if not kickstart_path:
return "# kickstart is missing or invalid: %s" % meta["kickstart"]
ksmeta = meta["ks_meta"]
del meta["ks_meta"]
meta.update(ksmeta) # make available at top level
meta["yum_repo_stanza"] = self.generate_repo_stanza(obj, (system is None))
meta["yum_config_stanza"] = self.generate_config_stanza(obj, (system is None))
meta["kernel_options"] = utils.hash_to_string(meta["kernel_options"])
# add extra variables for other distro types
if "tree" in meta:
urlparts = urlparse.urlsplit(meta["tree"])
meta["install_source_directory"] = urlparts[2]
try:
raw_data = utils.read_file_contents(
kickstart_path, self.api.logger,
self.settings.template_remote_kickstarts)
if raw_data is None:
return "# kickstart is sourced externally: %s" % meta["kickstart"]
distro = profile.get_conceptual_parent()
if system is not None:
distro = system.get_conceptual_parent().get_conceptual_parent()
data = self.templar.render(raw_data, meta, None, obj)
if distro.breed == "suse":
# AutoYaST profile
data = self.generate_autoyast(profile, system, data)
return data
except FileNotFoundException:
self.api.logger.warning("kickstart not found: %s" % meta["kickstart"])
return "# kickstart not found: %s" % meta["kickstart"]
def set_community_icon(self, com, icon_fname):
if icon_fname == None:
delete_community_icon(com)
else:
icon = read_file_contents(icon_fname)
if icon == None:
warning('Can not set community icon from %s\n' % icon_fname)
return False
if not save_community_icon(com, icon):
warning('Could not save community icon from %s\n' % icon_fname)
return False
# New icon version so other users will be notified. Random number
# because this is a distributed system
version = randint(0, 1 << 32 - 1)
com.set('iconversion', version)
com.set('myiconversion', version)
self.announce_community_change(com)
return True
def set_my_face(self, face_fname):
""" Set new profile picture for given user. Should be myself! """
if not face_fname:
delete_face(self.myself)
else:
face = read_file_contents(face_fname)
if face == None:
warning('Can not set user face from %s\n' % face_fname)
return False
if not save_face(self.myself, face):
warning('Could not save user face from %s\n' % face_fname)
return False
if self.myself.get('faceversion') == None:
self.myself.set('faceversion', 0)
else:
self.myself.set('faceversion', self.myself.get('faceversion') + 1)
self.announce_user_change(self.myself, allowme=True)
return True
if not (args.down or args.build or args.install):
sys.exit()
# checking packages are present
_, _, rc_curl = run_cmd('which curl')
_, _, rc_debuild = run_cmd('which debuild')
_, _, rc_gdebi = run_cmd('which gdebi')
if rc_curl or rc_debuild or rc_gdebi:
sys.exit('Run prepare_system.sh first')
# start
root_dir = os.getcwd()
token = read_file_contents('token')
github = 'https://api.github.com/repos/KanoComputing/{}/tarball/{}'
for name, branch in repos_selected:
url = github.format(name, branch)
dir_str = '{}___{}'.format(name, branch)
dir_path = os.path.join(root_dir, dir_str)
debfile = ''
print
if args.down:
print 'Downloading {} ...'.format(dir_str)
delete_dir(dir_path)
ensure_dir(dir_path)
def index_lyrics(csv_input, csv_output, artist_first_letter=None):
logger.info('Reading in input csv {0}'.format(csv_input))
start = time.time()
df = pd.read_csv(csv_input,
encoding='utf-8',
dtype={
'msd_artist': str,
'msd_title': str
})
# if starting from mxm_mapping csv, we need to add additional cols
df = add_col_if_dne(df, 'is_english', -1)
df = add_col_if_dne(df, 'lyrics_available', -1)
df = add_col_if_dne(df, 'wordcount', -1)
df = add_col_if_dne(df, 'lyrics_filename', -1)
df = df.sort_values('msd_artist')
end = time.time()
elapsed_time = end - start
logger.debug('Elapsed Time: {0} minutes'.format(elapsed_time / 60))
logger.info('Processing Lyrics...')
count_total = 0
count_nonenglish = 0
count_nolyrics = 0
count_success = 0
count_language_error = 0
start = time.time()
try:
for index, row in df.iterrows():
lyrics_filename = make_lyric_file_name(row['msd_artist'],
row['msd_title'])
txt_lyricfile = 'data/lyrics/txt/{0}.txt'.format(lyrics_filename)
if artist_first_letter and not row['msd_artist'].lower(
).startswith(artist_first_letter.lower()):
continue
if row.get('lyrics_available', None) >= 0:
count_total += 1
logger.debug('{0}, {1}: already processed, skipping'.format(
count_total, txt_lyricfile))
continue
wordcount = 0
lyrics_available = 0
is_english = 0
if not os.path.exists(txt_lyricfile):
logger.debug('{0} {1}: no lyric file'.format(
count_total, txt_lyricfile))
else:
contents, encoding = read_file_contents(txt_lyricfile)
if contents:
lyrics_available = 1
# drop the non-english
# possible speed improvement available via pandas:
# https://stackoverflow.com/questions/49261711/detecting-language-of-a-text-document-other-than-using-iterrows
try:
lang = detect(str(contents))
except LangDetectException as e:
logger.info(str(e))
logger.debug(
'{0} {1} {2} caused a language error.'.format(
count_total, txt_lyricfile, encoding))
count_language_error += 1
continue
is_english = 1 if lang == 'en' else 0
if not is_english:
logger.debug('{0} {1}: not english'.format(
count_total, txt_lyricfile))
count_nonenglish += 1
wordcount = len(contents.split())
# success!
if lyrics_available:
logger.debug('{0} {1} {2}: success'.format(
count_total, txt_lyricfile, encoding))
count_success += 1
df.loc[index, 'lyrics_filename'] = lyrics_filename
df.loc[index, 'lyrics_available'] = lyrics_available
df.loc[index, 'wordcount'] = wordcount
df.loc[index, 'is_english'] = is_english
count_total += 1
except KeyboardInterrupt as kbi:
logger.info(str(kbi))
except Exception as e:
print(txt_lyricfile, contents)
raise e
logger.info('saving indexed lyric data to {0}'.format(csv_output))
df = df.sort_values('msd_artist')
df.to_csv(csv_output, encoding='utf-8', index=False)
end = time.time()
elapsed_time = end - start
logger.info('{0} Artist/pairs processed'.format(count_total))
logger.info('{0} deemed not english'.format(count_nonenglish))
logger.info('{0} total rows'.format(count_total))
logger.info('{0} lacking lyrics'.format(count_total - count_success))
logger.info('{0} songs ready'.format(count_success))
logger.info(
'{0} songs had an error for language.'.format(count_language_error))
logger.info('Elapsed Time: {0} minutes'.format(elapsed_time / 60))
return
print r[0], r[1]
if not (args.down or args.build or args.install):
sys.exit()
# checking packages are present
_, _, rc_curl = run_cmd('which curl')
_, _, rc_debuild = run_cmd('which debuild')
_, _, rc_gdebi = run_cmd('which gdebi')
if rc_curl or rc_debuild or rc_gdebi:
sys.exit('Run prepare_system.sh first')
# start
root_dir = os.getcwd()
token = read_file_contents('token')
github = 'https://api.github.com/repos/KanoComputing/{}/tarball/{}'
for name, branch in repos_selected:
url = github.format(name, branch)
dir_str = '{}___{}'.format(name, branch)
dir_path = os.path.join(root_dir, dir_str)
debfile = ''
print
if args.down:
print 'Downloading {} ...'.format(dir_str)
delete_dir(dir_path)
ensure_dir(dir_path)
os.chdir(dir_path)
