def download_audios(word): """ download all audio files from word """ for pronunciation in word['pronunciations']: url = pronunciation['url'] try: if url is not None: download(url, AUDIO_PATH) except urllib.error.HTTPError: # 'ginkgo': audio urls are available but not valid put(word['name'], CORRUPTED_AUDIOS_FILE)
def place_disc():
global player
if check_column_full(arrow_position):
return
if player == 1:
coordinate = util.put(board, arrow_position)
util.write_one(board, coordinate)
squares.append(paint_square(movements.arrowhead(arrow_position), movements.grid_space(coordinate[1]), player))
player = 2
else:
coordinate = util.put(board, arrow_position)
util.write_two(board, coordinate)
squares.append(paint_square(movements.arrowhead(arrow_position), movements.grid_space(coordinate[1]), player))
player = 1
def insert(json):
""" insert json data in db """
id = json['id']
name = json['name'].strip()
try:
name = name.encode('ascii') # remove unicode character (’tis -> tis)
except UnicodeEncodeError:
try:
name = to_ascii(name)
except KeyError:
util.put(name, 'untracked_unicode.txt')
json_str = uglify(json)
CURSOR.execute(
'INSERT INTO {} ([ID], [Name], [Definition]) VALUES (?, ?, ?)'.format(
TABLE_NAME), (id, name, json_str))
def preview_template(id, params):
print("Previewing template")
payload = {"templateId": id, "params": json.loads(params)}
#print(json.dumps(payload))
response = put("dna/intent/api/v1/template-programmer/template/preview",
payload)
if "cliPreview" in response:
print(response['cliPreview'])
else:
print(json.dumps(response, indent=2))
def extract_data(word):
""" get word info using api and save data in filesystem
argument: word name to extract data from
return (statuscode, references)
status code:
0: success
1: word not found (error 404)
2: connection error
3: corrupted data
references: a list of other ids (same word with different wordforms)
"""
try:
print("Request html page of '{}'...".format(word))
LOG.info("Request html page of '%s'...", word)
Word.get(word)
except WordNotFound: # 404
print("No data for '{}' word. Skipping".format(word))
LOG.info("No data for '%s' word. Skipping", word)
put(word, NOT_FOUND_WORDS_FILE)
return (1, None)
except (ConnectionError, HTTPError, Timeout) as error:
print("Requests failed: '{}'".format(error))
LOG.debug("Requests failed: '%s'", error)
put(word, SKIPPED_WORDS_FILE)
return (2, None)
try:
print("Extracting data from '{}'...".format(word))
LOG.info("Extracting data from '%s'...", word)
data = Word.info()
if NEED_AUDIO:
# download pronounce audio file
print("Downloading audio file of '{}'...".format(word))
LOG.info("Downloading audio file of '%s'...", word)
download_audios(data)
# save word information (definitions, examples, idioms,...)
print("saving '{}' in json format to {}...".format(word, DEF_PATH))
LOG.info("saving '%s' in json format to %s...", word, DEF_PATH)
save(data, DEF_PATH)
except Exception as error:
print(traceback.format_exc())
LOG.debug(traceback.format_exc())
put(word, CORRUPTED_WORDS_FILE)
return (3, None)
else:
return (0, data['similars'])
def ensure():
select_package("apt")
package_ensure(["znc"]) # On debian will automatically be enabled
user_ensure('znc')
group_ensure('znc')
group_user_ensure('znc', 'znc')
dir_ensure("/var/znc", mode='755')
dir_ensure("/var/znc/configs", mode='755')
run("chown znc:znc /var/znc")
util.put("/srv/znc.conf", "/var/znc/configs", user="******", mode="600")
util.put("config/keys/znc.pem", "/var/znc", user="******", mode="600")
util.put("config/znc/modules", "/var/znc", user="******", mode="755")
run("cp /var/znc/modules/*.so /usr/lib/znc")
systemd.add_unit("config/systemd/znc.service")
run("systemctl enable znc")
run("systemctl restart znc")
def ensure():
select_package("apt")
package_ensure(["znc"]) # On debian will automatically be enabled
user_ensure('znc')
group_ensure('znc')
group_user_ensure('znc', 'znc')
dir_ensure("/var/znc", mode='755')
dir_ensure("/var/znc/configs", mode='755')
run("chown znc:znc /var/znc")
util.put("/srv/znc.conf", "/var/znc/configs", user="******", mode="600")
util.put("config/keys/znc.pem", "/var/znc", user="******", mode="600")
util.put("config/znc/modules", "/var/znc", user="******", mode="755")
run("cp /var/znc/modules/*.so /usr/lib/znc")
systemd.add_unit("config/systemd/znc.service")
run("systemctl enable znc")
run("systemctl restart znc")
def put_cert(cert, user=None):
with mode_sudo():
dir_ensure('/etc/ssl/certs', mode='1777')
if user is None:
user = '******'
return util.put(cert, '/etc/ssl/certs', mode='0644', user=user)[0]
def write_tunetop(fp):
put(f"\n\n%% --- tune {common.tunenum+1} {info.titles[0]}\n")
if not common.epsf:
common.bskip(cfmt.topspace)
def write_text_block(fp, job: int, words_of_text='') -> None:
if not words_of_text:
return
baseskip = cfmt.textfont.size * cfmt.lineskipfac
parskip = cfmt.textfont.size * cfmt.parskipfac
cfmt.textfont.set_font_str(common.page_init)
swfac = set_swfac(cfmt.textfont.name)
# Now this is stupid. All that work to just set it to 1.0
spw = util.cwid(' ')
put("/LF \{0 "
f"{-baseskip:.1f}"
" rmoveto} bind def\n")
# output by pieces, separate at newline token
ntxt = len(words_of_text)
i1 = 0
while i1 < ntxt:
i2 = -1
for i in range(i1, ntxt):
if words_of_text[i] == '$$NL$$':
i2 = i
break
if i2 == -1:
i2 = ntxt
bskip(baseskip)
if job == OBEYLINES:
put("0 0 M(")
for i in range(i1, i2):
line, wwidth = tex_str(words_of_text[i])
put(f"{line} ")
put(") rshow\n")
elif job == OBEYCENTER:
put(f"{cfmt.staff_width / 2:.1f} 0 M(")
for i in range(i1, i2):
line, wwidth = tex_str(words_of_text[i])
put(f"{line}")
if i<i2-1:
put(" ")
put(") cshow\n")
elif job == OBEYRIGHT:
put(f"{cfmt.staff_width:.1f} 0 M(")
for i in range(i1, i2):
line, wwidth = tex_str(words_of_text[i])
put(f"{line}")
if i<i2-1:
put(" ")
put(") lshow\n")
else:
put("0 0 M mark\n")
nc = 0
mc = -1
wtot = -spw
for i in range(i2-1, i1, -1):
line, wwidth = tex_str(words_of_text[i])
mc += len(words_of_text)
wtot += wwidth+spw
nc += len(line)+2
if nc >= 72:
nc = 0
put("\n")
put(f"({line})")
if job == RAGGED:
put(" %.1f P1\n", cfmt.staff_width)
else:
put(" %.1f P2\n", cfmt.staff_width)
# first estimate:(total textwidth)/(available width)
textwidth=wtot*swfac*cfmt.textfont.size
if "Courier" in cfmt.textfont.name:
textwidth = 0.60 * mc * cfmt.textfont.size
ftline0 = textwidth / cfmt.staff_width
# revised estimate: assume some chars lost at each line end
nbreak = int(ftline0)
textwidth = textwidth + 5 * nbreak * util.cwid('a') * swfac * cfmt.textfont.size
ftline = textwidth/cfmt.staff_width
ntline = (int)(ftline+1.0)
if common.vb >= 10:
print(f"first estimate {ftline0:.2f}, revised {ftline:.2f}")
if common.vb >= 10:
print(f"Output {i2-i1} words, about {ftline:.2f} lines(fac {swfac:.2f})")
bskip((ntline-1)*baseskip)
buffer.buffer_eob(fp)
# next line to allow pagebreak after each text "line"
# if(!epsf && !within_tune) write_buffer(fp);
i1=i2+1
bskip(parskip)
buffer.buffer_eob(fp)
# next line to allow pagebreak after each paragraph
if not common.epsf and not common.within_tune:
buffer.write_buffer(fp)
common.page_init = ""
'humidity': humidity,
'timestamp': long(datetime.now().strftime('%Y%m%d%H%M%S'))
}
return doc
if __name__ == '__main__':
GPIO.output(GREEN_LED, True)
print(datetime.now())
result = util.getHumidityAndTemperature(SENSOR, PIN)
temp_data = get_temperature_data(result['temperature'])
hum_data = get_humidity_data(result['humidity'])
host = ['http://192.168.1.195:9200']
es = Elasticsearch(host)
index = 'ambience'
doc = 'temperature'
util.put(es, index, doc, temp_data)
# util.search(es, index, {'query': {'match_all': {}}, 'sort': [{'_timestamp': {'order': 'desc'}}]})
doc = 'humidity'
util.put(es, index, doc, hum_data)
# util.search(es, index, {'query': {'match_all': {}}, 'sort': [{'_timestamp': {'order': 'desc'}}]})
GPIO.output(GREEN_LED, False)
GPIO.cleanup()
def draw_gracenotes(self, x, w, d):
"""
This is called only by draw_note.
:param float x:
:param float w:
:param float d:
"""
n = len(self.agr)
if not n:
return
gr_len = self.gr.note_len
fac_x = 0.3
fac = d / w - 1
if fac < 0:
fac = 0
fac = 1 + (fac * fac_x) / (fac + fac_x)
ii = 0
a = 0
b = 35
dx = 0
for m in range(self.npitch): # room for accidentals
dd = -self.shhd[m]
if self.accs[m]:
dd = -self.shhd[m] + self.shac[m]
if self.accs[m] == constants.A_FT or self.accs[m] == constants.A_NT:
dd = dd - 2
if dx < dd:
dx = dd
xx = x - fac * (dx + constants.GSPACE0)
xg = list()
yg = [3 * (i - 18) + self.yadd for i in self.gr.p]
if yg[:-1] >= self.ymx:
xx += 1
if yg[:-1] < self.ymn and n == 1:
xx -= 2
for i in range(n - 1):
if yg[i] > yg[i + 1] + 8:
xx += fac * 1.8
xg.append(xx)
xx -= fac * constants.GSPACE
if self.agr[i]:
xx -= 3.5
if n > 1:
# linear fit through stems
px = [i + constants.GSTEM_XOFF for i in xg]
py = [i + constants.GSTEM for i in yg]
s1 = n
sx = sum(px)
sy = sum(py)
sxx = sum([i * i for i in px])
sxy = sum([a * b for a, b in zip(px, py)])
delta = s1 * sxx - sx * sx # beam fct: y = ax+b
a = (s1 * sxy - sx * sy) / delta
if a > constants.BEAM_SLOPE:
a = constants.BEAM_SLOPE
if a < -constants.BEAM_SLOPE:
a = -constants.BEAM_SLOPE
b = (sy - a * sx) / s1
if key.bagpipe:
a = 0
b = 35
lmin = 100 # shift to get min stems
px = [i + constants.GSTEM_XOFF for i in xg]
py = [a * i + b for i in px]
for a, b in zip(py, yg):
if a - b < lmin:
lmin = a - b
if lmin < 10:
b += 10 - lmin
for i in range(n): # draw grace notes
if n > 1 and not gr_len or gr_len < constants.HALF:
px = xg[i] + constants.GSTEM_XOFF
py = a * px + b
lg = py - yg[i]
put(f"%{xg[i]:.1f} %{yg[i]:.1f} %{lg:.1f} gnt ")
else:
lg = constants.GSTEM
if gr_len > constants.EIGHTH:
lg += 1
if not gr_len and cfmt.nogracestroke:
put(f"{xg[i]:.1f} {yg[i]:.1f} {lg:.1f} gn8 ")
elif not gr_len:
put(f"{xg[i]:.1f} {yg[i]:.1f} {lg:.1f} gn8s ")
elif gr_len > constants.HALF:
put(f"{xg[i]:.1f} {yg[i]:.1f} gn1 ")
elif gr_len == constants.HALF:
put(f"{xg[i]:.1f} {yg[i]:.1f} {lg:.1f} gn2 ")
elif gr_len == constants.QUARTER:
put(f"{xg[i]:.1f} {yg[i]:.1f} {lg:.1f} gnt ")
elif gr_len == constants.EIGHTH:
put(f"{xg[i]:.1f} {yg[i]:.1f} {lg:.1f} gn8 ")
else: # gr_len < EIGHTH
put(f"{xg[i]:.1f} {yg[i]:.1f} {lg:.1f} gn16 ")
acc = self.agr[i]
if acc == constants.A_SH:
put(f"{xg[i] - 4.5:.1f} {yg[i]:.1f} gsh0 ")
if acc == constants.A_FT:
put(f"{xg[i] - 4.5:.1f} {yg[i]:.1f} gft0 ")
if acc == constants.A_NT:
put(f"{xg[i] - 4.5:.1f} {yg[i]:.1f} gnt0 ")
if acc == constants.A_DS:
put(f"{xg[i] - 4.5:.1f} {yg[i]:.1f} gds0 ")
if acc == constants.A_DF:
put(f"{xg[i] - 4.5:.1f} {yg[i]:.1f} gdf0 ")
y = int(yg[i]) # ledger lines
if y <= -6:
if y % 6:
put(f"{xg[i]:.1f} {y + 3} ghl ")
else:
put(f"{xg[i]:.1f} {y} ghl ")
if y >= 30:
if y % 6:
put(f"{xg[i]:.1f} {y - 3} ghl ")
else:
put(f"{xg[i]:.1f} {y} ghl ")
if n > 1: # beam
if not gr_len and key.bagpipe:
put(f"{px[0]:.1f} {py[0]:.1f} {px[n - 1]:.1f} {py[n - 1]:.1f} gbm3 "
)
elif not gr_len:
put(f"{px[0]:.1f} {py[0]:.1f} {px[n - 1]:.1f} {py[n - 1]:.1f} gbm2 "
)
elif gr_len == constants.EIGHTH:
put(f"{px[0]:.1f} {py[0]:.1f} {px[n - 1]:.1f} {py[n - 1]:.1f} gbm1 "
)
elif gr_len == constants.SIXTEENTH:
put(f"{px[0]:.1f} {py[0]:.1f} {px[n - 1]:.1f} {py[n - 1]:.1f} gbm2 "
)
elif gr_len < constants.SIXTEENTH:
put(f"{px[0]:.1f} {py[0]:.1f} {px[n - 1]:.1f} {py[n - 1]:.1f} gbm3 "
)
bet1 = 0.2 # slur
bet2 = 0.8
yy = 1000
for i in range(n - 1, 0, -1):
if yg[i] <= yy:
yy = yg[i]
ii = i
x0 = xg[ii]
y0 = yg[ii] - 5
if i > 0:
x0 = x0 - 4
y0 = y0 + 1
x3 = x - 1
if self.npitch > 1:
yslurhead = 3 * (self.grcpit - 18) + self.yadd
else:
yslurhead = self.ymn
y3 = yslurhead - 5
dy1 = (x3 - x0) * 0.4
if dy1 > 3:
dy1 = 3
dy2 = dy1
if yg[ii] > yslurhead + 7:
x0 = xg[ii] - 1
y0 = yg[ii] - 4.5
y3 = yslurhead + 1.5
x3 = x - dx - 5.5
dy2 = (y0 - y3) * 0.2
dy1 = (y0 - y3) * 0.8
bet1 = 0.0
if y3 > y0 + 4:
y3 = y0 + 4
x0 = xg[ii] + 2
y0 = yg[ii] - 4
x1 = bet1 * x3 + (1 - bet1) * x0
y1 = bet1 * y3 + (1 - bet1) * y0 - dy1
x2 = bet2 * x3 + (1 - bet2) * x0
y2 = bet2 * y3 + (1 - bet2) * y0 - dy2
put(f" {x1:.1f} {y1:.1f} {x2:.1f} {y2:.1f}")
put(f" {x3:.1f} {y3:.1f} {x0:.1f} {y0:.1f} gsl\n")
def deadtree():
"""Deadtree is the main services machine. It can be taken down at any time and rebuilt."""
apt.sudo_ensure() # cuisine.package_ensure is broken otherwise
# Set up /etc/skel
sudo("mkdir /etc/skel/.ssh || true")
sudo("chmod 700 /etc/skel/.ssh")
# Add a 'nobody' user
user_ensure('nobody')
group_ensure('nobody')
group_user_ensure('nobody', 'nobody')
sudo('usermod -s /bin/false nobody')
# Set up the firewall
put("config/firewalls/deadtree.sh", "/usr/local/bin", use_sudo=True)
sudo("sh /usr/local/bin/deadtree.sh")
# Set up nginx
already_installed = nginx.ensure()
nginx.ensure_site('config/nginx/default', cert='config/certs/za3k.com.pem', key='config/keys/blog.za3k.com.key')
nginx.ensure_fcgiwrap(children=4)
if not already_installed:
nginx.restart() # IPv[46] listener only changes on restart
# Set up letsencrypt
letsencrypt.ensure()
# Set up authorization to back up to the data server
#public_key = ssh.ensure_key('/root/.ssh/id_rsa')
#with settings(user='******', host_string='burn'):
# #put(public_key, '/home/zachary/test_authorized_keys')
# files.append('/home/deadtree/.ssh/authorized_keys', public_key)
# ddns.za3k.com (TCP port 80, web updater for DDNS)
# ns.za3k.com (UDP port 53, DNS server)
user_ensure('nsd')
group_ensure('nsd')
group_user_ensure('nsd', 'nsd')
with mode_sudo():
dir_ensure('/var/lib/nsd', mode='755')
sudo("chown nsd:nsd /var/lib/nsd")
package_ensure(["nsd"])
with cd("/var/lib/nsd"):
sudo("touch /var/lib/nsd/moreorcs.com.zone && chown nsd:nsd /var/lib/nsd/moreorcs.com.zone")
node.ensure()
put("config/ddns/moreorcs.com.zonetemplate", "/etc/nsd", mode='644', use_sudo=True)
supervisord.ensure()
git.ensure_clone_github('thingless/ddns', '/var/lib/nsd/ddns', user='******')
supervisord.ensure_config("config/supervisor/ddns.conf")
put("config/ddns/config.json", "/var/lib/nsd", mode='644', use_sudo=True)
sudo("chown nsd:nsd /var/lib/nsd/config.json")
# [Manual] Copy dnsDB.json from backup
sudo("cd /var/lib/nsd && ln -sf ddns/index.txt index.txt && chown nsd:nsd index.txt")
supervisord.update() # Run ddns
package_ensure(["nsd"])
put("config/ddns/nsd.conf", "/etc/nsd", mode='644', use_sudo=True)
sudo("systemctl restart nsd")
nginx.ensure_site('config/nginx/ddns.za3k.com', csr='config/certs/ddns.za3k.com.csr', key='config/keys/ddns.za3k.com.key', domain="ddns.za3k.com", letsencrypt=True, cert="config/certs/ddns.za3k.com.pem")
nginx.reload()
# blog.za3k.com
package_ensure(["php5-fpm", "mysql-server", "php5-mysql"])
nginx.ensure_site('config/nginx/blog.za3k.com', cert='config/certs/blog.za3k.com.pem', key='config/keys/blog.za3k.com.key', domain="blog.za3k.com", letsencrypt=True, csr="config/certs/blog.za3k.com.csr")
git.ensure_clone_za3k('za3k_blog', '/var/www/za3k_blog', user='******')
# TODO: Replace a database-specific password or make it more obvious it's not used? Currently we're using user ACLs and this gets ignored anyway, I think?
# [Manual] Load the blog database from backup at /srv/mysql -> /var/lib/mysql
sudo('systemctl restart mysql')
# etherpad.za3k.com
package_ensure(["sqlite3"])
user_ensure('etherpad')
group_ensure('etherpad')
group_user_ensure('etherpad', 'etherpad')
git.ensure_clone_github('ether/etherpad-lite', '/var/www/etherpad', commit='1.6.0', user='******')
nginx.ensure_site('config/nginx/etherpad.za3k.com', csr='config/certs/etherpad.za3k.com.csr', key='config/keys/etherpad.za3k.com.key', domain="etherpad.za3k.com", letsencrypt=True, cert="config/certs/etherpad.za3k.com.pem")
util.put("config/etherpad/APIKEY.txt", "/var/www/etherpad", user='******', mode='600')
util.put("config/etherpad/settings.json", "/var/www/etherpad", user='******', mode='644')
if not files.exists("/var/www/etherpad/var/sqlite.db"):
sudo("mkdir -p /var/www/etherpad/var", user='******')
with cd("/var/www/etherpad"):
sudo("npm install sqlite3")
sudo("rsync -av burn.za3k.com::etherpad --delete /var/www/etherpad/var", user='******')
supervisord.ensure()
supervisord.ensure_config("config/supervisor/etherpad.conf")
supervisord.update()
# forsale
nginx.ensure_site('config/nginx/forsale')
util.put('data/forsale', '/var/www', user='******', mode='755')
# gipc daily sync
# github personal backup
# github repo list
# -> updater
# irc.za3k.com -> irc
# -> webchat (qwebirc)
# jsfail.com
user_ensure('jsfail')
group_ensure('jsfail')
group_user_ensure('jsfail', 'jsfail')
nginx.ensure_site('config/nginx/jsfail.com')
util.put('data/jsfail', '/var/www', 'jsfail', mode='755')
# library.za3k.com -> website
# -> sync script
# -> card catalog
# MUST be user 2001 to match remote rsync
user_ensure('library', uid=2001)
group_ensure('library', gid=2001)
group_user_ensure('library', 'library')
with mode_sudo():
dir_ensure('/var/www/library', mode='755')
sudo("chown library:library /var/www/library")
put("config/library/library.sync", "/etc/cron.daily", mode='755', use_sudo=True)
sudo("/etc/cron.daily/library.sync")
nginx.ensure_site('config/nginx/library.za3k.com', csr='config/certs/library.za3k.com.csr', key='config/keys/library.za3k.com.key', domain="library.za3k.com", letsencrypt=True, cert="config/certs/library.za3k.com.pem")
# logs (nginx) and analysis (analog)
# mint sync
# moreorcs.com
user_ensure('moreorcs')
group_ensure('moreorcs')
group_user_ensure('moreorcs', 'moreorcs')
nginx.ensure_site('config/nginx/moreorcs.com', cert='config/certs/moreorcs.com.pem', key='config/keys/moreorcs.com.key', domain="moreorcs.com", letsencrypt=True, csr="config/certs/moreorcs.com.csr")
git.ensure_clone_github('za3k/moreorcs', '/var/www/moreorcs', user='******')
# nanowrimo.za3k.com
nginx.ensure_site('config/nginx/nanowrimo.za3k.com', csr='config/certs/nanowrimo.za3k.com.csr', key='config/keys/nanowrimo.za3k.com.key', domain="nanowrimo.za3k.com", letsencrypt=True, cert="config/certs/nanowrimo.za3k.com.pem")
util.put('data/nanowrimo', '/var/www', user='******', mode='755')
# nntp.za3k.com - Discontinued
# petchat.za3k.com
nginx.ensure_site('config/nginx/petchat.za3k.com')
if not files.exists('/var/www/petchat'):
git.ensure_clone_za3k('petchat', '/var/www/petchat', user='******')
# publishing.za3k.com
# thinkingtropes.com
nginx.ensure_site('config/nginx/thinkingtropes.com')
util.put('data/thinkingtropes', '/var/www', user='******', mode='755')
# thisisashell.com
nginx.ensure_site('config/nginx/thisisashell.com', csr='config/certs/thisisashell.com.csr', key='config/keys/thisisashell.com.key', domain="thisisashell.com", letsencrypt=True, cert="config/certs/thisisashell.com.pem")
# twitter archive
# za3k.com
user_ensure('za3k')
group_ensure('za3k')
group_user_ensure('za3k', 'za3k')
nginx.ensure_site('config/nginx/za3k.com', cert='config/certs/za3k.com.pem', key='config/keys/za3k.com.key', domain="za3k.com", letsencrypt=True, csr="config/certs/za3k.com.csr")
git.ensure_clone_za3k('za3k', '/var/www/za3k', user='******')
# Markdown .md
ruby.ensure()
ruby.ensure_gems(["redcarpet"])
# colony on the moon
sudo("rsync -av burn.za3k.com::colony --delete /var/www/colony", user='******')
# |-- status.za3k.com
sudo("mkdir -p /var/www/status && chmod 755 /var/www/status")
util.put("/srv/keys/backup_check", "/var/www/status", user='******', mode='600')
#util.put("/srv/keys/backup_check.pub", "/var/www/status", user='******', mode='644')
package_ensure(["parallel", "curl"])
nginx.reload()
