def main():
db = Database(DATABASE)
db.update()
count = 1
total = len(db.tables)
for table in db.tables:
url = db.tables[table]['url']
timezone = db.tables[table]['timezone']
try:
site = BeautifulSoup(urlopen(url))
except urllib.error.HTTPError:
print("error on %s" % (url))
data = []
for tr in site('tr'):
raw_datum = tr.text.strip().split('\n')
try:
date = datetime.strptime(
raw_datum[0],
"%d/%m/%Y %H:%M").replace(tzinfo=pytz.timezone(timezone))
value = float(raw_datum[1])
datum = (date, value)
except ValueError:
continue
data.append(datum)
db.store_data(data, table)
sleep(5)
print("%d / %d is complete" % (count, total))
count += 1
def listen_query(seconds=10):
"""
records audio and prints if a match is found from the database
:param seconds:
:return:
"""
# open stream
au = pyaudio.PyAudio()
stream = au.open(format=pyaudio.paInt16,
channels=1,
rate=44100,
input=True,
frames_per_buffer=8192)
print("* recording")
query = []
for i in range(0, int(44100 / 8192 * seconds)):
data = stream.read(8192)
nums = np.fromstring(data, np.int16)
query.extend(nums[0::2])
print("* done recording")
# close and stop the stream
stream.stop_stream()
stream.close()
spec = bo.spectrogram(query, sample_rate=44100)
peaks = bo.get_peaks(spec)
hash_list = bo.gen_hash(peaks)
db_client = Database('fingerprints.db')
matches = db_client.fetch_result(hash_list)
print(matches)
def http_json(tables, start, end):
db = Database(DATABASE)
table_list = tables.split("+")
table_set = set()
table_data = dict()
for table in table_list:
if table in db.tables:
table_set.add(table)
name = get_cache_name(table, start, end)
cached = cache.get(name)
if cached is not None:
table_data[table] = cached
else:
data = get_data(db, table, start, end)
table_data[table] = data
cache.set(name, data)
# We don't fail if at least one table is found. While the client should
# never request an unknown table, it will not error if it doesn't receive
# a requested table, and will just draw those given.
if len(table_set) == 0:
abort(404)
return json.jsonify(table_data)
def runTest(self):
print "Experiment.runTest>"
## main objects
db = Database()
objects = db.getObjects()
building = buildings.Test()
building.tag(objects)
#building.check(objects)
## Identify tags and set watch to enabled
o19 = objects.getTag('point', 19).single()
o20 = objects.getTag('point', 20).single()
o21 = objects.getTag('point', 21).single()
test = [o19, o20, o21]
db.enablePoints(test)
## Schedule object.
t = db.now()
# db.scheduleObject(o19, t+1, 3, 51)
# db.scheduleObject(o20, t+1 , 2, 1)
# db.scheduleObject(o20, t+4 , 1, 1)
# db.scheduleObject(o20, t+7 , 1, 1)
# db.scheduleObject(o20, t+6 , 3, 1)
#
db.scheduleObject(o21, t + 1, 10, 1)
db.scheduleObject(o21, t + 3, 5, None)
db.close()
def runPughHall(self):
print "Experiment.runPughHall>"
## main objects
db = Database()
objects = db.getObjects()
building = buildings.PughHall()
building.tag(objects)
#building.check(objects)
## Identify tags and set watch to enabled
rooms = objects.getTag('room', ['104', '105'])
print rooms
db.enablePoints(rooms)
## Schedule object.
# t=db.now()
# db.scheduleObject(o20, t+1 , 2, 1)
# db.scheduleObject(o20, t+4 , 1, 1)
# db.scheduleObject(o20, t+7 , 1, 1)
# db.scheduleObject(o20, t+6 , 3, 1)
#
# db.scheduleObject(o21, t+2 , 5, 1)
db.close()
def choices(self):
"""Present available choices to choose from"""
run = Sub(self.parent, False) # False value is normal scan
run.create_sub_table()
user_input = input(USER_CHOICE)
while user_input != '0':
if user_input == '1':
t0 = time.time()
run.start_scan()
total = (time.time() - t0)
format_timer(total)
elif user_input == '2':
t0 = time.time()
run = Sub(self.parent, True) # Full scan
run.start_scan()
total = (time.time() - t0)
format_timer(total)
elif user_input == '3':
# Make the sorting case insensitive
ordered_blacklist = sorted(BLACKLIST, key=str.lower)
for line in ordered_blacklist:
print(line)
user_input_2 = input("\n1. Back to menu \n0. Exit ")
if user_input_2 == '1':
Menu().choices()
elif user_input_2 == '0':
sys.exit()
else:
print("Command not recognized, please try again... ")
elif user_input == '4':
Database().get_statistics()
user_input_2 = input("\n1. Back to menu \n0. Exit ")
if user_input_2 == '1':
Menu().choices()
elif user_input_2 == '0':
sys.exit()
else:
print("Command not recognized, please try again... ")
else:
print("Command not recognized, please try again... ")
user_input = input(USER_CHOICE)
def update_status():
db = Database()
nodes = mysql.select_nodes()
db.delete('nodes')
for node in nodes:
arg = '%s\ %s\ %s' % (node[2], node[3], node[4])
r = os.system('./app/is_online.sh ' + arg)
if r == 0:
db.write('nodes', node[3])
return '0'
def home():
db = Database(DATABASE)
template = env.get_template('graph.html')
return template.render(
title="Reservoir Levels",
#tables=[db.tables[table] for table in db.tablesl],
tables=db.tables,
areas=db.areas,
states=db.states,
static_dir="static")
def runTest(self):
print "Virtual.runTest>"
db = Database("baclog", 5432)
db.getObjects()
av1 = db.instance[(9001, 2, 0)] ## hard coded AV1
print av1
db.enablePoints([av1])
t = db.now()
db.scheduleObject(av1, t + 1, 30, 10)
db.close()
def from_file(sec=6):
files = [file for file in glob.glob('**/*.wav', recursive=True)]
fi = random.randint(0, len(files) - 1)
samples, sample_rate = lb.load(files[fi], sr=None)
len_sample_start = random.randint(0, len(samples) - 1 - sample_rate * sec)
len_sample_stop = len_sample_start + sample_rate * sec
print(len_sample_start)
print(len_sample_stop)
sample_query = samples[len_sample_start:len_sample_stop]
spec = bo.spectrogram(samples=sample_query, sample_rate=sample_rate)
peaks = bo.get_peaks(spec)
hash_list = bo.gen_hash(peaks)
db_client = Database('fingerprints.db')
match = db_client.fetch_result(hash_list)
print(match)
def enablePughHall(self):
print "Experiment.enablePughHall>"
## main objects
db = Database()
objects = db.getObjects()
building = buildings.PughHall()
building.tag(objects)
#building.check(objects)
## Identify tags and set watch to enabled
rooms = objects.getTag('room', ['104', '105'])
print rooms
db.enablePoints(rooms)
db.close()
def _get_labeled(self, pseudo_count):
db = Database(fullpath("data/data.db"))
data = Data(mediatype="movie")
titles = data.titles_list()
labeled = []
try:
select_existing = db.select(table="main", fields="all")
except OperationalError:
data.collect(args=data.titles_list(),
addtl_items=["boxoffice", "oscars", "review"])
i = 0
for movie in labeled:
if i == 0:
print("Making Table main")
db.make_table(table="main",
fields=[k for k in movie.keys()])
else:
pass
db.update(row=movie)
db.conn.commit()
i += 1
else:
if len(select_existing) == len(titles):
header = select_existing[0].keys()
for x in select_existing:
labeled.append(dict(map(lambda x, y: (x, y), header, x)))
else:
data.collect(args=data.titles_list(),
addtl_items=["boxoffice", "oscars", "review"])
i = 0
for movie in labeled:
if i == 0:
print("Making Table main")
db.make_table(table="main",
fields=[k for k in movie.keys()])
else:
pass
db.update(row=movie)
db.conn.commit()
i += 1
lab = self._pseudo_cases(data=labeled, n=pseudo_count, classx="0")
return lab
def run(self):
print "Tag.run>"
## main objects
db = Database(database='mtim', port=5432)
## Series 1
for d in [1, 2, 3, 4]:
objects = db.getObjects(where="Devices.deviceID=%s" % d)
building = buildings.PughHall()
building.tag(objects)
building.check(objects)
## Setup Metadata
db.writeTags(objects)
building.points(objects)
db.writePoints(objects)
## Series 2
for d in [5, 6, 7, 8]:
objects = db.getObjects(where="Devices.deviceID=%s" % d)
building = buildings.PughHall()
building.tag(objects)
building.check(objects)
## Setup Metadata
db.writeTags(objects)
## Join on nn
db.execute("DELETE FROM PointObjectMap")
db.execute("""
INSERT INTO PointObjectMap (pointID,objectID,name)
SELECT Points.pointID,Objects.objectID,Points.value
FROM Objects
JOIN Tags ON (Objects.objectID=Tags.objectID AND tag='nn')
JOIN Points ON (Tags.value=Points.value AND Points.tag='nn')
""")
db.close()
def mainLoop():
dataFile = Database()
# main menu
while True:
clearScreen()
print('''
C.I.Pedersen
Timeregistrering
''')
if dataFile.getUserName() == 'main':
print('\tIngen bruker valgt\n')
else:
print(f'\tValgt bruker: {dataFile.getUserName()}\n')
print('\t1. Velg bruker\n\t2. Registrer Arbeid' +
'\n\t3. Fjern Arbeid\n\t4. Legg til bruker' +
'\n\t5. Fjern Bruker\n\t0. Avslutt')
mainMenu = input('\tVelg: ')
try:
if mainMenu == '1':
clearScreen()
while True:
print('\n\tVelg bruker fra listen')
dataFile.showUsers()
print('\n\t0 Gå tilbake')
userID = input('\tSkriv inn id: ')
if userID == '0':
break
# returns None of no id was found
userName = dataFile.choseUser(userID)
if userName == None:
if userID == '':
clearScreen()
continue
else:
print(f'\n\tIngen bruker med id nr {userID}')
input('\tTrykk Enter for å gå videre')
clearScreen()
else:
dataFile = Database(userName)
break
elif mainMenu == '2':
clearScreen()
if dataFile.getUserName() == 'main':
input(
'\n\tVelg en bruker først\n\tTrykk Enter for å gå tilbake'
)
continue
dataFile.addWork()
elif mainMenu == '4':
clearScreen()
if dataFile.getUserName() == 'main':
input(
'\n\tVelg en bruker først\n\tTrykk Enter for å gå tilbake'
)
continue
print('\n\tEksisterende brukere\n')
dataFile.showUsers()
dataFile.addUser()
elif mainMenu == '5':
clearScreen()
if dataFile.getUserName() == 'main':
input(
'\n\tVelg en bruker først\n\tTrykk Enter for å gå tilbake'
)
continue
print('\n\tFjern bruker fra listen\n')
while True:
dataFile.showUsers()
print('\n\t0 Gå tilbake')
choice = input('\tSkriv: ')
if choice == '0':
clearScreen()
break
else:
dataFile.removeUser(choice)
break
elif mainMenu == '3':
clearScreen()
if dataFile.getUserName() == 'main':
input(
'\n\tVelg en bruker først\n\tTrykk Enter for å gå tilbake'
)
continue
dataFile.removeWork()
elif mainMenu == '0':
clearScreen()
exit()
except ValueError: # ignore all invalid values
pass
def __init__(self):
self.db = Database()
self.dl = html_down()
self.pr = parse()
import base64
import imghdr
from flask import render_template, request, session, redirect, url_for, flash
from data import Database
from forms import RegisterForm, PlaylistForm, SongForm, UpdateUser
from functools import wraps
from user import User
from passlib.hash import sha256_crypt
# initialize database class
db = Database()
# home page
def home():
playlists = db.get_public_playlists()
return render_template('home.html', playlists=playlists)
def user_check(playlistid):
playlist = db.get_playlist(playlistid)
if playlist is None or not ('logged_in' in session):
return False
elif playlist['userid'] == session['id']:
return True
else:
return False
def is_logged_in(f):
@wraps(f)
from data import Database
from random import shuffle
from neural_network import Network, loss
from copy import deepcopy
db = Database("iris_data.txt")
nn = Network((4, 5, 3))
verbose = True
num_folds = 10
num_reports = 4
def n_fold(n):
"""Should return a list of tuples of training and testing data for every
run"""
folds = list()
training_size = len(db.rows) // n
for fold in range(n):
start = training_size * fold
end = training_size * fold + training_size
testing = db.rows[start:end]
training = db.rows[:start] + db.rows[end:]
folds.append((training, testing))
return folds
def run(num_epoch, training, testing):
# Find a better place for this stuff
epochs_per_report = num_epoch // num_reports
min_loss = 5000
#w2v = Word2Vec(db)
#w2v.train(workers = 64)
#w2v.save("dump/w2v-zhwiki.dump")
#lda = LDA(db)
#lda.train()
#lda.save("dump/lda2-zhwiki.dump")
@function_timer
def t2wv(db) :
return dict((attrib["title"], model[jieba.cut(text)]) for text, attrib in db)
if __name__ == "__main__" :
db = Database("data/zhwiki-extracted/", conditions = [
Database.cond_length(50),
Database.cond_title(lambda t: not t.startswith("Wikipedia:")),
Database.cond_title(lambda t: not t.startswith("File:")),
Database.cond_title(lambda t: not t.startswith("Draft:"))
])
# 767125 loaded, 451657 filtered, 31997 fails
model = Method.load("dump/lda-model.dump")
title2topic = t2wv(db)
pickle.dump(title2topic, open("dump/title2topic.dump", "w"))
#title2wv = pickle.load(open("dump/title2wv.dump"))
vectors = []
id2title = {}
for i, (t, v) in enumerate(title2topic.items()) :
vectors.append(v)
id2title[i] = t
#from download import Downloader
from data import Database
from matplotlib.ticker import MultipleLocator
import matplotlib.pyplot as plt
import numpy as np
if __name__ == '__main__':
#选做任务1：年末总人口+年末男性总人口+年末女性总人口
database1 = Database('summary.db', 'A030101_sj', [], [], {},
'[{"wdcode":"sj","valuecode":"LAST20"}]')
database1.save()
database2 = Database('man.db', 'A030102_sj', [], [], {},
'[{"wdcode":"sj","valuecode":"LAST20"}]')
database2.save()
database3 = Database('woman.db', 'A030103_sj', [], [], {},
'[{"wdcode":"sj","valuecode":"LAST20"}]')
database3.save()
malerate = []
femalerate = []
for i in range(20):
malerate.append(
float(database2.data_list[i] / database1.data_list[i]) * 100)
femalerate.append(
float(database3.data_list[i] / database1.data_list[i]) * 100)
# 必做题目:查询森林火灾次数+严重程度
#森林火灾次数
database4 = Database('fire.db', 'A0C0E01_sj', [], [], {},
'[{"wdcode":"zb","valuecode":"A0C0E"}]')
database4.save()
db1_name = 'stock'
db2_name = 'stock_prophet'
def row_to_market(row):
return {'id': row[0], 'name': row[1], 'link_name': row[2]}
def row_to_company(row):
return {
'id': row[0],
'name': row[1],
'link_name': row[2],
'code': row[3],
'id_market': row[4]
}
db1 = Database(db_name=db1_name)
db2 = Database(db_name=db2_name)
query = db1.markets.select()
markets = db1.engine.execute(query)
markets = list(map(row_to_market, markets))
db2.engine.execute(db2.markets.insert().values(markets))
query = db1.companies.select()
companies = db1.engine.execute(query)
companies = list(map(row_to_company, companies))
db2.engine.execute(db2.companies.insert().values(companies))
def run(self):
print "Analysis.run>"
## main objects
db=Database()
graph=Graph()
## populate meta information
#devices=data.getDevices()
objects=db.getObjects()
building=buildings.PughHall()
building.tag(objects)
#building.check(objects)
## Compute streams
data=Data('data')
vav={}
ahu={}
total={}
plot=Plot("plot")
## Connections
source=Connection('source') # data connection
dest={} ## VAV connections
output=Connection("output")
## Connect
source.addIn(data)
## Build network
ahus=[1,3]
for ah in ahus:
zone=objects.getTag('zone',ah) ## All zone objects (VAV and AHU)
data.addOut(zone)
vavs=zone.getValues('vav')
total[ah]=Total("total%d" % ah,vavs)
dest[ah]=Connection("dest%d" % ah)
dest[ah].addOut(total[ah]) ## zone totalizer
a=AHU("ahu%d" % ah, ah,objects.getTag('ahu',ah))
ahu[ah]=a
source.addOut(a)
for v in vavs:
zv=VAV(('zone%d-VAV%s') % (ah,v), zone.getTag('vav',v), objects.getTag('ahu',ah)) ## Zone stream
vav[v]=zv
source.addOut(zv)
dest[ah].addIn(zv)
## Per ahu plots
output.addIn(total[ah])
## add trace points
output.addIn(ahu[1])
output.addIn(vav[114])
## connect plot (last)
output.addOut(plot)
if trace:
print "Analysis.run>", repr(source)
print "Analysis.run>", repr(dest)
print "Analysis.run>", repr(total)
print "Analysis.run>", repr(plot)
## Process DataStream
#limit="WHERE time >= '2011-09-27 20:00' AND time <= '2011-09-27 23:00'"
#limit="WHERE time >= '2011-09-27 20:00' AND time <= '2011-09-28 07:00'"
#limit="WHERE time >= '2011-09-27' AND time <= '2011-09-29'"
limit=None
## Debug
monitor=InstanceList()
#monitor.add(objects.getTags({'descriptor':'SAIR-TE'}))
#monitor.add(objects.getTags({'descriptor':'RAIR-H'}))
#monitor=monitor.getTag('ahu',1)
#monitor.add(objects.getTags({'vav':114,'descriptor':'CTL TEMP'}))
#monitor.add(objects.getTags({'vav':114,'descriptor':'FLOW'}))
monitor.add(objects.getTags({'vav':114,'descriptor':'HTG LOOPOUT'}))
#monitor.add(objects.getTags({'vav':114}))
## Stream compute
i=0;
for time,device,otype,oinstance,value in db.getData(limit):
v=Value(db.getInstance(device,otype,oinstance),value,time,0) ## Build value
data.send(v) ## input data
## Debug
if v.var in monitor:
print "DATA:", time, v
if i%100000==0:
print "TICK:", i,time
i+=1
## Plot
#limit=None
#limit=['total1-qsum','total3-qsum']
limit=['sat','sa']
graph.add(plot,limit)
graph.run()
from tkinter import *
from data import Database
database=Database("books.db")
def get_selected_row(event):
global selected_tuple
index=list1.curselection()[0]
selected_tuple=list1.get(index)
e1.delete(0,END)
e1.insert(END,selected_tuple[1])
e2.delete(0,END)
e2.insert(END,selected_tuple[2])
e3.delete(0,END)
e3.insert(END,selected_tuple[3])
e4.delete(0,END)
e4.insert(END,selected_tuple[4])
def view_command():
list1.delete(0, END)
for i in database.view():
list1.insert(END, i)
def search_command():
list1.delete(0, END)
for i in database.search(title_text.get(),author_text.get(),year_text.get(),isbn_text.get()):
list1.insert(END, i)
def add_command():
database.insert(title_text.get(),author_text.get(),year_text.get(),isbn_text.get())
list1.delete(0, END)
import basic_ops as bo
from data import Database
import glob
import librosa as lb
import pyaudio
import scipy.io.wavfile as sw
import random
import numpy as np
db_client = Database('fingerprints.db')
def upload():
"""
uploads fingerprints to database
:return:
"""
path = 'data/'
total_hashes = 0
for files in glob.glob(path + '*.wav'):
samples, sample_rate = lb.load(path=files, sr=None)
spec = bo.spectrogram(samples=samples, sample_rate=sample_rate)
peaks = bo.get_peaks(spec)
song_id = files.replace('-', ' ').replace('_', ' ').strip()[5:-4]
hash_list = bo.gen_hash(peaks, song_id=song_id)
for hash_item in hash_list:
db_client.insert_fingerprint(hash_item)
print(hash_item)
total_hashes += len(hash_list)
print('TOTAL FINGERPRINTS INSERTED: {}'.format(total_hashes))
#!/usr/bin/env python
from tkinter import *
from tkinter import messagebox
from tkinter import ttk
from data import Database
import matplotlib.pyplot as plt
import numpy as np
db = Database('applications.db')
# Defines global variable for buttons
global application_status
application_status = ''
# Creates style object
# style = ttk.Style()
def populate_list():
applications_list.delete(0, END)
for row in db.fetch():
applications_list.insert(END, row)
def add_active():
global application_status
application_status = "Active"
def add_reject():
global application_status
def __init__(self):
self.database = Database()
self.database.create_tables()
self.message_response = {}
from argparse import ArgumentParser
from extract import Extract
from data import Database
parser = ArgumentParser(
description='Import old workouts to workoutlog.bak sqlite file.')
parser.add_argument('spreadsheet',
help='Spreadsheet with old workouts to import.')
parser.add_argument('--sqlite_file',
help='Sqlite file to import old workouts into.',
default='./workoutlog.bak')
args = parser.parse_args()
oldworkouts = Extract.getoldworkouts(args.spreadsheet)
with Database(args.sqlite_file) as db:
db.loadworkouts(oldworkouts)
