def listen_query(seconds=10):
"""
records audio and prints if a match is found from the database
:param seconds:
:return:
"""
# open stream
au = pyaudio.PyAudio()
stream = au.open(format=pyaudio.paInt16,
channels=1,
rate=44100,
input=True,
frames_per_buffer=8192)
print("* recording")
query = []
for i in range(0, int(44100 / 8192 * seconds)):
data = stream.read(8192)
nums = np.fromstring(data, np.int16)
query.extend(nums[0::2])
print("* done recording")
# close and stop the stream
stream.stop_stream()
stream.close()
spec = bo.spectrogram(query, sample_rate=44100)
peaks = bo.get_peaks(spec)
hash_list = bo.gen_hash(peaks)
db_client = Database('fingerprints.db')
matches = db_client.fetch_result(hash_list)
print(matches)
def main():
db = Database(DATABASE)
db.update()
count = 1
total = len(db.tables)
for table in db.tables:
url = db.tables[table]['url']
timezone = db.tables[table]['timezone']
try:
site = BeautifulSoup(urlopen(url))
except urllib.error.HTTPError:
print ("error on %s" %(url))
data = []
for tr in site('tr'):
raw_datum = tr.text.strip().split('\n')
try:
date = datetime.strptime(raw_datum[0], "%d/%m/%Y %H:%M").replace(
tzinfo=pytz.timezone(timezone)
)
value = float(raw_datum[1])
datum = (date, value)
except ValueError:
continue
data.append(datum)
db.store_data(data, table)
sleep(5)
print ("%d / %d is complete" %(count, total))
count += 1
def save(self):
if (self.name is not None):
if hasattr(self, "_id"):
Database.replace(Database.COLLECTIONS.BOT, self.toDict())
else:
self._id = Database.insert(Database.COLLECTIONS.BOT,
self.toDict())
else:
raise Exception("Bot can't be saved without name.")
def save(self):
if (self.text is not None):
if hasattr(self, "_id"):
Database.replace(Database.COLLECTIONS.SENTENCE, self.to_dict())
else:
self._id = Database.insert(Database.COLLECTIONS.SENTENCE,
self.to_dict())
self.data = self.to_dict()
else:
raise Exception("Category can't be saved without name.")
def runTest(self):
print "Virtual.runTest>"
db=Database("baclog",5432)
db.getObjects()
av1=db.instance[(9001,2,0)] ## hard coded AV1
print av1
db.enablePoints([av1])
t=db.now()
db.scheduleObject(av1, t+1 , 30, 10)
db.close()
def save(self):
if(self.name is not None):
self.sentence_ids = []
for s in self.sentences:
s.save()
self.sentence_ids.append(s._id)
if hasattr(self, "_id"):
Database.replace(Database.COLLECTIONS.CATEGORY, self.toDict())
else:
self._id = Database.insert(Database.COLLECTIONS.CATEGORY, self.toDict())
self.isSaved = True
self.data = self.toDict()
else:
raise Exception("Category can't be saved without name.")
def save(self):
if self.cfg.dryrun: return
if self.cfg.gxpmode:
# Gather results
self.send_res()
if self.gxp.rank == 0:
reslist = []
for i in range(0, self.gxp.size):
reslist.append(self.recv_res())
else: return
if self.cfg.logdir is None: # generate random logdir in cwd
self.cfg.logdir = os.path.abspath("./pmlog-%s-%s" %
(self.runtime.user, time.strftime("%j-%H-%M-%S")))
# Initial log directory and database
if self.cfg.gxpmode:
self.cfg.confirm = False
self.cfg.logdir = smart_makedirs(self.cfg.logdir,
self.cfg.confirm)
logdir = os.path.abspath(self.cfg.logdir)
if self.cfg.nolog:
message("Saving data in memory ...")
else:
message("Saving data to %s/fsbench.db ..." % logdir)
# Save used configuration file
if not self.cfg.nolog:
verbose("Saving configurations to %s/fsbench.conf ..."
% logdir, VERBOSE)
self.opts.save_conf("%s/fsbench.conf" % logdir)
# Save results
if self.cfg.nolog: self.db = Database(":memory:")
else: self.db = Database("%s/fsbench.db" % logdir)
self.db.insert_runtime(self.runtime)
self.db.insert_conf(self.opts.cfgParser)
if self.cfg.gxpmode:
for res in reslist:
for r in res: self.db.insert_rawdata(r)
else:
for t in self.threads:
self.db.insert_rawdata(t.get_res())
self.db.commit()
if self.cfg.noreport: self.db.close()
def from_file(sec=6):
files = [file for file in glob.glob('**/*.wav', recursive=True)]
fi = random.randint(0, len(files) - 1)
samples, sample_rate = lb.load(files[fi], sr=None)
len_sample_start = random.randint(0, len(samples) - 1 - sample_rate * sec)
len_sample_stop = len_sample_start + sample_rate * sec
print(len_sample_start)
print(len_sample_stop)
sample_query = samples[len_sample_start:len_sample_stop]
spec = bo.spectrogram(samples=sample_query, sample_rate=sample_rate)
peaks = bo.get_peaks(spec)
hash_list = bo.gen_hash(peaks)
db_client = Database('fingerprints.db')
match = db_client.fetch_result(hash_list)
print(match)
def http_json(tables, start, end):
db = Database(DATABASE)
table_list = tables.split("+")
table_set = set()
table_data = dict()
for table in table_list:
if table in db.tables:
table_set.add(table)
name = get_cache_name(table, start, end)
cached = cache.get(name)
if cached is not None:
table_data[table] = cached
else:
data = get_data(db, table, start, end)
table_data[table] = data
cache.set(name, data)
# We don't fail if at least one table is found. While the client should
# never request an unknown table, it will not error if it doesn't receive
# a requested table, and will just draw those given.
if len(table_set) == 0:
abort(404)
return json.jsonify(table_data)
def __load__data__(self, data):
if(isinstance(data, Category)):
if hasattr(data, "_id"):
self._id = data._id
self.name = data.name
if hasattr(data, "domain"):
self.domain = data.domain
self.patterns = data.patterns
self.sentences = data.sentences
self.sentence_ids = data.sentence_ids
else:
self._id = data["_id"]
self.name = data["name"]
self.domain = data.get("domain")
if self.domain is None: self.domain = "none"
patterns = data.get("patterns")
if patterns is None: patterns = []
for pat in patterns:
self.patterns.append(pat)
self.sentence_ids = data.get("sentences")
self.sentences = []
if self.sentence_ids is None:
self.sentence_ids = []
for id in self.sentence_ids:
data = Database.find_one(Database.COLLECTIONS.SENTENCE, {"_id": id})
s = Sentence()
s.load_from_dict(data)
self.sentences.append(s)
def main():
try:
Database.initialize()
except Exception as e:
print(e)
return
app = QApplication(sys.argv)
app.setWindowIcon(QtGui.QIcon('logo.png'))
c = MainController()
w = MainWindow()
adapter = Adapter(c, w)
w.show()
sys.exit(app.exec_())
def choices(self):
"""Present available choices to choose from"""
run = Sub(self.parent, False) # False value is normal scan
run.create_sub_table()
user_input = input(USER_CHOICE)
while user_input != '0':
if user_input == '1':
t0 = time.time()
run.start_scan()
total = (time.time() - t0)
format_timer(total)
elif user_input == '2':
t0 = time.time()
run = Sub(self.parent, True) # Full scan
run.start_scan()
total = (time.time() - t0)
format_timer(total)
elif user_input == '3':
# Make the sorting case insensitive
ordered_blacklist = sorted(BLACKLIST, key=str.lower)
for line in ordered_blacklist:
print(line)
user_input_2 = input("\n1. Back to menu \n0. Exit ")
if user_input_2 == '1':
Menu().choices()
elif user_input_2 == '0':
sys.exit()
else:
print("Command not recognized, please try again... ")
elif user_input == '4':
Database().get_statistics()
user_input_2 = input("\n1. Back to menu \n0. Exit ")
if user_input_2 == '1':
Menu().choices()
elif user_input_2 == '0':
sys.exit()
else:
print("Command not recognized, please try again... ")
else:
print("Command not recognized, please try again... ")
user_input = input(USER_CHOICE)
def runPughHall(self):
print "Experiment.runPughHall>"
## main objects
db = Database()
objects = db.getObjects()
building = buildings.PughHall()
building.tag(objects)
#building.check(objects)
## Identify tags and set watch to enabled
rooms = objects.getTag('room', ['104', '105'])
print rooms
db.enablePoints(rooms)
## Schedule object.
# t=db.now()
# db.scheduleObject(o20, t+1 , 2, 1)
# db.scheduleObject(o20, t+4 , 1, 1)
# db.scheduleObject(o20, t+7 , 1, 1)
# db.scheduleObject(o20, t+6 , 3, 1)
#
# db.scheduleObject(o21, t+2 , 5, 1)
db.close()
def removeEntry(entry):
pos = Entries.entries.index(entry)
if (entry in Entries.unsaved_entries):
Entries.unsaved_entries.remove(entry)
Entries.entries.remove(entry)
if hasattr(entry, "_id"):
Database.delete_one(Database.COLLECTIONS.CATEGORY,
{"_id": entry._id})
if (entry is Entries.currentEntry):
if ((pos - 1) < 0):
if (len(Entries.entries) > 0):
Entries.setCurrentEntry(Entries.entries[0])
else:
Entries.newEntry()
else:
Entries.setCurrentEntry(Entries.entries[pos - 1])
Entries._notify(Entries.UPDATE.ALL)
def home():
db = Database(DATABASE)
template = env.get_template('graph.html')
return template.render(
title="Reservoir Levels",
#tables=[db.tables[table] for table in db.tablesl],
tables=db.tables,
areas=db.areas,
states=db.states,
static_dir="static")
def graph():
# Imports counted values from Database
total = Database.count_total(db)
active = Database.count_active(db)
rejected = Database.count_rejected(db)
accepted = Database.count_accepted(db)
# X-Axis Labels
x_labels = np.array(['Total', 'Active', 'Rejected', 'Accepted'])
# Y-Axis Values
y_axis = np.array([total, active, rejected, accepted])
w = 4
nitems = len(y_axis)
x_axis = np.arange(0, nitems * w, w) # Sets an array of x-coordinates
fig, ax = plt.subplots(1)
ax.bar(x_axis, y_axis, width=w, align='center')
ax.set_xticks(x_axis)
ax.set_xticklabels(x_labels, rotation=90)
plt.show()
def __init__(self, path):
self.db = Database("%s/trace.sqlite" % path)
self.path = "%s/figures" % path
if not os.path.exists(self.path):
utils.smart_makedirs(self.path)
self.ptree = None
self.COLORS = ["blue", "yellow", "red", "green"]
self.N_COLORS = len(self.COLORS)
self.c = Gnuplot.Gnuplot()
self.terminal = "png"
def runPughHall(self):
print "Experiment.runPughHall>"
## main objects
db=Database()
objects=db.getObjects()
building=buildings.PughHall()
building.tag(objects)
#building.check(objects)
## Identify tags and set watch to enabled
rooms=objects.getTag('room',['104','105'])
print rooms
db.enablePoints(rooms)
## Schedule object.
# t=db.now()
# db.scheduleObject(o20, t+1 , 2, 1)
# db.scheduleObject(o20, t+4 , 1, 1)
# db.scheduleObject(o20, t+7 , 1, 1)
# db.scheduleObject(o20, t+6 , 3, 1)
#
# db.scheduleObject(o21, t+2 , 5, 1)
db.close()
def main():
db = Database(DATABASE)
db.update()
count = 1
total = len(db.tables)
for table in db.tables:
url = db.tables[table]['url']
timezone = db.tables[table]['timezone']
try:
site = BeautifulSoup(urlopen(url))
except urllib.error.HTTPError:
print("error on %s" % (url))
data = []
for tr in site('tr'):
raw_datum = tr.text.strip().split('\n')
try:
date = datetime.strptime(
raw_datum[0],
"%d/%m/%Y %H:%M").replace(tzinfo=pytz.timezone(timezone))
value = float(raw_datum[1])
datum = (date, value)
except ValueError:
continue
data.append(datum)
db.store_data(data, table)
sleep(5)
print("%d / %d is complete" % (count, total))
count += 1
def loadBots():
bots = Database.find(Database.COLLECTIONS.BOT, {})
Bots.__bots__ = []
for document in bots:
bot = Bot(document["name"])
bot.load(document)
Bots.__bots__.append(bot)
if not any(bot.name == Bots.GENERAL_NAME_TAG for bot in Bots.__bots__):
bot = Bot(Bots.GENERAL_NAME_TAG)
bot.save()
Bots.__bots__.insert(0, bot)
Bots._notify()
def loadEntries(bot):
Entries.entries = []
if hasattr(bot, "_id"):
categories = Database.find(Database.COLLECTIONS.CATEGORY,
{"bot": bot._id})
for data in categories:
c = Category(bot, data)
Entries.entries.append(c)
if (len(Entries.entries) > 0):
Entries.currentEntry = Entries.entries[0]
else:
Entries.currentEntry = Entries.newEntry(bot)
Entries._notify(Entries.UPDATE.ALL)
def update_status():
db = Database()
nodes = mysql.select_nodes()
db.delete('nodes')
for node in nodes:
arg = '%s\ %s\ %s' % (node[2], node[3], node[4])
r = os.system('./app/is_online.sh ' + arg)
if r == 0:
db.write('nodes', node[3])
return '0'
def enablePughHall(self):
print "Experiment.enablePughHall>"
## main objects
db = Database()
objects = db.getObjects()
building = buildings.PughHall()
building.tag(objects)
#building.check(objects)
## Identify tags and set watch to enabled
rooms = objects.getTag('room', ['104', '105'])
print rooms
db.enablePoints(rooms)
db.close()
def enablePughHall(self):
print "Experiment.enablePughHall>"
## main objects
db=Database()
objects=db.getObjects()
building=buildings.PughHall()
building.tag(objects)
#building.check(objects)
## Identify tags and set watch to enabled
rooms=objects.getTag('room',['104','105'])
print rooms
db.enablePoints(rooms)
db.close()
def runTest(self):
print "Virtual.runTest>"
db = Database("baclog", 5432)
db.getObjects()
av1 = db.instance[(9001, 2, 0)] ## hard coded AV1
print av1
db.enablePoints([av1])
t = db.now()
db.scheduleObject(av1, t + 1, 30, 10)
db.close()
import base64
import imghdr
from flask import render_template, request, session, redirect, url_for, flash
from data import Database
from forms import RegisterForm, PlaylistForm, SongForm, UpdateUser
from functools import wraps
from user import User
from passlib.hash import sha256_crypt
# initialize database class
db = Database()
# home page
def home():
playlists = db.get_public_playlists()
return render_template('home.html', playlists=playlists)
def user_check(playlistid):
playlist = db.get_playlist(playlistid)
if playlist is None or not ('logged_in' in session):
return False
elif playlist['userid'] == session['id']:
return True
else:
return False
def is_logged_in(f):
@wraps(f)
def load():
domains = Database.find(Database.COLLECTIONS.DOMAIN, {})
Domains.__domains__ = []
for document in domains :
Domains.__domains__ .append(document["name"])
Domains._notify()
def save():
Database.delete_all(Database.COLLECTIONS.DOMAIN)
for domain in Domains.__domains__:
Database.insert(Database.COLLECTIONS.DOMAIN, {"name": domain})
#from download import Downloader
from data import Database
from matplotlib.ticker import MultipleLocator
import matplotlib.pyplot as plt
import numpy as np
if __name__ == '__main__':
#选做任务1：年末总人口+年末男性总人口+年末女性总人口
database1 = Database('summary.db', 'A030101_sj', [], [], {},
'[{"wdcode":"sj","valuecode":"LAST20"}]')
database1.save()
database2 = Database('man.db', 'A030102_sj', [], [], {},
'[{"wdcode":"sj","valuecode":"LAST20"}]')
database2.save()
database3 = Database('woman.db', 'A030103_sj', [], [], {},
'[{"wdcode":"sj","valuecode":"LAST20"}]')
database3.save()
malerate = []
femalerate = []
for i in range(20):
malerate.append(
float(database2.data_list[i] / database1.data_list[i]) * 100)
femalerate.append(
float(database3.data_list[i] / database1.data_list[i]) * 100)
# 必做题目:查询森林火灾次数+严重程度
#森林火灾次数
database4 = Database('fire.db', 'A0C0E01_sj', [], [], {},
'[{"wdcode":"zb","valuecode":"A0C0E"}]')
database4.save()
def run(self):
print "Tag.run>"
## main objects
db=Database(database='mtim',port=5432)
## Series 1
for d in [1,2,3,4]:
objects=db.getObjects(where="Devices.deviceID=%s" % d)
building=buildings.PughHall()
building.tag(objects)
building.check(objects)
## Setup Metadata
db.writeTags(objects)
building.points(objects)
db.writePoints(objects)
## Series 2
for d in [5,6,7,8]:
objects=db.getObjects(where="Devices.deviceID=%s" % d)
building=buildings.PughHall()
building.tag(objects)
building.check(objects)
## Setup Metadata
db.writeTags(objects)
## Join on nn
db.execute("DELETE FROM PointObjectMap")
db.execute(
"""
INSERT INTO PointObjectMap (pointID,objectID,name)
SELECT Points.pointID,Objects.objectID,Points.value
FROM Objects
JOIN Tags ON (Objects.objectID=Tags.objectID AND tag='nn')
JOIN Points ON (Tags.value=Points.value AND Points.tag='nn')
"""
)
db.close()
def __init__(self):
self.database = Database()
self.database.create_tables()
self.message_response = {}
db1_name = 'stock'
db2_name = 'stock_prophet'
def row_to_market(row):
return {'id': row[0], 'name': row[1], 'link_name': row[2]}
def row_to_company(row):
return {
'id': row[0],
'name': row[1],
'link_name': row[2],
'code': row[3],
'id_market': row[4]
}
db1 = Database(db_name=db1_name)
db2 = Database(db_name=db2_name)
query = db1.markets.select()
markets = db1.engine.execute(query)
markets = list(map(row_to_market, markets))
db2.engine.execute(db2.markets.insert().values(markets))
query = db1.companies.select()
companies = db1.engine.execute(query)
companies = list(map(row_to_company, companies))
db2.engine.execute(db2.companies.insert().values(companies))
class Bench:
def __init__(self, opts):
self.opts = opts
self.cfg = self.opts.vals
self.runtime = Values()
self.runtime.version = version.PARAMARK_VERSION
self.runtime.date = version.PARAMARK_DATE
self.runtime.uid = os.getuid()
self.runtime.pid = os.getpid()
self.runtime.user = pwd.getpwuid(os.getuid())[0]
self.runtime.hostname = socket.gethostname()
self.runtime.platform = " ".join(os.uname())
self.runtime.cmdline = " ".join(sys.argv)
self.runtime.mountpoint = get_filesystem_info(self.cfg.wdir)
self.runtime.wdir = self.cfg.wdir
# May be set later in GXP mode
self.runtime.hid = 0
self.runtime.nhosts = 1
self.cfg.hid = self.runtime.hid
self.cfg.pid = self.runtime.pid
self.loader = BenchLoad(self.cfg)
self.threads = []
self.db = None
self.gxp = None
def load(self):
if self.cfg.gxpmode:
self.gxp = Values()
self.gxp.wp = os.fdopen(3, "wb")
self.gxp.rp = os.fdopen(4, "rb")
self.gxp.rank = gxp.get_rank()
self.gxp.size = gxp.get_size()
self.runtime.hid = self.gxp.rank
self.runtime.nhosts = self.gxp.size
self.cfg.hid = self.runtime.hid
self.threadsync = ThreadSync(self.cfg.nthreads)
for i in range(0, self.cfg.nthreads):
self.threads.append(BenchThread(i, self.threadsync,
self.loader, self.gxp))
def run(self):
if self.runtime.hid == 0:
message("Start benchmarking ...")
self.start = timer()
for t in self.threads: t.start()
for t in self.threads: t.join()
self.end = timer()
if self.cfg.dryrun and self.runtime.hid == 0:
message("Dryrun, nothing was executed.\n")
self.runtime.start = "%r" % self.start
self.runtime.end = "%r" % self.end
def save(self):
if self.cfg.dryrun: return
if self.cfg.gxpmode:
# Gather results
self.send_res()
if self.gxp.rank == 0:
reslist = []
for i in range(0, self.gxp.size):
reslist.append(self.recv_res())
else: return
if self.cfg.logdir is None: # generate random logdir in cwd
self.cfg.logdir = os.path.abspath("./pmlog-%s-%s" %
(self.runtime.user, time.strftime("%j-%H-%M-%S")))
# Initial log directory and database
if self.cfg.gxpmode:
self.cfg.confirm = False
self.cfg.logdir = smart_makedirs(self.cfg.logdir,
self.cfg.confirm)
logdir = os.path.abspath(self.cfg.logdir)
if self.cfg.nolog:
message("Saving data in memory ...")
else:
message("Saving data to %s/fsbench.db ..." % logdir)
# Save used configuration file
if not self.cfg.nolog:
verbose("Saving configurations to %s/fsbench.conf ..."
% logdir, VERBOSE)
self.opts.save_conf("%s/fsbench.conf" % logdir)
# Save results
if self.cfg.nolog: self.db = Database(":memory:")
else: self.db = Database("%s/fsbench.db" % logdir)
self.db.insert_runtime(self.runtime)
self.db.insert_conf(self.opts.cfgParser)
if self.cfg.gxpmode:
for res in reslist:
for r in res: self.db.insert_rawdata(r)
else:
for t in self.threads:
self.db.insert_rawdata(t.get_res())
self.db.commit()
if self.cfg.noreport: self.db.close()
def report(self):
if self.cfg.dryrun or self.cfg.noreport: return
if self.cfg.gxpmode and self.gxp.rank != 0: return
message("Generating report ...")
import report
logdir = self.cfg.logdir
if self.cfg.report:
logdir = self.cfg.report
if self.cfg.textreport:
self.report = report.TextReport(logdir, self.db, self.cfg)
elif self.cfg.csvreport:
self.report = report.CSVReport(logdir, self.db, self.cfg)
else:
self.report = report.HTMLReport(logdir, self.db, self.cfg)
self.report.write()
def send_res(self):
# Packing string without newlines
res = cPickle.dumps([t.get_res() for t in self.threads], 0)
self.gxp.wp.write('|'.join(res.split('\n')))
self.gxp.wp.write("\n")
self.gxp.wp.flush()
def recv_res(self):
res = self.gxp.rp.readline().strip('\n')
return cPickle.loads('\n'.join(res.split('|')))
def vs(self, msg):
sys.stderr.write(msg)
import basic_ops as bo
from data import Database
import glob
import librosa as lb
import pyaudio
import scipy.io.wavfile as sw
import random
import numpy as np
db_client = Database('fingerprints.db')
def upload():
"""
uploads fingerprints to database
:return:
"""
path = 'data/'
total_hashes = 0
for files in glob.glob(path + '*.wav'):
samples, sample_rate = lb.load(path=files, sr=None)
spec = bo.spectrogram(samples=samples, sample_rate=sample_rate)
peaks = bo.get_peaks(spec)
song_id = files.replace('-', ' ').replace('_', ' ').strip()[5:-4]
hash_list = bo.gen_hash(peaks, song_id=song_id)
for hash_item in hash_list:
db_client.insert_fingerprint(hash_item)
print(hash_item)
total_hashes += len(hash_list)
print('TOTAL FINGERPRINTS INSERTED: {}'.format(total_hashes))
def runTest(self):
print "Experiment.runTest>"
## main objects
db = Database()
objects = db.getObjects()
building = buildings.Test()
building.tag(objects)
#building.check(objects)
## Identify tags and set watch to enabled
o19 = objects.getTag('point', 19).single()
o20 = objects.getTag('point', 20).single()
o21 = objects.getTag('point', 21).single()
test = [o19, o20, o21]
db.enablePoints(test)
## Schedule object.
t = db.now()
# db.scheduleObject(o19, t+1, 3, 51)
# db.scheduleObject(o20, t+1 , 2, 1)
# db.scheduleObject(o20, t+4 , 1, 1)
# db.scheduleObject(o20, t+7 , 1, 1)
# db.scheduleObject(o20, t+6 , 3, 1)
#
db.scheduleObject(o21, t + 1, 10, 1)
db.scheduleObject(o21, t + 3, 5, None)
db.close()
class Plot:
def __init__(self, path):
self.db = Database("%s/trace.sqlite" % path)
self.path = "%s/figures" % path
if not os.path.exists(self.path):
utils.smart_makedirs(self.path)
self.ptree = None
self.COLORS = ["blue", "yellow", "red", "green"]
self.N_COLORS = len(self.COLORS)
self.c = Gnuplot.Gnuplot()
self.terminal = "png"
def _stacked_lines(self, path, x, yseries):
pyplot.clf()
y_data = np.row_stack(yseries)
y_data_stacked = np.cumsum(y_data, axis=0)
fig = pyplot.figure()
ax1 = fig.add_subplot(111)
y_start = 0
y_end = y_data_stacked[0]
for i in range(0, len(y_data_stacked)):
y_end = y_data_stacked[i]
ax1.fill_between(x, y_start, y_end,
facecolor=self.COLORS[i % self.N_COLORS],
alpha=0.7)
y_start = y_end
pyplot.savefig(path)
def init_proctree(self):
self.ptree = DiGraph()
for pid, ppid in self.db.proc_sel("pid,ppid"):
self.ptree.add_edge(ppid, pid)
assert nx.is_directed_acyclic_graph(self.ptree)
def plot_procs_stats(self):
if self.ptree is None: self.init_proctree()
# sort process in topology order
procs = nx.topological_sort(self.ptree)
procs.remove(0)
n_procs = len(procs)
utime_sum = 0.0
stime_sum = 0.0
utime_cums = []
stime_cums = []
rtime_sum = 0.0
wtime_sum = 0.0
rtime_cums = []
wtime_cums = []
rbytes_sum = 0.0
wbytes_sum = 0.0
rbytes_cums = []
wbytes_cums = []
for p in procs:
live, elapsed, utime, stime = \
self.db.proc_sel("live,elapsed,utime,stime", pid=p)[0]
if live:
n_procs -= 1
continue
"""
sysc_elapsed = self.db.sysc_sum("elapsed", pid=p)[0]
if sysc_elapsed is None:
sysc_elapsed = 0
"""
rtime, rbytes = self.db.sysc_sum("elapsed,aux1", pid=p,
sysc=SYSCALL['read'])[0]
if rtime is None:
rtime = 0
rbytes = 0
wtime, wbytes = self.db.sysc_sum("elapsed,aux1", pid=p,
sysc=SYSCALL['write'])[0]
if wtime is None:
wtime = 0
wbytes = 0
rtime_sum += rtime
wtime_sum += wtime
rbytes_sum += rbytes
wbytes_sum += wbytes
print rtime_sum, wtime_sum, rbytes_sum, wbytes_sum
continue
utime_sum += utime
utime_cums.append(utime_sum)
stime_sum += stime
stime_cums.append(stime_sum)
rtime_sum += rtime
wtime_sum += wtime
rtime_cums.append(rtime_sum)
wtime_cums.append(wtime_sum)
rbytes_sum += rbytes
wbytes_sum += wbytes
rbytes_cums.append(rbytes_sum)
wbytes_cums.append(wbytes_sum)
self._stacked_lines("%s/%s" % (self.path, "procs_cputime_ratio"),
np.arange(0, n_procs), (utime_cums, stime_cums))
self._stacked_lines("%s/%s" % (self.path, "procs_iotime_ratio"),
np.arange(0, n_procs), (rtime_cums, wtime_cums))
self._stacked_lines("%s/%s" % (self.path, "procs_iobytes_ratio"),
np.arange(0, n_procs), (rbytes_cums, wbytes_cums))
def plot(self, plist=[]):
if "procs_stats" in plist:
self.plot_procs_stats()
def workflow(self, path):
g = WorkflowDAG(self.db)
g.draw(path)
return g
def points_chart(self, data, prefix="points_chart", title="points_chart",
xlabel="x label", ylabel="y label"):
self.c.reset()
self.c.title(title)
self.c.xlabel(xlabel)
self.c.ylabel(ylabel)
self.c("set terminal %s" % self.terminal)
self.c("set output '%s.%s'" % (prefix, self.terminal))
self.c("set data style points")
self.c.plot(data)
return "%s.%s" % (prefix, self.terminal)
def lines_chart(self, data, prefix="lines_chart", title="lines_chart",
xlabel="x label", ylabel="y label"):
self.c.reset()
self.c.title(title)
self.c.xlabel(xlabel)
self.c.ylabel(ylabel)
self.c("set terminal %s" % self.terminal)
self.c("set output '%s.%s'" % (prefix, self.terminal))
self.c("set data style linespoints")
self.c.plot(data)
return "%s.%s" % (prefix, self.terminal)
def run(self):
print "Analysis.run>"
## main objects
db=Database()
graph=Graph()
## populate meta information
#devices=data.getDevices()
objects=db.getObjects()
building=buildings.PughHall()
building.tag(objects)
#building.check(objects)
## Compute streams
data=Data('data')
vav={}
ahu={}
total={}
plot=Plot("plot")
## Connections
source=Connection('source') # data connection
dest={} ## VAV connections
output=Connection("output")
## Connect
source.addIn(data)
## Build network
ahus=[1,3]
for ah in ahus:
zone=objects.getTag('zone',ah) ## All zone objects (VAV and AHU)
data.addOut(zone)
vavs=zone.getValues('vav')
total[ah]=Total("total%d" % ah,vavs)
dest[ah]=Connection("dest%d" % ah)
dest[ah].addOut(total[ah]) ## zone totalizer
a=AHU("ahu%d" % ah, ah,objects.getTag('ahu',ah))
ahu[ah]=a
source.addOut(a)
for v in vavs:
zv=VAV(('zone%d-VAV%s') % (ah,v), zone.getTag('vav',v), objects.getTag('ahu',ah)) ## Zone stream
vav[v]=zv
source.addOut(zv)
dest[ah].addIn(zv)
## Per ahu plots
output.addIn(total[ah])
## add trace points
output.addIn(ahu[1])
output.addIn(vav[114])
## connect plot (last)
output.addOut(plot)
if trace:
print "Analysis.run>", repr(source)
print "Analysis.run>", repr(dest)
print "Analysis.run>", repr(total)
print "Analysis.run>", repr(plot)
## Process DataStream
#limit="WHERE time >= '2011-09-27 20:00' AND time <= '2011-09-27 23:00'"
#limit="WHERE time >= '2011-09-27 20:00' AND time <= '2011-09-28 07:00'"
#limit="WHERE time >= '2011-09-27' AND time <= '2011-09-29'"
limit=None
## Debug
monitor=InstanceList()
#monitor.add(objects.getTags({'descriptor':'SAIR-TE'}))
#monitor.add(objects.getTags({'descriptor':'RAIR-H'}))
#monitor=monitor.getTag('ahu',1)
#monitor.add(objects.getTags({'vav':114,'descriptor':'CTL TEMP'}))
#monitor.add(objects.getTags({'vav':114,'descriptor':'FLOW'}))
monitor.add(objects.getTags({'vav':114,'descriptor':'HTG LOOPOUT'}))
#monitor.add(objects.getTags({'vav':114}))
## Stream compute
i=0;
for time,device,otype,oinstance,value in db.getData(limit):
v=Value(db.getInstance(device,otype,oinstance),value,time,0) ## Build value
data.send(v) ## input data
## Debug
if v.var in monitor:
print "DATA:", time, v
if i%100000==0:
print "TICK:", i,time
i+=1
## Plot
#limit=None
#limit=['total1-qsum','total3-qsum']
limit=['sat','sa']
graph.add(plot,limit)
graph.run()
def loadEmotions():
emotions = Database.find(Database.COLLECTIONS.EMOTION, {})
__emotions__ = []
for document in emotions:
__emotions__.append(document["name"])
#w2v = Word2Vec(db)
#w2v.train(workers = 64)
#w2v.save("dump/w2v-zhwiki.dump")
#lda = LDA(db)
#lda.train()
#lda.save("dump/lda2-zhwiki.dump")
@function_timer
def t2wv(db) :
return dict((attrib["title"], model[jieba.cut(text)]) for text, attrib in db)
if __name__ == "__main__" :
db = Database("data/zhwiki-extracted/", conditions = [
Database.cond_length(50),
Database.cond_title(lambda t: not t.startswith("Wikipedia:")),
Database.cond_title(lambda t: not t.startswith("File:")),
Database.cond_title(lambda t: not t.startswith("Draft:"))
])
# 767125 loaded, 451657 filtered, 31997 fails
model = Method.load("dump/lda-model.dump")
title2topic = t2wv(db)
pickle.dump(title2topic, open("dump/title2topic.dump", "w"))
#title2wv = pickle.load(open("dump/title2wv.dump"))
vectors = []
id2title = {}
for i, (t, v) in enumerate(title2topic.items()) :
vectors.append(v)
id2title[i] = t
class Execute(object):
def __init__(self):
self.database = Database()
self.database.create_tables()
self.message_response = {}
# Parse emails to be inserted into MySQL.
def extract_data(self):
parsed = Extract()
data = []
files_contain = set()
for top, d, agg in os.walk(os.getcwd() + "/enron_with_categories"):
for f in agg:
if f.endswith(".txt") and f not in files_contain:
files_contain.add(f)
f = os.path.join(top, f)
data.append(parsed.level(open(f, 'r').readlines()))
return data
# Insert parsed data into MySQL.
def insert_into_database(self, list_of_value_dicts):
for row in list_of_value_dicts:
erow = row.get('message_id'), row.get(
'from'), row.get('subject'), self.database.convert_date_format(
row.get('date')), self.get_label(len(
row.get('to'))), row.get('sub_md5')
erows = [erow]
rrows = []
for to in row.get('to'):
rrow = row.get('message_id'), row.get('from'), to, 1, 0, 0
rrows.append(rrow)
for cc in row.get('cc'):
rrow = row.get('message_id'), row.get('from'), cc, 0, 1, 0
rrows.append(rrow)
for bcc in row.get('bcc'):
rrow = row.get('message_id'), row.get('from'), bcc, 0, 0, 1
rrows.append(rrow)
self.database.insert('email', erows)
self.database.insert('recipient', rrows)
return True
# Solve the first question.
def per_day(self):
print "Each person recieved this many emails each day: "
print "EMAIL", '|', "COUNT", '|', "DATE"
print "_________________________________"
for e, c, date in self.database.run_query(
"select recipient,count(r.message_id) as cnt, email_date from recipient r, email e where e.message_id = r.message_id group by recipient,DATE(email_date) order by cnt desc;"
):
print e, '|', c, '|', unicode(date)
print "_________________________________"
# Solve the third question
def fast_responses(self):
prev_sub_md5 = None
prev_ts = None
for sub_md5, sender, message_id, email_date in self.database.run_query(
"SELECT sub_md5,sender, message_id, email_date from email where sub_md5 is NOT NULL order by sub_md5,email_date asc;"
):
if sub_md5 != prev_sub_md5:
prev_sub_md5 = sub_md5
prev_ts = email_date
continue
else:
prev_sub_md5 = sub_md5
self.message_response[message_id] = email_date - prev_ts
prev_ts = email_date
print "These people responded the fastest: "
print[
str(unicode(k)) for k, v in (
sorted(self.message_response.items(), key=lambda x: x[1])[:6])
]
print "_________________________________"
# Broadcast part of question 2
def broadcast(self):
print "BROADCAST"
e, c = self.database.run_query(
"select sender,count(*) as cnt from email where label='broadcast' group by sender order by cnt desc;",
1)
print "The email " + unicode(
e
) + " recieved the most broadcast emails with " + c + " emails recieved."
print "_________________________________"
# Direct part of question 2
def direct(self):
print "DIRECT"
e, c = self.database.run_query(
"select recipient,count(e.message_id) as cnt from recipient r, email e where e.message_id = r.message_id and e.label = 'direct' and r.is_cc = 1 group by recipient,label order by cnt desc;",
1)
print "The email " + unicode(
e) + " sent the most direct emails with " + c + " emails sent."
print "_________________________________"
# Make sure the directory of emails can be found.
def check_for_emails(self):
if os.path.isdir("enron_with_categories"):
print "Data Set has been Accessed!"
return True
return False
# Get Count of how many people email was sent out to!
def get_label(self, to):
if to <= 1:
return 'direct'
return 'broadcast'
# Run Full Job
def execute(self):
if self.check_for_emails():
data = self.extract_data()
if data and self.insert_into_database(data):
return True
return False
def runTest(self):
print "Experiment.runTest>"
## main objects
db=Database()
objects=db.getObjects()
building=buildings.Test()
building.tag(objects)
#building.check(objects)
## Identify tags and set watch to enabled
o19=objects.getTag('point',19).single()
o20=objects.getTag('point',20).single()
o21=objects.getTag('point',21).single()
test=[o19,o20,o21]
db.enablePoints(test)
## Schedule object.
t=db.now()
# db.scheduleObject(o19, t+1, 3, 51)
# db.scheduleObject(o20, t+1 , 2, 1)
# db.scheduleObject(o20, t+4 , 1, 1)
# db.scheduleObject(o20, t+7 , 1, 1)
# db.scheduleObject(o20, t+6 , 3, 1)
#
db.scheduleObject(o21, t+1 , 10, 1)
db.scheduleObject(o21, t+3 , 5, None)
db.close()