def __init__(self, params):
# A constructor that stores the provided parameters internally, as well
# as calling both PageConstructor() and DataSource().
self.page_type = params['page_type']
self.params = params
self.page_maker = PageConstructor()
self.db_source = DataSource()
def __init__(self, **kwargs):
DataSource.__init__(self)
self.train_dirs = kwargs['train_dirs'] if 'train_dirs' in kwargs else []
files = [ (d,os.listdir(d)) for d in self.train_dirs ]
self.setup(paths=[os.path.join(d,f) for d,filesInDir in files for f in filesInDir if f.split('.')[-1] == "png" ])
def __init__(self):
# The constructor opens a connection to the database, creates an empty
# string of HTML, and creates an empty dictionary to be filled with
# database-specific replacements to the HTML templates.
self.page = ""
self.replacements = {}
self.dbSource = DataSource()
def __init__(self, **kwargs):
DataSource.__init__(self)
self.train_dirs = kwargs['train_dirs'] if 'train_dirs' in kwargs else []
self.chunksize = kwargs['chunksize'] if 'chunksize' in kwargs else (64,64)
files = [ (d,os.listdir(d)) for d in self.train_dirs ]
images = [ (os.path.join(d,f), os.path.splitext(os.path.join(d,f))[0]) for d,filesInDir in files for f in filesInDir if f.split('.')[-1] == "ndpi" and f.find("HE") > 0 ]
self.setup(images=images)
def __init__(self, filename_1m, filename_3m, filename_5m, filename_15m,
lrwindowsize):
self.Filename = filename
self.Datasource_1minute = DataSource(filename_1m, lrwindowsize)
self.Datasource_3minute = DataSource(filename_3m, lrwindowsize)
self.Datasource_5minute = DataSource(filename_5m, lrwindowsize)
self.Datasource_15minute = DataSource(filename_15m, lrwindowsize)
LoadDatasetFromfile(filename)
return
def __init__(self, **kwargs):
DataSource.__init__(self)
self.train_dirs = kwargs['train_dirs'] if 'train_dirs' in kwargs else []
self.chunksize = kwargs['chunksize'] if 'chunksize' in kwargs else (64,64)
self.eval_dirs = kwargs['eval_dirs'] if 'eval_dirs' in kwargs else []
files = [ (d,os.listdir(d)) for d in self.train_dirs ]
images = [ (Image.open(os.path.join(d,f)), os.path.join(d,f), os.path.splitext(os.path.join(d,f))[0]) for d,filesInDir in files for f in filesInDir if f.split('.')[-1] == "bmp" ]
self.setup(images=images)
def __init__(self):
# The constructor opens a connection to the database, creates an empty
# string of HTML, and creates an empty dictionary to be filled with
# database-specific replacements to the HTML templates.
self.page = ""
self.replacements = {}
self.dbSource = DataSource()
def test_wide_deep():
utils.config_logging('log_{}.log'.format('WideEstimator'))
n_epoch = 10
batch_size = 32
data_source = DataSource('./dataset/train.csv', './dataset/test.csv', batch_size)
wide_hparams = WideHParams(field_names=CATEGORY_FIELDS,
alpha=0.1,
beta=1,
L1=0.1,
L2=0.1)
deep_hparams = get_deep_hparams(embed_size=16,
hidden_units=[64, 16],
L2=0.01,
learning_rate=0.001)
start_time = time.time()
estimator = WideDeepEstimator(wide_hparams=wide_hparams, deep_hparams=deep_hparams, data_source=data_source)
metrics_history = estimator.train(n_epoch)
elapsed = time.time() - start_time
# ************ display result
logging.info("\n************** TIME COST **************")
logging.info('{:.2f} seconds for {} epoches'.format(elapsed, n_epoch))
logging.info('{:.2f} examples per second'.format(
n_epoch * (data_source.n_train_examples + data_source.n_test_examples) / elapsed))
logging.info("\n************** LEARNING CURVE **************")
metrics_history = pd.DataFrame(metrics_history)
logging.info(metrics_history)
metrics_history.to_csv('learn_curve_{}.csv'.format(estimator), index=False)
def __init__(self, params):
# A constructor that stores the provided parameters internally, as well
# as calling both PageConstructor() and DataSource().
self.page_type = params['page_type']
self.params = params
self.page_maker = PageConstructor()
self.db_source = DataSource()
def test_trim_time_series_with_missing_start_date(self):
# Setup test object
start_date = pd.to_datetime('2015-12-01')
end_date = pd.to_datetime('2016-01-06')
trimed_series = DataSource('test').trim_series_observations(self.test_series, 'dummy ticker', \
'dummy series name', start_date, end_date, 0)
# Check results
self.assertEqual(trimed_series.index[0], self.test_series.index[0])
self.assertEqual(trimed_series.index[-1], end_date)
def test_trim_time_series_with_existing_bording_dates_and_history_window(
self):
# Setup test object
start_date = pd.to_datetime('2015-12-31')
end_date = pd.to_datetime('2016-01-06')
trimed_series = DataSource('test').trim_series_observations(self.test_series, 'dummy ticker', \
'dummy series name', start_date, end_date, 1)
# Check results
self.assertEqual(trimed_series.index[0], pd.to_datetime('2015-12-30'))
self.assertEqual(trimed_series.index[-1], end_date)
def LoadData():
source = DataSource()
print("Loading movie ratings...")
data = source.loadMovieLensRating()
print("Prepare movie information...")
source.computeMovieInformation()
print("Creating ranking for each movie ...")
rankings = source.getPopularityRanksByRating()
return (source, data, rankings)
def __init__(self):
self.dq = Queue.Queue
self.ds = DataSource(self, self.dq, 21)
self.ds.start()
self.x_flag = False
self.running = True
self.maxsamp = 4000
self.run_cnt = 0
self.run_cnt_active = 0
self.lim_run_cnt_active = 1500
self.eval_cnt = 0
self.armed = False
self.numdata = 6 # mode, alt, vel, throttle, zacc, battery
self.mode = 0
self.alt = 0
self.vz = 0
self.throttle = 0
self.zacc = 0
self.batt = 0
self.storage_version = "v2"
# data storage
self.tblfilename = "bf_optimize_mavlink.h5"
self.h5file = tb.open_file(self.tblfilename, mode = "a")
# check if top group exists
try:
self.g1 = self.h5file.get_node("/%s" % self.storage_version)
# self.table = self.h5file.list_nodes("/v1")[0]
self.table = self.h5file.get_node("/%s/evaluations" % self.storage_version)
except:
self.g1 = self.h5file.create_group(self.h5file.root, self.storage_version,
"Optimization run params, perf and logdata")
if self.storage_version == "v1":
self.table = self.h5file.create_table(self.g1, 'evaluations', BFOptML,
"Single optimizer evaluations")
elif self.storage_version == "v2":
self.table = self.h5file.create_table(self.g1, 'evaluations', BFOptML2,
"Single optimizer evaluations")
print self.g1, self.table
self.bfoptml = self.table.row
def __init__(self, country: Country):
self.country = country
# contains all the information to access data and mapping from field of the csv to enum fields (see Enums).
self.list_file_info: List[
FileInformation] = DataSource.get_info_for_country(country)
# contains the data (in pandas DataFrame form).
self.data_dic: Dict[PatientCase, List[pd.DataFrame]] = dict()
# download data
for item in self.list_file_info:
# download content
data_raw = requests.get(item.http_file).content
# make a dic with pandas DataFrame
try:
# if this does not work you can try 'utf-8'
temp_data: pd.DataFrame = pd.read_csv(io.StringIO(
data_raw.decode('latin_1')),
sep=country.sep)
except UnicodeDecodeError:
print('Format is not the right one.')
else:
temp_data = temp_data[[
header for header in item.dic_category.values()
]]
temp_data.columns = [
univ_header.name
for univ_header in item.dic_category.keys()
]
# set date as date format.
temp_data[PatientCategory.date.name] = pd.to_datetime(
temp_data[PatientCategory.date.name])
# add the dataFrame to the list for the current case.
if item.case not in self.data_dic.keys():
self.data_dic[item.case] = list()
self.data_dic[item.case].append(temp_data)
class JamrDao(object):
def __init__(self):
self.userDS = DataSource("database.pickle.user")
self.roomDS = DataSource("database.pickle.room")
self.uidToWsKeyDic = {}
self.uidToWs = {}
def associateKey(self, uuid, wskey):
self.uidToWsKeyDic[uuid] = wskey
def getKey(self,uuid):
return self.uidToWsKeyDic[uuid]
def insertWebsocket(self, peer_address, ws):
self.uidToWs[peer_address] = ws
def getWebsocket(self,peer_address):
return self.uidToWs[peer_address]
def insertUser(self, user):
self.userDS.put(user['uid'], user)
def getUser(self, uuid):
'''
Returns null on not found
'''
return self.userDS.get(uuid)
def insertRoom(self, room):
self.roomDS.put(room['uid'], room)
def getRoom(self, uuid):
'''
Returns null on not found
'''
return self.roomDS.get(uuid)
def getAllUsers(self):
return self.userDS.getAll()
def getAllWebSockets(self):
return self.uidToWs.values()
DataSource.PATH: '../scraping/data/sentences_data.txt'
}, {
DataSource.NAME: 'Twitter',
DataSource.PATH: '../../data/SemEval-2014.csv'
}]
srcs = [{
DataSource.NAME: src[DataSource.NAME],
DataSource.PATH: os.path.abspath(src[DataSource.PATH]),
DataSource.TYPE: os.path.splitext(src[DataSource.PATH])[1]
} for src in srcs]
for src in srcs:
print("Name: '%s'\nType: '%s'" %
(src[DataSource.NAME], src[DataSource.TYPE]))
dSrc = DataSource(src)
srcStats = dSrc.getStats()
print("\nUsable data:")
pprint.pprint(srcStats[DataSource.REGULAR])
print("\nUnusable data:")
pprint.pprint(srcStats[DataSource.BROKEN])
print("-----------------")
print("Finished!")
for src in srcs:
srcName = src[DataSource.NAME]
dSrc = DataSource(src)
srcStats = dSrc.getStats()
kws, ocs = [], []
for word in srcStats[DataSource.REGULAR].items():
class TestDataSource(unittest.TestCase):
"""Test cases for some methods in DataSource.py. Not necessary but we left
what we had.
NOTE: will not work if run from command line without uncommenting
#f = open('testpassfileworkaround') in DataSource.py and with password file
existing in the directory.
NOTE: on piazza jadrian pointed out that we probably shouldnt be doing unit
tests on database source that have dependencies on fickle databse files, as
a result these tests are not to be trusted and are not totally complete """
global db_source
db_source = DataSource()
# For testing purposes, I will rely on several specific datapoints hardcoded
# into the test code. Here are the databse entries for the particular
# objects. I will be testing so it is clear to see what I am emulating in
# code.
#Bill example = (100, 109, 188, datetime.date(2005, 7, 14), 'amendment', 'On
#the Amendment S.Amdt. 1222 to H.R. 2360 (Department of Homeland Security
#Appropriations Act, 2006)', [300001, 300005, 300006, 300008, 300009,
#300011, 300016, 300018, 300019, 300022, 300026, 300028, 300032, 300034,
#300037, 300038, 300042, 300043, 300051, 300056, 300057, 300058, 300059,
#300060, 300061, 300063, 300064, 300065, 300066, 300067, 300068, 300076,
#300078, 300077, 400629, 300080, 300081, 300082, 300084, 400619, 300086,
#300087, 300093, 300100], [300002, 300003, 300004, 300007, 300010, 300013,
#300014, 300015, 400054, 300020, 300021, 400576, 300023, 300024, 300025,
#300027, 300029, 300030, 300033, 300035, 300036, 300040, 300041, 300045,
#300047, 300048, 300049, 300050, 300052, 300054, 300055, 400194, 300062,
#300070, 400621, 300071, 300072, 300075, 300083, 300085, 300088, 300089,
#300090, 300091, 300092, 300094, 300095, 300096, 300097, 400546, 400418,
#300098, 300099], [], [400105, 300069, 300073])
#senator example = (407661, 'Metcalfe', 'Thomas', datetime.date(1780, 3, 20), 'M', 'KY', 'Whig', 'Thomas Metcalfe (Kentucky)', False)
global test_senator
test_senator = Senator([412390, 'Coons', 'Chris', datetime.date(2063, 9, 9)
, 'M', 'DE', 'Democrat', 'Chris Coons', True])
short_bill_list = [100, 109, 188, datetime.date(2005, 7, 14), 'amendment', 'On the Amendment S.Amdt. 1222 to H.R. 2360 (Department of Homeland Security Appropriations Act, 2006)']
global test_billnv
test_billnv = Bill(short_bill_list)
#note, this is not entirely accurate as what db_source really passes to the
#constructor are lists of senator objects but in this case it is still
#possible to test by comparing the id integer to senator.getId()
global test_billwv test_billwv = Bill([100, 109, 188, datetime.date(2005, 7,
14), 'amendment', 'On the Amendment S.Amdt. 1222 to H.R. 2360 (Department of'
+ 'Homeland Security Appropriations Act, 2006)', [300001, 300005, 300006,
300008, 300009, 300011, 300016, 300018, 300019, 300022, 300026, 300028,
300032, 300034, 300037, 300038, 300042, 300043, 300051, 300056, 300057,
300058, 300059, 300060, 300061, 300063, 300064, 300065, 300066, 300067,
300068, 300076, 300078, 300077, 400629, 300080, 300081, 300082, 300084,
400619, 300086, 300087, 300093, 300100], [300002, 300003, 300004, 300007,
300010, 300013, 300014, 300015, 400054, 300020, 300021, 400576, 300023,
300024, 300025, 300027, 300029, 300030, 300033, 300035, 300036, 300040,
300041, 300045, 300047, 300048, 300049, 300050, 300052, 300054, 300055,
400194, 300062, 300070, 400621, 300071, 300072, 300075, 300083, 300085,
300088, 300089, 300090, 300091, 300092, 300094, 300095, 300096, 300097,
400546, 400418, 300098, 300099], [], [400105, 300069, 300073]])
## testing that GetBill does indeed return the expectd bill object
def testGetBill(self):
actual = db_source.getBill(100)
self.assertEquals(test_billnv.getId(), actual.getId())
self.assertEquals(test_billnv.getSession(), actual.getSession())
self.assertEquals(test_billnv.getRoll(), actual.getRoll())
self.assertEquals(test_billnv.getVoteDate(), actual.getVoteDate())
self.assertEquals(test_billnv.getType(), actual.getType())
self.assertEquals(test_billnv.getQuestion(), actual.getQuestion())
# testing that GetBillWithVotes works properly, makes the assumption that a
# senator object that gets built and passed works properly and that a
# matching id is all that's necessary
def testGetBillWithVotes(self):
actual = db_source.getBillWithVotes(100)
self.assertEquals(test_billwv.getId(), actual.getId())
self.assertEquals(test_billwv.getSession(), actual.getSession())
self.assertEquals(test_billwv.getRoll(), actual.getRoll())
self.assertEquals(test_billwv.getVoteDate(), actual.getVoteDate())
self.assertEquals(test_billwv.getType(), actual.getType())
self.assertEquals(test_billwv.getQuestion(), actual.getQuestion())
yea = []
nay = []
present = []
not_voting = []
#print actual.getYea_Votes() TODO
for senator in actual.getYea_Votes():
yea.append(senator.getId())
for senator in actual.getNay_Votes():
nay.append(senator.getId())
for senator in actual.getAbstaining():
present.append(senator.getId())
for senator in actual.getAbsent():
not_voting.append(senator.getId())
self.assertEqual(tuple(test_billwv.getYea_Votes()),tuple(yea))
self.assertEqual(tuple(test_billwv.getNay_Votes()),tuple(nay))
self.assertEqual(tuple(test_billwv.getAbstaining()),tuple(present))
self.assertEqual(tuple(test_billwv.getAbsent()),tuple(not_voting))
def testGetSenator(self):
actual = db_source.getSenator(412390)
self.assertEquals(test_senator.getId(), actual.getId())
self.assertEquals(test_senator.getFirst(), actual.getFirst())
self.assertEquals(test_senator.getLast(), actual.getLast())
self.assertEquals(test_senator.getBirthday(), actual.getBirthday())
self.assertEquals(test_senator.getGender(), actual.getGender())
self.assertEquals(test_senator.getParty(), actual.getParty())
self.assertEquals(test_senator.getWiki(), actual.getWiki())
self.assertEquals(test_senator.isCurrent(), actual.isCurrent())
def testGetVotesBySenator(self):
# The last 5 bill id's and votes that the test senator voted on
expected_table = [[3224,"yea"],[3223,"nay"],[3222,"nay"],[3220,"yea"],[3219,"nay"]]
actual = db_source.getVotesBySenator(412390,5)
i = 0;
assertEquals(len(actual),len(expected_table))
while i < len(actual):
assertEquals(expected_table[i],[actual[i][0].getid(),actual[i][1]])
i+=1
def __init__(self, parent, url, queue, timeout):
DataSource.__init__(self, parent, queue, timeout)
self.url_info = url
self.objInfo = None
from DataSource import DataSource
ds = DataSource()
ds.put("1", "1")
print ds.get("1")
print ds.get("2")
ds.save()
'stochastic_Slow_Signal', 'momentumIndicator', 'Fractal_Upper'
] # , 'Fractal_Lower']#'stochasticArr_Slow_Signal', 42 = 'momentumIndicator' 43 = 'Fractal.Upper' 44 = 'Fractal.Lower'
x = df.loc[subColumnList].values
return self.rfBigMove.getPredSimpleTest1(x.reshape(1, -1))[0]
def randomForestDir(self, dataScoure):
row = dataScoure.getRow()
df = pd.Series(row).T
subColumnList = [
'Bollinger.Upper.MediumPrice.SD2', 'RSI(close)', 'MFI', 'MacDMain',
'SMA.weighted', 'Alligator.LipsCurChart', 'Alligator.Jaw5min',
'SDPriceValueMedium', 'stochastic_Fast_Main',
'stochastic_Fast_Signal', 'stochastic_Slow_Main',
'stochastic_Slow_Signal', 'momentumIndicator', 'Fractal_Upper',
'Fractal_Lower'
]
x = df.loc[subColumnList].values
return self.rfDirection.getPredSimpleTest1(x.reshape(1, -1))[0]
#TODO create better test cases, I dunno even know what is happening below or why?
if __name__ == "__main__":
test = FeatureProvider()
from DataSource import DataSource
path = 'C:/Users/garre/AppData/Roaming/MetaQuotes/Terminal/B83207E76A7859F5556693074AFE91E8/MQL4/Files/Data/Math748PriceSending.csv'
new = DataSource.CleanBrokerData(path)
new.advance()
print(test.randomForest20PosMov(new))
def __init__(self, parent, queue, url, timeout):
DataSource.__init__(self, parent, queue, timeout)
self.url_info = url + "/api.php?mode=info"
self.url_coins = url + "/api.php?mode=coins"
self.objDiff = None
self.objReward = None
from InventoryAllocator import Allocator
from DataSource import DataSource
from InventoryInput import InventoryInputDialog
from MainWindow import MyMainWindow
# set things up, and run it. :)
if __name__ == '__main__':
inventoryList = {
'A': 150,
'B': 150,
'C': 100,
'D': 100,
'E': 200
} # Dummy inventory List
inventory = Inventory(inventoryList)
# Get Datasource object which initially creates blank data
datasource = DataSource()
allocator = Allocator(datasource, inventory)
# The order list will be initially empty and will get filled up as the user enters the order
# quantities in the GUI Screen
orderList = allocator.allocateOrders()
app = QtWidgets.QApplication(sys.argv)
# This invokes the input for initial Inventory which upon success invokes main window as below
dialog = InventoryInputDialog(inventory)
dialog.exec_()
# Invoke the main window
w = MyMainWindow(orderList, datasource, allocator, inventory)
w.show()
app.exec_()
sys.exit()
def _load_upcoming( self, data ): ds = DataSource( self.url ) return ds.single_page( data )
class BaseflightOptimize(object):
def __init__(self):
self.dq = Queue.Queue
self.ds = DataSource(self, self.dq, 21)
self.ds.start()
self.x_flag = False
self.running = True
self.maxsamp = 4000
self.run_cnt = 0
self.run_cnt_active = 0
self.lim_run_cnt_active = 1500
self.eval_cnt = 0
self.armed = False
self.numdata = 6 # mode, alt, vel, throttle, zacc, battery
self.mode = 0
self.alt = 0
self.vz = 0
self.throttle = 0
self.zacc = 0
self.batt = 0
self.storage_version = "v2"
# data storage
self.tblfilename = "bf_optimize_mavlink.h5"
self.h5file = tb.open_file(self.tblfilename, mode = "a")
# check if top group exists
try:
self.g1 = self.h5file.get_node("/%s" % self.storage_version)
# self.table = self.h5file.list_nodes("/v1")[0]
self.table = self.h5file.get_node("/%s/evaluations" % self.storage_version)
except:
self.g1 = self.h5file.create_group(self.h5file.root, self.storage_version,
"Optimization run params, perf and logdata")
if self.storage_version == "v1":
self.table = self.h5file.create_table(self.g1, 'evaluations', BFOptML,
"Single optimizer evaluations")
elif self.storage_version == "v2":
self.table = self.h5file.create_table(self.g1, 'evaluations', BFOptML2,
"Single optimizer evaluations")
print self.g1, self.table
self.bfoptml = self.table.row
# if g1 exists
# pass
# else
# g1 = h5file.create_group(h5file.root, "v1", "Optimization run params, perf and logdata")
def mavlink_sys_status_handler(self, msg):
self.mode = msg.onboard_control_sensors_health # ugly hack
self.batt = msg.voltage_battery #
# print "sys status", msg
def mavlink_raw_imu_handler(self, msg):
self.zacc = msg.zacc
# print "raw imu", msg
pass
def mavlink_attitude_handler(self, msg):
# print "attitude", msg
pass
def mavlink_global_position_int_handler(self, msg):
self.alt = msg.alt
self.vz = msg.vz
# print "global position int", msg
# pass
def mavlink_rc_channels_raw_handler(self, msg):
self.throttle = msg.chan4_raw
# print "rc channels raw", msg
# pass
def mavlink_servo_output_raw_handler(self, msg):
# print "servo output raw", msg
pass
def reset(self):
self.running = True
self.run_cnt = 0
self.run_cnt_active = 0
self.armed = False
def objective(self, params):
self.eval_cnt += 1
print "eval#", self.eval_cnt, "params", params
# reset stuff
self.reset()
# check if we have data for this parameterization
# stored_params = [(x["alt_p"], x["alt_i"], x["alt_d"], x["vel_p"], x["vel_i"], x["vel_d"]) for x in table.]
existing_run_data = \
[ (x["alt_p"], x["alt_i"], x["alt_d"], x["vel_p"], x["vel_i"], x["vel_d"], x["mse"])
for x in self.table.where("""(alt_p == %d) & (alt_i == %d) & (alt_d == %d) & \
(vel_p == %d) & (vel_i == %d) & (vel_d == %d)""" %
tuple([params[i] for i in range(6)]))]
if len(existing_run_data) > 0:
mse = existing_run_data[-1][-1]
print "reusing existing run data: mse = %f" % mse
time.sleep(0.1)
return mse
# # set PID values
# # first request the values
# for i in range(3):
# self.ds.mavo.mav.param_request_list_send(21, 46)
# time.sleep(0.5)
# alt_offset = BF_PID.PIDALT*3
for i in range(2):
self.ds.mavo.mav.param_set_send(21, 46, "P_ALT", params[0], mavlink.MAVLINK_TYPE_UINT8_T)
self.ds.mavo.mav.param_set_send(21, 46, "I_ALT", params[1], mavlink.MAVLINK_TYPE_UINT8_T)
self.ds.mavo.mav.param_set_send(21, 46, "D_ALT", params[2], mavlink.MAVLINK_TYPE_UINT8_T)
self.ds.mavo.mav.param_set_send(21, 46, "P_VEL", params[3], mavlink.MAVLINK_TYPE_UINT8_T)
self.ds.mavo.mav.param_set_send(21, 46, "I_VEL", params[4], mavlink.MAVLINK_TYPE_UINT8_T)
self.ds.mavo.mav.param_set_send(21, 46, "D_VEL", params[5], mavlink.MAVLINK_TYPE_UINT8_T)
# vel_offset = BF_PID.PIDVEL*3
# init logdata
logdata = np.zeros((self.maxsamp, self.numdata))
while self.running:
# print self.mode
logdata[self.run_cnt,0] = np.clip(self.mode, 0., 1000.)
logdata[self.run_cnt,1] = np.clip(self.alt, -500., 1000.)
logdata[self.run_cnt,2] = np.clip(self.vz, -1000., 1000.)
logdata[self.run_cnt,3] = np.clip(self.throttle, 0., 2000.)
logdata[self.run_cnt,4] = np.clip(self.zacc, -1000., 1000.)
logdata[self.run_cnt,5] = np.clip(self.batt, 0., 20000.) #
# detect premature termination: landing and disarm
if self.mode == 3 and not self.armed:
self.armed = True
elif self.mode == 2 and self.armed:
self.armed = False
self.running = False
self.run_cnt += 1
# match MSP telemetry update rate
time.sleep(0.02)
# count number of controller active samples
if self.armed and self.mode == 11:
self.run_cnt_active += 1
# terminate if sufficient number of controller active samples
if self.run_cnt_active == self.lim_run_cnt_active:
self.running = False
# terminate if maximum number of samples
if self.run_cnt >= self.maxsamp:
self.running = False
# save data
ts = time.strftime("%Y%m%d%H%M%S")
np.save("logs/bf_optimize_mavlink_%s_log" % ts, logdata)
params_array = np.array(params)
np.save("logs/bf_optimize_mavlink_%s_params" % ts, params_array)
# compute performance
# alt_active_idx = logdata[:,0] == 27 # with mag lock
alt_active_idx = logdata[:,0] == 11
# catch empty index
if np.sum(alt_active_idx) == 0:
# set two elements True to have at least 2 element arrays below, even if they're bogus
alt_active_idx[0] = True
alt_active_idx[1] = True
# it was so bad, alt hold was disabled prior to lim_cnt_active
if np.sum(alt_active_idx) < (self.lim_run_cnt_active - 100):
alt_mse = 1e5
vel_mse = 1e5
alt_target = 0.
vel_target = 0.
else:
alt_data = logdata[alt_active_idx,1]
alt_target = np.mean(alt_data[0:10])
alt_mse = np.mean(np.square(alt_target - alt_data))
vel_data = logdata[alt_active_idx,2]
vel_target = 0.
vel_mse = np.mean(np.square(vel_target - vel_data))
mse_array = np.array((alt_target, alt_mse, vel_target, vel_mse))
np.save("logs/bf_optimize_mavlink_%s_mse" % ts, mse_array)
# save data to pytable
self.bfoptml["id"] = int(ts)
self.bfoptml["alt_p"] = params[0]
self.bfoptml["alt_i"] = params[1]
self.bfoptml["alt_d"] = params[2]
self.bfoptml["vel_p"] = params[3]
self.bfoptml["vel_i"] = params[4]
self.bfoptml["vel_d"] = params[5]
# set run performance measure
self.bfoptml["alt_target"] = alt_target
self.bfoptml["alt_mse"] = alt_mse
self.bfoptml["vel_target"] = vel_target
self.bfoptml["vel_mse"] = vel_mse
self.bfoptml["mse"] = alt_mse + vel_mse
# set run logdata
self.bfoptml["timeseries"] = logdata
# append new row
self.bfoptml.append()
self.table.flush()
# in place inspection
pl.subplot(411)
pl.title("mode")
pl.plot(logdata[:,0])
pl.subplot(412)
pl.title("alt")
pl.plot(logdata[:,1])
pl.plot(np.ones_like(logdata[:,1]) * alt_target)
pl.subplot(413)
pl.title("alt vel")
pl.plot(logdata[:,2])
pl.subplot(414)
pl.title("thr + acc z")
pl.plot(logdata[:,3])
pl.plot(logdata[:,4])
pl.show()
# generate params, set and run
# read data
# - min: altitude, RC5
# - max: record all
# compute performance
# return np.random.uniform(0, 1)
print "alt_mse = %f, vel_mse = %f" % (alt_mse, vel_mse)
print "mse total", alt_mse + vel_mse
return (alt_mse + vel_mse)
sys.path.append("../../Util")
sys.path.append("../../HTTP")
sys.path.append("../../Torrent")
from DataSource import DataSource
from OnDemandIntrospector import OnDemandIntrospector
from HTTPClient import SimpleHTTPClient
from TorrentClient import BasicTorrentExplainer
from TorrentPatron import TorrentPatron
from Kamaelia.Internet.TCPClient import TCPClient
if __name__ == '__main__':
# download a linux distro
X = pipeline(
DataSource( ["http://www.legaltorrents.com/bit/trusted-computing.torrent"] ),
SimpleHTTPClient(),
TorrentPatron(),
BasicTorrentExplainer(),
ConsoleEchoer(),
)
Y = pipeline(
DataSource( ["http://www.legaltorrents.com/bit/trusted-computing.torrent",
"http://www.legaltorrents.com/bit/freeculture.zip.torrent"] ),
SimpleHTTPClient(),
TorrentPatron(),
BasicTorrentExplainer(),
ConsoleEchoer(),
)
Z = pipeline(
class PageConstructor:
"""Contains all of the methods to access and fill out the HTML templates."""
global STATE_LIST
global STATE_ABBREVIATION_LIST
STATE_LIST = ['Alabama', 'Alaska', 'Arizona', 'Arkansas', 'California',
'Colorado', 'Connecticut', 'Delaware', 'Florida', 'Georgia', 'Hawaii',
'Idaho', 'Illinois', 'Indiana', 'Iowa', 'Kansas', 'Kentucky',
'Louisiana', 'Maine', 'Maryland', 'Massachusetts', 'Michigan',
'Minnesota', 'Mississippi', 'Missouri', 'Montana', 'Nebraska', 'Nevada',
'New Hampshire', 'New Jersey', 'New Mexico', 'New York',
'North Carolina', 'North Dakota', 'Ohio', 'Oklahoma', 'Oregon',
'Pennsylvania', 'Rhode Island', 'South Carolina', 'South Dakota',
'Tennessee', 'Texas', 'Utah', 'Vermont', 'Virginia', 'Washington',
'West Virginia', 'Wisconsin', 'Wyoming']
STATE_ABBREVIATION_LIST = ['AL', 'AK', 'AZ', 'AR', 'CA', 'CO', 'CT', 'DE',
'FL', 'GA', 'HI', 'ID', 'IL', 'IN', 'IA', 'KS', 'KY', 'LA', 'ME', 'MD',
'MA', 'MI', 'MN', 'MS', 'MO', 'MT', 'NE', 'NV', 'NH', 'NJ', 'NM', 'NY',
'NC', 'ND', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC', 'SD', 'TN', 'TX', 'UT',
'VT', 'VA', 'WA', 'WV', 'WI', 'WY']
def __init__(self):
# The constructor opens a connection to the database, creates an empty
# string of HTML, and creates an empty dictionary to be filled with
# database-specific replacements to the HTML templates.
self.page = ""
self.replacements = {}
self.dbSource = DataSource()
def readTemplate(self):
# A method that reads template.html, adds the template to self.page, and
# adds the HTML for the dropdown menus to self.replacements.
templateFile = open("Website/template.html", "r")
templateString = templateFile.read()
self.page = templateString
self.replacements["SenatorDropdown"] = ""
i = 0
while i < len(STATE_LIST):
self.replacements["SenatorDropdown"] += (
'<li><a href = "index.py?page_type=state&state='
+ STATE_ABBREVIATION_LIST[i] + '">'
+ STATE_LIST[i] + '</a></li>')
i+=1
committee_list = self.dbSource.getCommitteesBySession(114)
self.replacements["CommitteeDropdown"] = ""
for entry in committee_list:
self.replacements["CommitteeDropdown"] += ('<li>' +
entry.getCommitteeLink() + "</li>")
def makeHomepage(self):
# Gets HTML from HomepageTemplate.html and places it in the body of the
# generated page.
self.readTemplate()
pageFile = open("Website/HomepageTemplate.html", "r")
pageString = pageFile.read()
self.replacements["results"] = pageString
def makeBillPage(self, bill):
# Gets HTML from BillPageTemplate.html, fills in the info of the bill
# passed to it, and places it in the body of the generated page.
self.readTemplate()
billFile = open("Website/BillPageTemplate.html", "r")
billString = billFile.read()
session_link = ('<a href="index.py?page_type=session&session=' +
str(bill.getSession()) + '">' + str(bill.getSession()) +
"</a>")
votes = (str(len(bill.getYea_Votes())) + " yea | " +
str(len(bill.getNay_Votes())) + " nay | " +
str(len(bill.getAbstaining())) + " abstain | " +
str(len(bill.getAbsent())) + " absent")
# Loops through the vote data and addsa tally to a dictonary based
# on the party of the senator with keyed values that correspond
# to the votes in order
# all yea votes
party_dict = {}
for senator in bill.getYea_Votes():
party_votes = party_dict.get(senator.getParty())
if party_votes == None:
party_dict[senator.getParty()] = [1,0,0,0]
else:
party_votes[0] = party_votes[0] + 1
party_dict[senator.getParty()] = party_votes
# all nay votes
for senator in bill.getNay_Votes():
party_votes = party_dict.get(senator.getParty())
if party_votes == None:
party_dict[senator.getParty()] = [0,1,0,0]
else:
party_votes[1] = party_votes[1] + 1
party_dict[senator.getParty()] = party_votes
# all abstaining votes
for senator in bill.getAbstaining():
party_votes = party_dict.get(senator.getParty())
if party_votes == None:
party_dict[senator.getParty()] = [0,0,1,0]
else:
party_votes[2] = party_votes[2] + 1
party_dict[senator.getParty()] = party_votes
# all absent votes
for senator in bill.getAbsent():
party_votes = party_dict.get(senator.getParty())
if party_votes == None:
party_dict[senator.getParty()] = [0,0,0,1]
else:
party_votes[2] = party_votes[2] + 1
party_dict[senator.getParty()] = party_votes
# Displays the the vote tallys for each party in the vote rolls
breakdown_string = ""
for party in party_dict.iterkeys():
breakdown_string += "<p>" + party + ": "
breakdown_string += (str(party_dict.get(party)[0]) + " yea | " +
str(party_dict.get(party)[1]) + " nay | " +
str(party_dict.get(party)[2]) + " abstain | " +
str(party_dict.get(party)[3]) + " absent" + "</p>" )
# Table headers: Vote | Senator | Party | State
table_string = ""
for s in bill.getYea_Votes():
table_string += ('<tr><td>Yea</td>' +
'<td>' + s.getSenatorLink() +
'</td><td>' + s.getParty() +
'</td><td>' + s.getStateLink() +
'</td></tr>')
for s in bill.getNay_Votes():
table_string += ('<tr><td>Nay</td>' +
'<td>' + s.getSenatorLink() +
'</td><td>' + s.getParty() +
'</td><td>' + s.getStateLink() +
'</td></tr>')
for s in bill.getAbstaining():
table_string += ('<tr><td>Abstain</td>' +
'<td>' + s.getSenatorLink() +
'</td><td>' + s.getParty() +
'</td><td>' + s.getStateLink() +
'</td></tr>')
for s in bill.getAbsent():
table_string += ('<tr><td>Absent</td>' +
'<td>' + s.getSenatorLink() +
'</td><td>' + s.getParty() +
'</td><td>' + s.getStateLink() +
'</td></tr>')
fill_tags = {"BillName": bill.getQuestion(),
"VotesBreakdown": breakdown_string,
"BillType": bill.getType(),
"BillSession": session_link,
"BillDate": bill.getVoteDate().isoformat(),
"BillVotes": votes,
"SenatorTable": table_string}
content_string = billString.format(**fill_tags)
self.replacements["results"] = content_string
def makeCommitteePage(self, committee):
# Gets HTML from CommitteePageTemplate.html, fills in the info of the
# committee passed to it, and places it in the body of the generated
# page.
self.readTemplate()
committeeFile = open("Website/CommitteePageTemplate.html", "r")
committeeString = committeeFile.read()
session_link = ('<a href="index.py?page_type=session&session=' +
str(committee.getSession()) + '">' +
str(committee.getSession()) + "</a>")
# Table headers: Position | Senator | Party | State
table_string = ""
for senator_pair in committee.getSenators():
# A senator_pair has a senator object at index 0 and a string
# describing the senator's role in the committee at index 1.
senator = senator_pair[0]
table_string += ('<tr><td>' + senator_pair[1] +
'</td><td>' +
senator.getSenatorLink() + '</td>' +
'<td>' + senator.getParty() +
'</td><td>' +
senator.getStateLink() + '</td></tr>')
associated_string = ""
if committee.isSuper():
associated_list = committee.getAssociated()
if len(associated_list)>0:
associated_string += ("<h5>Sub-Committees:</h5>" +
'<ul id="committees">')
for pair in associated_list:
associated_string += ('<li><a href = "index.py?' +
"page_type=committee&committee='" +
str(pair[0]) + '">' + str(pair[1]) +
'</a></li>')
associated_string += "</ul>"
else:
pair = committee.getAssociated()[0]
associated_string += ("<p><strong>Super-Committee: </strong>" +
'<a href = "index.py?' +
"page_type=committee&committee='" +
str(pair[0]) + '">' + str(pair[1]) + '</a></p>')
fill_tags = {"CommitteeName": committee.getName(),
"SessionNumber": session_link,
"Supercommittee": associated_string,
"SenatorTable": table_string}
content_string = committeeString.format(**fill_tags)
self.replacements["results"] = content_string
def makeSenatorPage(self, senator, vote_pair_list):
# Gets HTML from SenatorPageTemplate.html, fills in the info of the
# senator passed to it, and places it in the body of the generated page.
self.readTemplate()
senatorFile = open("Website/SenatorPageTemplate.html", "r")
senatorString = senatorFile.read()
committee_list = ""
for committee_pair in senator.getCommittees():
# A committee_pair contains a committee object at index 0 and a
# string detailing the senator's role at index 1.
committee_list += ("<li>" + committee_pair[1] + " of the " +
committee_pair[0].getCommitteeLink() +
' during the <a href="index.py?page_type='
+ 'session&session=' +
str(committee_pair[0].getSession()) + '">' +
str(committee_pair[0].getSession()) +
"<sup>th</sup></a>" +
" congressional session</li>")
# Adds in a wikipedia link if it exists
wiki_string = ""
wiki_link = senator.getWikiLink()
if wiki_link != "":
wiki_string = '<p><strong>Wikipedia:</strong> ' + wiki_link + '</p>'
# Checks for empty birthdays
date_string = ""
if senator.getBirthday() != None:
date_string = senator.getBirthday().isoformat()
# Table headers: Date | Number | Bill Name | Vote
table_string = ""
for bill_pair in vote_pair_list:
# A bill_pair contains a bill object at index 0 and a string of
# the senator's vote at index 1.
if bill_pair[1] == "not_voting":
bill_pair[1] = "Absent"
elif bill_pair[1] == "present":
bill_pair[1] = "Abstain"
else:
bill_pair[1] = bill_pair[1].capitalize()
table_string += ('<tr><td>' +
bill_pair[0].getVoteDate().isoformat() +
'</td><td>' +
str(bill_pair[0].getRoll()) +
'</td><td>' +
bill_pair[0].getBillLink() +
'</td><td>' + bill_pair[1] +
'</td></tr>')
fill_tags = {"SenatorName": senator.getName(),
"Wikipedia": wiki_string,
"SenatorParty": senator.getParty(),
"SenatorStateLink": senator.getStateLink(),
"Birthday": date_string,
"Currently": senator.isCurrent(),
"CommitteeMemberList": committee_list,
"BillTable": table_string}
content_string = senatorString.format(**fill_tags)
self.replacements["results"] = content_string
def makeSenatorIndexPage(self, senator_list):
# Gets HTML from SenatorIndexPageTemplate.html, fills in the table with
# info on all of the senators, and places the HTML in the body of the
# generated page.
self.readTemplate()
senatorIndexFile = open("Website/SenatorIndexPageTemplate.html", "r")
senatorIndexString = senatorIndexFile.read()
# Table headers: Senator | Party | State
table_string = ""
for senator in senator_list:
table_string += ('<tr><td>' +
senator.getSenatorLink() +
'</td><td>' + senator.getParty() +
'</td><td>' +
senator.getStateLink() + '</td></tr>')
fill_tags = {"SenatorTable": table_string}
content_string = senatorIndexString.format(**fill_tags)
self.replacements["results"] = content_string
def makeBillIndexPage(self, bill_list):
# Gets HTML from BillIndexPageTemplate.html, fills in the table with
# info on all of the bills, and places the HTML in the body of the
# generated page.
self.readTemplate()
billIndexFile = open("Website/BillIndexPageTemplate.html", "r")
billIndexString = billIndexFile.read()
# Table headers: Date | # | Bill
table_string = ""
for bill in bill_list:
table_string += ('<tr><td>' + bill.getSessionLink() + '</td><td>' +
bill.getVoteDate().isoformat() +
'</td><td>' +
" s" + str(bill.getRoll()) +
'</td><td>' + bill.getBillLink() +
'</td></tr>')
fill_tags = {"BillData": table_string}
content_string = billIndexString.format(**fill_tags)
self.replacements["results"] = content_string
def makeStatePage(self, state_name, senator_list):
# Gets HTML from StatePageTemplate.html, fills in the table with
# info on the state's senators, and places the HTML in the body of the
# generated page.
self.readTemplate()
stateFile = open("Website/StatePageTemplate.html", "r")
stateString = stateFile.read()
# Table headers: Current? | Senator | Party
table_string = ""
for s in senator_list:
table_string += ('<tr><td>' + str(s.isCurrent()) +
'</td><td>' + s.getSenatorLink() +
'</td><td>' + s.getParty() +
'</td></tr>')
full_state_name = STATE_LIST[STATE_ABBREVIATION_LIST.index(state_name)]
fill_tags = {"StateName": full_state_name,
"SenatorTable": table_string}
content_string = stateString.format(**fill_tags)
self.replacements["results"] = content_string
def makeSessionPage(self, session):
# Gets HTML from SessionPageTemplate.html, fills in the two tables with
# info on the session's senators and bills, and places the HTML in the
# body of the generated page.
self.readTemplate()
sessionFile = open("Website/SessionPageTemplate.html", "r")
sessionString = sessionFile.read()
c_list_string = ""
committee_list = session.getCommittees()
for c in committee_list:
if c.isSuper():
c_list_string += ('<li>' + c.getCommitteeLink() + ' </li>')
# Table headers: Senator | Party | State
s_table_string = ""
senator_list = session.getSenators()
for s in senator_list:
s_table_string += ('<tr><td>' + s.getSenatorLink() +
'</td><td>' + s.getParty() +
'</td><td>' + s.getStateLink() + '</td></tr>')
# Table headers: Date | # | Bill
b_table_string = ""
bill_list = session.getBills()
for b in bill_list:
b_table_string += ('<tr><td>' +
b.getVoteDate().isoformat() +
'</td><td>' + str(b.getRoll()) +
'</td><td>' + b.getBillLink() + '</td></tr>')
fill_tags = {"sessionID": str(session.getId()),
"StartDate": session.getStart_Date().isoformat(),
"EndDate": session.getEnd_Date().isoformat(),
"CommitteeList": c_list_string,
"SenatorTable": s_table_string,
"BillTable": b_table_string}
content_string = sessionString.format(**fill_tags)
self.replacements["results"] = content_string
def makeErrorPage(self):
# Gets HTML from ErrorPageTemplate.html and places it in the body of the
# generated page.
self.readTemplate()
errorFile = open("Website/ErrorPageTemplate.html", "r")
errorString = errorFile.read()
self.replacements["results"] = errorString
def getPage(self):
# When the appropriate make function has been called, this places the
# edited body of the page in between the header and the footer, fleshes
# out the menu dropdowns, and returns the finished page.
self.page = self.page.format(**self.replacements)
return self.page
def __init__(self):
self.userDS = DataSource("database.pickle.user")
self.roomDS = DataSource("database.pickle.room")
self.uidToWsKeyDic = {}
self.uidToWs = {}
def __init__(self, parent, queue, timeout):
DataSource.__init__(self, parent, queue, timeout)
self.objDiff = None
class PageConstructor:
"""Contains all of the methods to access and fill out the HTML templates."""
global STATE_LIST
global STATE_ABBREVIATION_LIST
STATE_LIST = [
'Alabama', 'Alaska', 'Arizona', 'Arkansas', 'California', 'Colorado',
'Connecticut', 'Delaware', 'Florida', 'Georgia', 'Hawaii', 'Idaho',
'Illinois', 'Indiana', 'Iowa', 'Kansas', 'Kentucky', 'Louisiana',
'Maine', 'Maryland', 'Massachusetts', 'Michigan', 'Minnesota',
'Mississippi', 'Missouri', 'Montana', 'Nebraska', 'Nevada',
'New Hampshire', 'New Jersey', 'New Mexico', 'New York',
'North Carolina', 'North Dakota', 'Ohio', 'Oklahoma', 'Oregon',
'Pennsylvania', 'Rhode Island', 'South Carolina', 'South Dakota',
'Tennessee', 'Texas', 'Utah', 'Vermont', 'Virginia', 'Washington',
'West Virginia', 'Wisconsin', 'Wyoming'
]
STATE_ABBREVIATION_LIST = [
'AL', 'AK', 'AZ', 'AR', 'CA', 'CO', 'CT', 'DE', 'FL', 'GA', 'HI', 'ID',
'IL', 'IN', 'IA', 'KS', 'KY', 'LA', 'ME', 'MD', 'MA', 'MI', 'MN', 'MS',
'MO', 'MT', 'NE', 'NV', 'NH', 'NJ', 'NM', 'NY', 'NC', 'ND', 'OH', 'OK',
'OR', 'PA', 'RI', 'SC', 'SD', 'TN', 'TX', 'UT', 'VT', 'VA', 'WA', 'WV',
'WI', 'WY'
]
def __init__(self):
# The constructor opens a connection to the database, creates an empty
# string of HTML, and creates an empty dictionary to be filled with
# database-specific replacements to the HTML templates.
self.page = ""
self.replacements = {}
self.dbSource = DataSource()
def readTemplate(self):
# A method that reads template.html, adds the template to self.page, and
# adds the HTML for the dropdown menus to self.replacements.
templateFile = open("Website/template.html", "r")
templateString = templateFile.read()
self.page = templateString
self.replacements["SenatorDropdown"] = ""
i = 0
while i < len(STATE_LIST):
self.replacements["SenatorDropdown"] += (
'<li><a href = "index.py?page_type=state&state=' +
STATE_ABBREVIATION_LIST[i] + '">' + STATE_LIST[i] +
'</a></li>')
i += 1
committee_list = self.dbSource.getCommitteesBySession(114)
self.replacements["CommitteeDropdown"] = ""
for entry in committee_list:
self.replacements["CommitteeDropdown"] += (
'<li>' + entry.getCommitteeLink() + "</li>")
def makeHomepage(self):
# Gets HTML from HomepageTemplate.html and places it in the body of the
# generated page.
self.readTemplate()
pageFile = open("Website/HomepageTemplate.html", "r")
pageString = pageFile.read()
self.replacements["results"] = pageString
def makeBillPage(self, bill):
# Gets HTML from BillPageTemplate.html, fills in the info of the bill
# passed to it, and places it in the body of the generated page.
self.readTemplate()
billFile = open("Website/BillPageTemplate.html", "r")
billString = billFile.read()
session_link = ('<a href="index.py?page_type=session&session=' +
str(bill.getSession()) + '">' +
str(bill.getSession()) + "</a>")
votes = (str(len(bill.getYea_Votes())) + " yea | " +
str(len(bill.getNay_Votes())) + " nay | " +
str(len(bill.getAbstaining())) + " abstain | " +
str(len(bill.getAbsent())) + " absent")
# Loops through the vote data and addsa tally to a dictonary based
# on the party of the senator with keyed values that correspond
# to the votes in order
# all yea votes
party_dict = {}
for senator in bill.getYea_Votes():
party_votes = party_dict.get(senator.getParty())
if party_votes == None:
party_dict[senator.getParty()] = [1, 0, 0, 0]
else:
party_votes[0] = party_votes[0] + 1
party_dict[senator.getParty()] = party_votes
# all nay votes
for senator in bill.getNay_Votes():
party_votes = party_dict.get(senator.getParty())
if party_votes == None:
party_dict[senator.getParty()] = [0, 1, 0, 0]
else:
party_votes[1] = party_votes[1] + 1
party_dict[senator.getParty()] = party_votes
# all abstaining votes
for senator in bill.getAbstaining():
party_votes = party_dict.get(senator.getParty())
if party_votes == None:
party_dict[senator.getParty()] = [0, 0, 1, 0]
else:
party_votes[2] = party_votes[2] + 1
party_dict[senator.getParty()] = party_votes
# all absent votes
for senator in bill.getAbsent():
party_votes = party_dict.get(senator.getParty())
if party_votes == None:
party_dict[senator.getParty()] = [0, 0, 0, 1]
else:
party_votes[2] = party_votes[2] + 1
party_dict[senator.getParty()] = party_votes
# Displays the the vote tallys for each party in the vote rolls
breakdown_string = ""
for party in party_dict.iterkeys():
breakdown_string += "<p>" + party + ": "
breakdown_string += (str(party_dict.get(party)[0]) + " yea | " +
str(party_dict.get(party)[1]) + " nay | " +
str(party_dict.get(party)[2]) +
" abstain | " +
str(party_dict.get(party)[3]) + " absent" +
"</p>")
# Table headers: Vote | Senator | Party | State
table_string = ""
for s in bill.getYea_Votes():
table_string += ('<tr><td>Yea</td>' + '<td>' + s.getSenatorLink() +
'</td><td>' + s.getParty() + '</td><td>' +
s.getStateLink() + '</td></tr>')
for s in bill.getNay_Votes():
table_string += ('<tr><td>Nay</td>' + '<td>' + s.getSenatorLink() +
'</td><td>' + s.getParty() + '</td><td>' +
s.getStateLink() + '</td></tr>')
for s in bill.getAbstaining():
table_string += ('<tr><td>Abstain</td>' + '<td>' +
s.getSenatorLink() + '</td><td>' + s.getParty() +
'</td><td>' + s.getStateLink() + '</td></tr>')
for s in bill.getAbsent():
table_string += ('<tr><td>Absent</td>' + '<td>' +
s.getSenatorLink() + '</td><td>' + s.getParty() +
'</td><td>' + s.getStateLink() + '</td></tr>')
fill_tags = {
"BillName": bill.getQuestion(),
"VotesBreakdown": breakdown_string,
"BillType": bill.getType(),
"BillSession": session_link,
"BillDate": bill.getVoteDate().isoformat(),
"BillVotes": votes,
"SenatorTable": table_string
}
content_string = billString.format(**fill_tags)
self.replacements["results"] = content_string
def makeCommitteePage(self, committee):
# Gets HTML from CommitteePageTemplate.html, fills in the info of the
# committee passed to it, and places it in the body of the generated
# page.
self.readTemplate()
committeeFile = open("Website/CommitteePageTemplate.html", "r")
committeeString = committeeFile.read()
session_link = ('<a href="index.py?page_type=session&session=' +
str(committee.getSession()) + '">' +
str(committee.getSession()) + "</a>")
# Table headers: Position | Senator | Party | State
table_string = ""
for senator_pair in committee.getSenators():
# A senator_pair has a senator object at index 0 and a string
# describing the senator's role in the committee at index 1.
senator = senator_pair[0]
table_string += ('<tr><td>' + senator_pair[1] + '</td><td>' +
senator.getSenatorLink() + '</td>' + '<td>' +
senator.getParty() + '</td><td>' +
senator.getStateLink() + '</td></tr>')
associated_string = ""
if committee.isSuper():
associated_list = committee.getAssociated()
if len(associated_list) > 0:
associated_string += ("<h5>Sub-Committees:</h5>" +
'<ul id="committees">')
for pair in associated_list:
associated_string += ('<li><a href = "index.py?' +
"page_type=committee&committee='" +
str(pair[0]) + '">' + str(pair[1]) +
'</a></li>')
associated_string += "</ul>"
else:
pair = committee.getAssociated()[0]
associated_string += ("<p><strong>Super-Committee: </strong>" +
'<a href = "index.py?' +
"page_type=committee&committee='" +
str(pair[0]) + '">' + str(pair[1]) +
'</a></p>')
fill_tags = {
"CommitteeName": committee.getName(),
"SessionNumber": session_link,
"Supercommittee": associated_string,
"SenatorTable": table_string
}
content_string = committeeString.format(**fill_tags)
self.replacements["results"] = content_string
def makeSenatorPage(self, senator, vote_pair_list):
# Gets HTML from SenatorPageTemplate.html, fills in the info of the
# senator passed to it, and places it in the body of the generated page.
self.readTemplate()
senatorFile = open("Website/SenatorPageTemplate.html", "r")
senatorString = senatorFile.read()
committee_list = ""
for committee_pair in senator.getCommittees():
# A committee_pair contains a committee object at index 0 and a
# string detailing the senator's role at index 1.
committee_list += ("<li>" + committee_pair[1] + " of the " +
committee_pair[0].getCommitteeLink() +
' during the <a href="index.py?page_type=' +
'session&session=' +
str(committee_pair[0].getSession()) + '">' +
str(committee_pair[0].getSession()) +
"<sup>th</sup></a>" +
" congressional session</li>")
# Adds in a wikipedia link if it exists
wiki_string = ""
wiki_link = senator.getWikiLink()
if wiki_link != "":
wiki_string = '<p><strong>Wikipedia:</strong> ' + wiki_link + '</p>'
# Checks for empty birthdays
date_string = ""
if senator.getBirthday() != None:
date_string = senator.getBirthday().isoformat()
# Table headers: Date | Number | Bill Name | Vote
table_string = ""
for bill_pair in vote_pair_list:
# A bill_pair contains a bill object at index 0 and a string of
# the senator's vote at index 1.
if bill_pair[1] == "not_voting":
bill_pair[1] = "Absent"
elif bill_pair[1] == "present":
bill_pair[1] = "Abstain"
else:
bill_pair[1] = bill_pair[1].capitalize()
table_string += ('<tr><td>' +
bill_pair[0].getVoteDate().isoformat() +
'</td><td>' + str(bill_pair[0].getRoll()) +
'</td><td>' + bill_pair[0].getBillLink() +
'</td><td>' + bill_pair[1] + '</td></tr>')
fill_tags = {
"SenatorName": senator.getName(),
"Wikipedia": wiki_string,
"SenatorParty": senator.getParty(),
"SenatorStateLink": senator.getStateLink(),
"Birthday": date_string,
"Currently": senator.isCurrent(),
"CommitteeMemberList": committee_list,
"BillTable": table_string
}
content_string = senatorString.format(**fill_tags)
self.replacements["results"] = content_string
def makeSenatorIndexPage(self, senator_list):
# Gets HTML from SenatorIndexPageTemplate.html, fills in the table with
# info on all of the senators, and places the HTML in the body of the
# generated page.
self.readTemplate()
senatorIndexFile = open("Website/SenatorIndexPageTemplate.html", "r")
senatorIndexString = senatorIndexFile.read()
# Table headers: Senator | Party | State
table_string = ""
for senator in senator_list:
table_string += ('<tr><td>' + senator.getSenatorLink() +
'</td><td>' + senator.getParty() + '</td><td>' +
senator.getStateLink() + '</td></tr>')
fill_tags = {"SenatorTable": table_string}
content_string = senatorIndexString.format(**fill_tags)
self.replacements["results"] = content_string
def makeBillIndexPage(self, bill_list):
# Gets HTML from BillIndexPageTemplate.html, fills in the table with
# info on all of the bills, and places the HTML in the body of the
# generated page.
self.readTemplate()
billIndexFile = open("Website/BillIndexPageTemplate.html", "r")
billIndexString = billIndexFile.read()
# Table headers: Date | # | Bill
table_string = ""
for bill in bill_list:
table_string += ('<tr><td>' + bill.getSessionLink() + '</td><td>' +
bill.getVoteDate().isoformat() + '</td><td>' +
" s" + str(bill.getRoll()) + '</td><td>' +
bill.getBillLink() + '</td></tr>')
fill_tags = {"BillData": table_string}
content_string = billIndexString.format(**fill_tags)
self.replacements["results"] = content_string
def makeStatePage(self, state_name, senator_list):
# Gets HTML from StatePageTemplate.html, fills in the table with
# info on the state's senators, and places the HTML in the body of the
# generated page.
self.readTemplate()
stateFile = open("Website/StatePageTemplate.html", "r")
stateString = stateFile.read()
# Table headers: Current? | Senator | Party
table_string = ""
for s in senator_list:
table_string += ('<tr><td>' + str(s.isCurrent()) + '</td><td>' +
s.getSenatorLink() + '</td><td>' + s.getParty() +
'</td></tr>')
full_state_name = STATE_LIST[STATE_ABBREVIATION_LIST.index(state_name)]
fill_tags = {
"StateName": full_state_name,
"SenatorTable": table_string
}
content_string = stateString.format(**fill_tags)
self.replacements["results"] = content_string
def makeSessionPage(self, session):
# Gets HTML from SessionPageTemplate.html, fills in the two tables with
# info on the session's senators and bills, and places the HTML in the
# body of the generated page.
self.readTemplate()
sessionFile = open("Website/SessionPageTemplate.html", "r")
sessionString = sessionFile.read()
c_list_string = ""
committee_list = session.getCommittees()
for c in committee_list:
if c.isSuper():
c_list_string += ('<li>' + c.getCommitteeLink() + ' </li>')
# Table headers: Senator | Party | State
s_table_string = ""
senator_list = session.getSenators()
for s in senator_list:
s_table_string += ('<tr><td>' + s.getSenatorLink() + '</td><td>' +
s.getParty() + '</td><td>' + s.getStateLink() +
'</td></tr>')
# Table headers: Date | # | Bill
b_table_string = ""
bill_list = session.getBills()
for b in bill_list:
b_table_string += ('<tr><td>' + b.getVoteDate().isoformat() +
'</td><td>' + str(b.getRoll()) + '</td><td>' +
b.getBillLink() + '</td></tr>')
fill_tags = {
"sessionID": str(session.getId()),
"StartDate": session.getStart_Date().isoformat(),
"EndDate": session.getEnd_Date().isoformat(),
"CommitteeList": c_list_string,
"SenatorTable": s_table_string,
"BillTable": b_table_string
}
content_string = sessionString.format(**fill_tags)
self.replacements["results"] = content_string
def makeErrorPage(self):
# Gets HTML from ErrorPageTemplate.html and places it in the body of the
# generated page.
self.readTemplate()
errorFile = open("Website/ErrorPageTemplate.html", "r")
errorString = errorFile.read()
self.replacements["results"] = errorString
def getPage(self):
# When the appropriate make function has been called, this places the
# edited body of the page in between the header and the footer, fleshes
# out the menu dropdowns, and returns the finished page.
self.page = self.page.format(**self.replacements)
return self.page
class UserInputParser:
"""Parses the user's input and passes the information between
PageConstructor and DataSource."""
def __init__(self, params):
# A constructor that stores the provided parameters internally, as well
# as calling both PageConstructor() and DataSource().
self.page_type = params['page_type']
self.params = params
self.page_maker = PageConstructor()
self.db_source = DataSource()
def generateHtmlPageOutput(self):
# Determines which type of page we're constructing, then calls the
# appropriate make____Page() method to generate the HTML.
HTML_string = ""
# The extra bits on the conditionals check that it's actually given a
# specific senator/state/bill by the CGI parameters. If not, it'll just
# go into the else.
function_dictionary = {"senator": self.makeSenatorPage,
"bill": self.makeBillPage, "state": self.makeStatePage,
"committee": self.makeCommitteePage,
"session": self.makeSessionPage,
"bill_index": self.makeBillIndexPage,
"senator_index": self.makeSenatorIndexPage,
"home": self.makeHomePage}
try:
HTML_string = function_dictionary[self.page_type]()
except:
HTML_string = self.makeErrorPage()
return HTML_string
# The "make" methods all do about the same thing, but with enough
# variations that it'd be a pain to make them all one. They get the major
# info we need from the database, then call the requisite PageConstructor
# method to make the page, after which they get it back as a string and
# return it.
#
# For Senators, Bills & Committees, the object returned by DataSource.py
# will be an object of the appropriate type; otherwise, it'll be a list of
# objects.
def makeSenatorPage(self):
id_tag = self.params["senator"]
senator_obj = self.db_source.getSenatorWithCommittees(id_tag)
senator_vote_pair = self.db_source.getVotesBySenator(id_tag,0)
self.page_maker.makeSenatorPage(senator_obj,senator_vote_pair)
HTML_string = self.page_maker.getPage()
return HTML_string
def makeSenatorIndexPage(self):
senator_list = self.db_source.getSenatorList()
self.page_maker.makeSenatorIndexPage(senator_list)
HTML_string = self.page_maker.getPage()
return HTML_string
def makeBillPage(self):
id_tag = self.params["bill"]
bill_obj = self.db_source.getBillWithVotes(id_tag)
self.page_maker.makeBillPage(bill_obj)
HTML_string = self.page_maker.getPage()
return HTML_string
def makeBillIndexPage(self):
bill_list = self.db_source.getBillList()
self.page_maker.makeBillIndexPage(bill_list)
HTML_string = self.page_maker.getPage()
return HTML_string
def makeStatePage(self):
state_name = self.params["state"]
senator_list = self.db_source.getSenatorsInState(state_name)
self.page_maker.makeStatePage(state_name, senator_list)
HTML_string = self.page_maker.getPage()
return HTML_string
def makeCommitteePage(self):
committee_id = self.params["committee"]
committee_obj = self.db_source.getCommitteeWithMembers(committee_id)
self.page_maker.makeCommitteePage(committee_obj)
HTML_string = self.page_maker.getPage()
return HTML_string
def makeSessionPage(self):
session_id = self.params["session"]
session = self.db_source.getSessionObject(session_id)
self.page_maker.makeSessionPage(session)
HTML_string = self.page_maker.getPage()
return HTML_string
def makeHomePage(self):
self.page_maker.makeHomepage()
HTML_string = self.page_maker.getPage()
return HTML_string
def makeErrorPage(self):
self.page_maker.makeErrorPage()
HTML_string = self.page_maker.getPage()
return HTML_string
def execute(self):
return DataSource().get(self.x, self.y)
def test_trim_time_series_with_missing_start_date_and_history_window(self):
start_date = pd.to_datetime('2015-12-01')
end_date = pd.to_datetime('2016-01-06')
with self.assertRaises(IndexError):
trimed_series = DataSource('test').trim_series_observations(self.test_series, 'dummy ticker', \
'dummy series name', start_date, end_date, 1)
from DataSource import DataSource
ds = DataSource()
ds.put("1","1")
print ds.get("1")
print ds.get("2")
ds.save()
def __init__(self, parent, queue, timeout):
DataSource.__init__(self, parent, queue, timeout)
self.objDiff = None
from DataSource import DataSource
from MLP import get_single_mlp, compile_model, train_model, auto_fit, select_best_model
# Datasource settings
filename = "../data/Titanic_train.csv"
target = 'Survived'
ignore_fields = ['Name', 'Cabin', 'Ticket']
# Network settings
use_all = False
hidden_layers = [1, 2]
units = [16, 32]
optimizers = ['adam']
lrs = [0.01]
ds = DataSource(filename)
ds.data_load_split(target=[target], ignore=ignore_fields)
ds.define_problem()
ds.train_val_split(ratio=0.2, random_state=42)
ds.data_preprocess(ds.X_train, ds.y_train, train_set=True)
ds.data_preprocess(ds.X_val, ds.y_val, train_set=False)
X_train, y_train = ds.trans_X_train, ds.trans_y_train
X_val, y_val = ds.trans_X_val, ds.trans_y_val
def train(hidden_layers, units, optimizers, lrs, use_all=False):
# fit models
start_time = time.time()
models, param_info, val_losses = auto_fit(ds.problem,
def __init__(self):
self.dao = DataSource.Instance()
self.LOC_TABLE = 'location'
self.WS_TABLE = 'websocket'
self.dao.makeTables([self.LOC_TABLE, self.WS_TABLE])
class UserInputParser:
"""Parses the user's input and passes the information between
PageConstructor and DataSource."""
def __init__(self, params):
# A constructor that stores the provided parameters internally, as well
# as calling both PageConstructor() and DataSource().
self.page_type = params['page_type']
self.params = params
self.page_maker = PageConstructor()
self.db_source = DataSource()
def generateHtmlPageOutput(self):
# Determines which type of page we're constructing, then calls the
# appropriate make____Page() method to generate the HTML.
HTML_string = ""
# The extra bits on the conditionals check that it's actually given a
# specific senator/state/bill by the CGI parameters. If not, it'll just
# go into the else.
function_dictionary = {
"senator": self.makeSenatorPage,
"bill": self.makeBillPage,
"state": self.makeStatePage,
"committee": self.makeCommitteePage,
"session": self.makeSessionPage,
"bill_index": self.makeBillIndexPage,
"senator_index": self.makeSenatorIndexPage,
"home": self.makeHomePage
}
try:
HTML_string = function_dictionary[self.page_type]()
except:
HTML_string = self.makeErrorPage()
return HTML_string
# The "make" methods all do about the same thing, but with enough
# variations that it'd be a pain to make them all one. They get the major
# info we need from the database, then call the requisite PageConstructor
# method to make the page, after which they get it back as a string and
# return it.
#
# For Senators, Bills & Committees, the object returned by DataSource.py
# will be an object of the appropriate type; otherwise, it'll be a list of
# objects.
def makeSenatorPage(self):
id_tag = self.params["senator"]
senator_obj = self.db_source.getSenatorWithCommittees(id_tag)
senator_vote_pair = self.db_source.getVotesBySenator(id_tag, 0)
self.page_maker.makeSenatorPage(senator_obj, senator_vote_pair)
HTML_string = self.page_maker.getPage()
return HTML_string
def makeSenatorIndexPage(self):
senator_list = self.db_source.getSenatorList()
self.page_maker.makeSenatorIndexPage(senator_list)
HTML_string = self.page_maker.getPage()
return HTML_string
def makeBillPage(self):
id_tag = self.params["bill"]
bill_obj = self.db_source.getBillWithVotes(id_tag)
self.page_maker.makeBillPage(bill_obj)
HTML_string = self.page_maker.getPage()
return HTML_string
def makeBillIndexPage(self):
bill_list = self.db_source.getBillList()
self.page_maker.makeBillIndexPage(bill_list)
HTML_string = self.page_maker.getPage()
return HTML_string
def makeStatePage(self):
state_name = self.params["state"]
senator_list = self.db_source.getSenatorsInState(state_name)
self.page_maker.makeStatePage(state_name, senator_list)
HTML_string = self.page_maker.getPage()
return HTML_string
def makeCommitteePage(self):
committee_id = self.params["committee"]
committee_obj = self.db_source.getCommitteeWithMembers(committee_id)
self.page_maker.makeCommitteePage(committee_obj)
HTML_string = self.page_maker.getPage()
return HTML_string
def makeSessionPage(self):
session_id = self.params["session"]
session = self.db_source.getSessionObject(session_id)
self.page_maker.makeSessionPage(session)
HTML_string = self.page_maker.getPage()
return HTML_string
def makeHomePage(self):
self.page_maker.makeHomepage()
HTML_string = self.page_maker.getPage()
return HTML_string
def makeErrorPage(self):
self.page_maker.makeErrorPage()
HTML_string = self.page_maker.getPage()
return HTML_string
DataSource.NAME:'PhoneArena',
DataSource.PATH:'../scraping/data/sentences_data.txt'
},
{
DataSource.NAME:'Twitter',
DataSource.PATH:'../../data/SemEval-2014.csv'
}
]
srcs = [{DataSource.NAME:src[DataSource.NAME],
DataSource.PATH:os.path.abspath(src[DataSource.PATH]),
DataSource.TYPE:os.path.splitext(src[DataSource.PATH])[1]} for src in srcs]
for src in srcs:
print("Name: '%s'\nType: '%s'" % (src[DataSource.NAME], src[DataSource.TYPE]))
dSrc = DataSource(src)
srcStats = dSrc.getStats()
print("\nUsable data:")
pprint.pprint(srcStats[DataSource.REGULAR])
print("\nUnusable data:")
pprint.pprint(srcStats[DataSource.BROKEN])
print("-----------------")
print("Finished!")
for src in srcs:
srcName = src[DataSource.NAME]
dSrc = DataSource(src)
srcStats = dSrc.getStats()
kws,ocs = [],[]
for word in srcStats[DataSource.REGULAR].items():
def __init__(self, parent, queue, url, timeout):
DataSource.__init__(self, parent, queue, timeout)
self.url_info = url + "/api.php?mode=info"
self.url_coins = url + "/api.php?mode=coins"
self.objDiff = None
self.objReward = None
def __init__(self, parent, url, queue, timeout):
DataSource.__init__(self, parent, queue, timeout)
self.url_info = url
self.objInfo = None
