def deep_logit(input_dim, hidden_layer_dims, output_dim, learning_rate, reg_rate, batch_size, max_num_iters):
x_train = tf.placeholder(tf.float32)
y_labels = tf.placeholder(tf.float32)
x_train_flattened = vectorize_input(x_train, batch_size)
# params_conv = init_conv_params(filter_size=3, num_input_channels=1, num_filters=1, strides=[1, 1, 1, 1])
# conv_op = conv_layer_forward(params_conv, x_train)
vectorised_conv_output = vectorize_input(x_train_flattened, batch_size)
[logit, _, params] = deep_softmax_forward_pass(vectorised_conv_output, input_dim, hidden_layer_dims, output_dim)
data_loss = tf.nn.softmax_cross_entropy_with_logits(logit, y_labels)
regularization_loss = compute_regularization_loss(params, reg_rate)
loss = tf.add(data_loss, regularization_loss)
optimizer = tf.train.MomentumOptimizer(learning_rate, momentum=.9)
train_op = optimizer.minimize(loss)
db = DataParser()
with tf.Session() as sess:
init_op = tf.initialize_all_variables()
sess.run(init_op)
for _ in range(max_num_iters):
# Get batch data:
[batch_x, batch_labels] = db.get_next_batch(batch_size)
# Single back propagation + update operation
[_, curr_loss, _] = sess.run([train_op, loss, logit], {x_train: batch_x, y_labels: batch_labels})
return curr_loss
def main():
searchType = input(
"Welcome to OSHA Compliant HeatMiser! \nPlease select your option by pressing the appropriate number:"
+ " 1 for decision tree, 2 for k means clustering, or 3 to quit: ")
if (searchType == "1"):
print("Decision tree approach selected!")
search = 0
dt = DecisionTree()
data = dp.getDataArrays()
real_f_ones = ten_fold_cross_validation(data)
fake_f_ones = generate_baseline(data)
plotComparison(real_f_ones, fake_f_ones)
elif (searchType == "2"):
print(
"K means clustering approach selected! Running model with optimal k = 2"
)
data = dp.getDataList()
kCluster = KMeansClustering(2)
kCluster.run(data)
# Uncomment below to get either OSHA (1) or location (2) plot
# getStatusPlot(data, 2)
# Uncomment below to run elbow method
# determineOptimalK(data)
elif (searchType == "3"):
print("Shutting down...")
else:
print("Sorry, that was an incorrect command. Shutting down...")
sys.exit()
def read_data(file_path):
parser = DataParser(file_path, ',', True)
features = []
labels = []
for row in parser.readrows():
feature = list()
feature.append(float(row['density']))
feature.append(float(row['sugar_content']))
features.append(feature)
labels.append(int(row['label']))
return features,labels
def checkQueue():
try:
data, source = mainQueue.get(timeout=0.05)
if (type(source) == RobotServer.RobotServer
or type(source) == DatabaseConnection.DatabaseConnection):
checkCommand(DataParser.parseData(data))
elif (type(source) == RobotMovement.RobotMovement):
serverThread.insertServerQueue(serverThread.sendData,
DataParser.createServerPacket(data))
if (data["robotID"] != None):
dbThread.insertDBQueue(dbThread.sendData,
DataParser.createDatabasePacket(data))
except Queue.Empty:
return
def train_neural_model(model):
DataParser.TOP = TOP
x_train, x_test, y_train, y_test = DataParser.get_splitted_data_for_model(
LOAD_DATA_FROM_FILE)
print(x_train.shape, y_train.shape)
# checkpoint_name = './modelDP/Weights-{epoch:03d}--{val_loss:.5f}.hdf5'
# checkpoint_name = './modelDP/Weights-best1.hdf5'
checkpoint = keras.callbacks.ModelCheckpoint(MODEL_FILE_NAME,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='auto')
earlyStopping = keras.callbacks.EarlyStopping(patience=16, mode="auto")
tensorBoard = keras.callbacks.TensorBoard(log_dir="./logs")
callbacks_list = [checkpoint, earlyStopping]
# model = create_neural_model()
model.fit(x_train,
y_train,
epochs=EPCHOES,
batch_size=444,
validation_split=0.2,
callbacks=callbacks_list)
print("Current one: ", model.evaluate(x_test, y_test, batch_size=44))
model.load_weights(MODEL_FILE_NAME)
print("Best one: ", model.evaluate(x_test, y_test, batch_size=44))
def __init__(self, isTrain, maxLength=0, n_mfcc=N_MFCC):
if isTrain in [True, False]:
self.N_MFCC = n_mfcc
self.isTrain = isTrain
self.maxLength = maxLength
self.dataParser = DataParser.DataParser()
self.vad = VAD.VAD()
self.dataPathDictionary = self.__createDicitonaryForDataPath__()
self.__create_random_noise_files__()
self.__makeEverySpectogramSame__()
def __init__(self):
# type: () -> None
self.plotter = PlotMaker()
self.fitter = FitFinder()
self.data_parser = DataParser()
self.use_fills = False
self.make_fits = False
self.use_fit_file = False # Currently unused outside of setupCheck
self.update_online_fits = False
self.plotter.compare_fits = False # Compare fits to multiple groups of runs
self.use_pileup = True # plot <PU> vs. rate
self.use_lumi = False # plot iLumi vs. rate
self.use_LS = False # plot LS vs. rate
self.use_stream_bandwidth = False
self.use_stream_size = False
self.use_grouping = False # Creates sub directories to group the outputs, utilizes self.group_map
# TESTING: START #
self.certify_mode = False
# TESTING: END #
self.group_map = {} # {'group_name': [trigger_name] }
self.fill_list = [] # Fills to get data from, Currently Unused
self.run_list = [] # Runs to get data from
self.object_list = [
] # List of *ALL* objects to plot, except for when using grouping
self.ops = None # The options specified at the command line
self.rate_mon_dir = os.getcwd()
self.save_dir = os.path.join(os.getcwd(), "tmp_rate_plots")
self.certify_dir = os.path.join(os.getcwd(), "tmp_certify_dir")
self.online_fits_dir = os.path.join(os.getcwd(), "Fits")
self.fit_file = None # Currently Unused
def run(self):
while 1:
xml_text = self.in_queue.get()
print "Received XML"
if xml_text == "None":
self.out_queue.put("None")
break
json_doc = DataParser.parse_get_state_stats_resp(xml_text)
print "Made JSON"
self.out_queue.put(json_doc)
def InsertData_DB( Type, RawData , DEVICE_ID ):
if( Type == "wst" ): Records = DataParser.get_wifi(RawData.split('\n'))
if( Type == "pst" ): Records = DataParser.get_phonestate(RawData.split('\n'))
if( Type == "bst" ): Records = DataParser.get_battery(RawData.split('\n'))
if( Type == "btst" ): Records = DataParser.get_bluetooth(RawData.split('\n'))
if( Type == "gst" ): Records = DataParser.get_gsm(RawData.split('\n'))
if( Type == "lst" ): Records = DataParser.get_location(RawData.split('\n'))
for record in Records:
DataObject = None
if( Type == "wst" ):
DataObject = WiFi( DeviceID=DEVICE_ID , **record )
if( Type == "pst" ):
DataObject = Phone( DeviceID=DEVICE_ID , **record )
if( Type == "bst" ):
DataObject = Battery( DeviceID=DEVICE_ID , **record )
if( Type == "btst" ):
DataObject = Bluetooth( DeviceID=DEVICE_ID , **record )
if( Type == "gst" ):
DataObject = Cellular( DeviceID=DEVICE_ID , **record )
if( Type == "lst" ):
DataObject = GPS( DeviceID=DEVICE_ID , **record )
DataObject.save()
def scatter_sample():
parser = DataParser('./watermelon_data_3a.txt', ',', True)
dataset = parser.load_data()
X = dataset[:, 0:2]
y = dataset[:, 2]
f1 = plt.figure(1)
plt.title('watermelon_3a')
plt.xlabel('density')
plt.ylabel('sugar_content')
plt.scatter(X[y == 0, 0],
X[y == 0, 1],
marker='o',
color='k',
s=100,
label='bad')
plt.scatter(X[y == 1, 0],
X[y == 1, 1],
marker='o',
color='g',
s=100,
label='good')
plt.legend(loc='upper right')
plt.show()
def run(self):
text = self.text_tup[0]
location = self.text_tup[1]
writers_params = self.text_tup[2]
writers = writers_params["writers"]
locks = writers_params["locks"]
match_rule = writers_params["match_rule"]
metadata_table = writers_params["metadata_table"]
print "parsing", location
fluentd_accum = " "
hbase_accum = " "
fluentd_accum, hbase_accum = DataParser.parse_get_state_stats_resp(text)
DatastoreWriterWorker.DatastoreWriterWorker( ( {"fluentd":fluentd_accum,"hbase":hbase_accum}, location) , writers, locks, match_rule, metadata_table).start()
print "done parsing", location
def __init__(self):
self.level_manager = LevelData.LevelProgressionManager()
"""
tile_bindings is used to give our tiles a meaning.
Each TileMap must be presented with a BindingType dictionary in order to understand what a number in a tilemap file means.
For example, we could create a dictionary that includes {"0", ["Solid"]}, {"1", ["Solid", "Damage"] }
When our tilemap parses in a file that contains a 0, that tile will have a binding_type of "Solid"
When our tilemap parses in a 1, that tile will have a binding_type of "Solid" and "Damage"
It is up to you to give these tiles meaning in your game.
"""
tile_bindings = {}
tile_bindings["0"] = ["Passthrough"]
tile_bindings["1"] = ["Solid"]
tile_bindings["2"] = ["Hazard"]
tile_bindings["3"] = ["Warp"]
tile_bindings["r"] = ["Reload"]
#Load all of our levels. MapCount is the number of levels we need to load. (Levels start at index 0.)
mapCount = 8
for i in range(mapCount):
data_path = "..//Maps//Level" + str(i) + "//Data.txt"
dta = DataParser.parse(data_path)
prefix = "..//Maps//Level" + str(i) + "//Lv" + str(i)
tile_path = prefix + "_TileMap.txt"
collision_path = prefix + "_CollisionMap.txt"
object_path = prefix + "_ObjectMap.txt"
level = "Level " + str(i)
self.level_manager.levels.append(level)
tile_map = r.SpriteSheet()
tile_map.load_texture("..//Textures//" + dta[0],
cell_size=r.Vector2(16, 16))
_map = r.TileMap(tile_map, tile_bindings, tile_path,
collision_path, object_path, LevelData.ObjAry,
level)
r.TileMapManager.add_map(_map)
#Change this variable to the level you want to test.
level_test = 0
self.level_manager.current_level = (level_test - 1)
self.level_manager.load_next_level()
#Prepare our hero tile for use.
self.hero = LevelData.load_character(r.TileMapManager.active_map, self)
self.hero.set_default_pos(r.TileMapManager.active_map.start_location)
r.Ragnarok.get_world().add_obj(self.hero)
def main(_):
x_train, x_test, y_train, y_test = DataParser.get_data_for_model(
LOAD_DATA_FROM_FILE)
c_t = []
c_test = []
xs = tf.placeholder('float')
ys = tf.placeholder('float')
output = neural_net_model(xs, NUM_OF_COLS)
cost = tf.reduce_mean(tf.square(output - ys))
train = tf.train.GradientDescentOptimizer(0.001).minimize(cost)
with tf.Session() as sess:
with tf.device("/cpu:0"):
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
# saver.restore(sess, MODEL_FILE_NAME)
for epoch in range(EPCHOES):
for j in range(x_train.shape[0]):
sess.run([cost, train],
feed_dict={
xs: x_train[j, :].reshape(1, NUM_OF_COLS),
ys: y_train[j]
})
c_t.append(sess.run(cost, feed_dict={
xs: x_train,
ys: y_train
}))
c_test.append(
sess.run(cost, feed_dict={
xs: x_test,
ys: y_test
}))
print("Epoch ", epoch, " cost: ", c_test[epoch])
# pred = sess.run(output, feed_dict={xs:x_test})
saver.save(sess, MODEL_FILE_NAME + "test.ckpt")
print("model saved")
_start_shell(locals())
def test_model(test_x, test_y, model_dict, checkpoint_path):
'''
Todo: test the model
:param test_x: test data
:param test_y: test labels
:param model_dict: model dictionary that store the graph
:param checkpoint_path: the checkpoint of the model
:return: predicted results of the input data, with the dimension of (length, 1, 1)
'''
TRAIN_POINTS = test_x.shape[0]
input_test_x = test_x[0, :, :][np.newaxis, :, :]
test_pred = np.zeros(shape=(TRAIN_POINTS, 1, 1))
loss = 0
with tf.Session(graph=model_dict['graph']) as sess:
# load the model
print('[INFO] Loading model...')
checkpoint = tf.train.latest_checkpoint(checkpoint_path)
saver = tf.train.Saver()
saver.restore(sess, checkpoint)
print(
"[INFO] model restore from the checkpoint {0}".format(checkpoint))
print('[INFO] Testing...')
for i in range(TRAIN_POINTS):
# input_train_x = train_x[i,:,:][np.newaxis,:,:]
input_test_y = test_y[i, :, :][np.newaxis, :, :]
feed_dict = {
model_dict['input_x']: input_test_x,
model_dict['input_y']: input_test_y,
model_dict['batch_size_ph']: 1
}
lossV, pred_tmp = sess.run(
[model_dict['loss'], model_dict['output']],
feed_dict=feed_dict)
loss += lossV
input_test_x = dp.get_test_data(input_test_x, pred_tmp[:, -1, :])
test_pred[i, :, :] = pred_tmp[:, -1, :]
print('[INFO] Testing over...')
print('Testing loss: %f' % (loss / TRAIN_POINTS))
return test_pred
def initData(): '''initData Initializes the data from the generated QT data set args: None return: [gdict, fig, ax, limits] this is the variable pack. Saved because gdict can be very large. ''' plt.clf() #ion is interactive mode on. This makes it so the window doesn't close after processing plt.ion() #Get data from the quadtree file gdict,limits,rectVects = DataParser.parseQT() fig, ax = Plotting.plotBox(gdict, limits, rectVects) ####this is pack return [gdict, fig, ax, limits]
import DataParser
import ClassFilter
import TimetableMaker
print("started")
# Mock user data
semester = 1
classesId = [178880, 178754]
# Parse csv into array of stripped data
dataRawArray = DataParser.csvIntoRawArray(DataParser.openFile("HorariosRaw.csv"))
dataStrippedArray = DataParser.rawArrayToStrippedArray(dataRawArray)
# Filter and separate classes
aulas = {}
for disciplina in classesId:
possibleClasses = ClassFilter.possibleClasses(dataStrippedArray, semester, disciplina) # All classes for the semester
noDuplicates = ClassFilter.removeDuplicates(possibleClasses, True)
aulas[disciplina] = ClassFilter.groupByType(noDuplicates)
# Make timetable
possibleCombinations = TimetableMaker.possibleCombinations(aulas)
noOverlaps = TimetableMaker.removeOverlaps(aulas, possibleCombinations)
# Output to Excel
wb = TimetableMaker.outputExcel(aulas, noOverlaps)
# Output to html
TimetableMaker.convertExcelToWeb(wb)
scores.append({'mean': mean, 'std': std, 'params': param})
newlist = list()
newlist = sorted(scores, key=lambda k: k['mean'], reverse=True)
melhoresParams = newlist[:5]
melhoresScore = scores.copy()
classificadores[nome][2] = melhoresParams.copy()
classificadores[nome][3] = melhoresScore.copy()
if __name__ == "__main__":
inputDir = "./DataSets/Raw/"
inputFiles = FileUtils.abrirDataSets(inputDir)
inputFiles = DataParser.parseFiles(inputFiles)
outputDir = "./DadosColetados/"
outputFiles = dict()
for file in inputFiles:
outputFiles[file] = outputDir + file + ".csv"
outputData = dict()
for file in inputFiles:
outputData[file] = dict()
if file == "Zoo": n_splits = 3
elif file == "Poker": n_splits = 5
elif file == "Flags": n_splits = 4
else: n_splits = 10
matched_r = db_match_suite(street_parameters, r)
if matched_r:
result = _map_to_dict(matched_r)
print "Suite match: {}-{} {}, suite {}".format(
result["start_address"], result["end_address"], result["street_name"], result["db_suite"]
)
return result
else:
print "Multiple results returned for address, no matching suite found."
else:
pass
return None
### MAIN
row_dicts = DP.get_csv_dict(INPUT_ADDRESSES, YEAR)
print "Rows for %s:" % (YEAR,), len(row_dicts)
index = 0
matches = 0
output_rows = []
missed_addresses = []
row_dicts = [row for row in row_dicts if row["prix_vendu"] != ""] # remove rows with blank sales price
for row in row_dicts:
street_parameters = DP.get_street_parameters(row)
# skip inputs without a designated street address
if not street_parameters["street_number_lower"] and not street_parameters["street_number_upper"]:
continue
if not street_parameters["street_number_lower"]:
continue
if street_parameters["street_number_upper"] and not street_parameters["street_number_lower"]:
street_parameters["street_number_lower"] = street_parameters["street_number_upper"]
def test_load_data(self):
parser = DataParser('./watermelon_data_3a.txt', ',', True)
dataset = parser.load_data()
row = dataset[0]
assert (row[0] - 0.697) < 0.1
assert row[2] == 1
class RateMonitor:
def __init__(self):
# type: () -> None
self.plotter = PlotMaker()
self.fitter = FitFinder()
self.data_parser = DataParser()
self.use_fills = False
self.make_fits = False
self.use_fit_file = False # Currently unused outside of setupCheck
self.update_online_fits = False
self.plotter.compare_fits = False # Compare fits to multiple groups of runs
self.use_pileup = True # plot <PU> vs. rate
self.use_lumi = False # plot iLumi vs. rate
self.use_LS = False # plot LS vs. rate
self.use_stream_bandwidth = False
self.use_stream_size = False
self.use_grouping = False # Creates sub directories to group the outputs, utilizes self.group_map
# TESTING: START #
self.certify_mode = False
# TESTING: END #
self.group_map = {} # {'group_name': [trigger_name] }
self.fill_list = [] # Fills to get data from, Currently Unused
self.run_list = [] # Runs to get data from
self.object_list = [
] # List of *ALL* objects to plot, except for when using grouping
self.ops = None # The options specified at the command line
self.rate_mon_dir = os.getcwd()
self.save_dir = os.path.join(os.getcwd(), "tmp_rate_plots")
self.certify_dir = os.path.join(os.getcwd(), "tmp_certify_dir")
self.online_fits_dir = os.path.join(os.getcwd(), "Fits")
self.fit_file = None # Currently Unused
# The main function of RateMonitor, handles all the stitching together of the other pieces of code
# NOTE1: This might be doing to many things --> break up into pieces?
def run(self):
# type: () -> None
if not self.setupCheck():
print "ERROR: Bad setup"
return
print "Using runs:", self.run_list
print "Using Prescaled rates:", self.data_parser.use_prescaled_rate
if self.update_online_fits:
# Ensures that DataParser gets all triggers
self.data_parser.hlt_triggers = []
self.data_parser.l1_triggers = []
### THIS IS WHERE WE GET ALL OF THE DATA ###
self.data_parser.parseRuns(self.run_list)
if self.data_parser.use_streams:
# We want to manually add the streams to the list of objects to plot
self.object_list += self.data_parser.getObjectList(
obj_type="stream")
if self.use_grouping:
self.group_map["Streams"] = self.data_parser.getObjectList(
obj_type="stream")
if self.data_parser.use_datasets:
# Same concept, but for datasets
self.object_list += self.data_parser.getObjectList(
obj_type="dataset")
if self.use_grouping:
self.group_map["Datasets"] = self.data_parser.getObjectList(
obj_type="dataset")
if self.data_parser.use_L1A_rate:
# Manually add L1A rates to the list of objects to plot
self.object_list += self.data_parser.getObjectList(obj_type="L1A")
if self.use_grouping:
self.group_map["L1A_Rates"] = self.data_parser.getObjectList(
obj_type="L1A")
bunch_map = self.data_parser.getBunchMap()
det_status = self.data_parser.getDetectorStatus()
phys_status = self.data_parser.getPhysStatus()
# Select the types of data we are going to plot
if self.use_pileup: # plot PU vs. rate
x_vals = self.data_parser.getPUData()
elif self.use_lumi: # plot iLumi vs. rate
x_vals = self.data_parser.getLumiData()
elif self.use_LS: # plot LS vs. rate
x_vals = self.data_parser.getLSData()
if self.use_stream_size:
y_vals = self.data_parser.getSizeData()
elif self.use_stream_bandwidth:
y_vals = self.data_parser.getBandwidthData()
else:
y_vals = self.data_parser.getRateData()
# Now we fill plot_data with *ALL* the objects we have data for
#plot_data = {} # {'object_name': { run_number: ( [x_vals], [y_vals], [det_status] , [phys_status] ) } }
#for name in self.data_parser.getNameList():
# if not plot_data.has_key(name):
# plot_data[name] = {}
# for run in sorted(self.data_parser.getRunsUsed()):
# if not x_vals[name].has_key(run):
# continue
# plot_data[name][run] = [x_vals[name][run],y_vals[name][run],det_status[name][run],phys_status[name][run]]
plot_data = self.getData(x_vals, y_vals, det_status, phys_status,
self.fitter.data_dict['user_input'])
# If no objects are specified, plot everything!
if len(self.object_list) == 0:
self.object_list = [x for x in self.data_parser.name_list]
self.setupDirectory()
### NOW SETUP PLOTMAKER ###
self.plotter.setPlottingData(plot_data)
self.plotter.bunch_map = bunch_map
normalization = 1
if self.data_parser.normalize_bunches:
max_key = max(bunch_map.iterkeys(),
key=(lambda key: bunch_map[key]))
normalization = bunch_map[max_key]
print "Fit Normalization: %d" % normalization
# Make a fit of each object to be plotted, and save it to a .pkl file
if self.make_fits:
fit_info = {
'run_groups': copy.deepcopy(self.fitter.data_dict),
'triggers': {}
}
for group, runs in self.fitter.data_dict.iteritems():
data = self.getData(x_vals, y_vals, det_status, phys_status,
runs)
data_fits = self.fitter.makeFits(data, self.object_list,
normalization, group)
fit_info['triggers'] = self.fitter.mergeFits(
fit_info['triggers'], data_fits)
print group, runs
self.plotter.setFits(fit_info)
self.fitter.saveFits(self.plotter.fit_info, "fit_file.pkl",
self.save_dir)
#self.fitter.saveFits(fit_info,"fit_file.pkl",self.save_dir)
#self.plotter.setFits(fit_info)
elif self.update_online_fits:
self.updateOnlineFits(plot_data, normalization)
return # This keeps us from having to worry about any additional output plots
elif self.certify_mode:
self.certifyRuns(plot_data)
return # Same as above
# We want fits and no fits were specified --> make some
# NOTE: This 'if' is true only when ZERO fits exist
if self.plotter.use_fit and len(self.plotter.fits.keys()) == 0:
#fits = self.fitter.makeFits(plot_data,plot_data.keys(),normalization)
#self.plotter.setFits(fits)
fit_info = {
'run_groups':
copy.deepcopy(self.fitter.data_dict),
'triggers':
self.fitter.makeFits(plot_data, plot_data.keys(),
normalization)
}
self.plotter.setFits(fit_info)
# This is after update_online_fits, so as to ensure the proper save dir is set
self.plotter.save_dir = self.save_dir
self.plotter.plot_dir = "png"
counter = 0
# Specifies how we want to organize the plots in the output directory
if self.use_grouping:
print "Making Plots..."
# Create plots for *EVERYTHING* we've queried for
objs_to_plot = set()
for obj in self.object_list:
objs_to_plot.add(obj)
plotted_objects = self.makePlots(list(objs_to_plot))
counter += len(plotted_objects)
# Create index.html files to display specific groups of plots
for grp in self.group_map:
grp_path = os.path.join(self.save_dir, grp)
grp_objs = set()
for obj in plotted_objects:
if obj in self.group_map[grp]:
grp_objs.add(obj)
self.printHtml(png_list=grp_objs,
save_dir=self.save_dir,
index_dir=grp_path,
png_dir="..")
#for grp in self.group_map:
# print "Plotting group: %s..." % grp
# grp_path = os.path.join(self.save_dir,grp)
# self.plotter.save_dir = grp_path
# objs_to_plot = set()
# for obj in self.object_list:
# # Get the the plot objs associated with this group
# if obj in self.group_map[grp]:
# objs_to_plot.add(obj)
# plotted_objects = self.makePlots(list(objs_to_plot))
# self.printHtml(plotted_objects,grp_path)
# counter += len(plotted_objects)
else:
plotted_objects = self.makePlots(self.object_list)
#self.printHtml(plotted_objects,self.plotter.save_dir)
self.printHtml(png_list=plotted_objects,
save_dir=self.save_dir,
index_dir=self.save_dir,
png_dir=".")
counter += len(plotted_objects)
print "Total plot count: %d" % counter
# Makes some basic checks to ensure that the specified options don't create conflicting problems
def setupCheck(self):
# type: () -> bool
## These two options are mutually exclusive
#if not (self.use_pileup ^ self.use_lumi): # ^ == XOR
# print "ERROR SETUP: Improper selection for self.use_pileup and self.use_lumi"
# return False
# We can't specify two different x_axis at the same time
if self.use_pileup and self.use_lumi:
print "ERROR SETUP: Improper selection for self.use_pileup and self.use_lumi"
return False
# If we specify a fit file, we don't want to make new fits
if self.use_fit_file and self.update_online_fits:
print "ERROR SETUP: Fit file specified while trying updating online fits"
return False
elif self.use_fit_file and self.make_fits:
print "ERROR SETUP: Fit file specified while trying to make fits"
return False
# Check to see if the online fits directory exists
if self.update_online_fits and not os.path.exists(
self.online_fits_dir):
print "ERROR SETUP: Could not find fit directory"
return False
# We need to make sure we have the map between the plot objects and directories
if self.use_grouping and len(self.group_map.keys()) == 0:
print "ERROR SETUP: Grouping selected, but no group map found"
return False
# We don't want to accidentally remove all the contents from the current working directory
if self.save_dir == self.rate_mon_dir:
print "ERROR SETUP: Save directory is the same as RateMon directory"
return False
# Certify mode doesn't create any fits, so we shouldn't be updating any existing fits
if self.update_online_fits + self.certify_mode > 1:
print "ERROR SETUP: Can't update online fits and user certify mode at the same time"
return False
# In certify_mode we need to specify pre-made fits to use
if self.certify_mode and len(self.plotter.fits.keys()) == 0:
print "ERROR SETUP: No fits were found while in certify mode"
return False
## We are configured to only create/display the default fit, so only generate one fit
#if self.make_fits and not self.fitter.use_best_fit and not self.plotter.use_multi_fit:
# print "WARNING: Only creating the default fit, %s" % self.plotter.default_fit
# self.fitter.fits_to_try = [self.plotter.default_fit]
return True
# Sets up the save directories, will setup the directories based on CLI options
def setupDirectory(self):
# type: () -> None
print "Setting up directories..."
if self.update_online_fits:
mon_trg_dir = os.path.join(
self.online_fits_dir,
"Monitor_Triggers") # $rate_mon_dir/Fits/Monitor_Triggers
all_trg_dir = os.path.join(
self.online_fits_dir,
"All_Triggers") # $rate_mon_dir/Fits/All_Triggers
if os.path.exists(mon_trg_dir):
shutil.rmtree(mon_trg_dir)
print "\tRemoving existing directory: %s " % (mon_trg_dir)
if os.path.exists(all_trg_dir):
shutil.rmtree(all_trg_dir)
print "\tRemoving existing directory: %s " % (all_trg_dir)
print "\tCreating directory: %s " % (mon_trg_dir)
os.mkdir(mon_trg_dir)
os.chdir(mon_trg_dir)
os.mkdir("plots")
os.chdir(self.rate_mon_dir)
print "\tCreating directory: %s " % (all_trg_dir)
os.mkdir(all_trg_dir)
os.chdir(all_trg_dir)
os.mkdir("plots")
os.chdir(self.rate_mon_dir)
return
elif self.certify_mode:
# Ex: Certification_1runs_2016-11-02_13_27
dir_str = "Certification_%druns_%s" % (
len(self.run_list),
datetime.datetime.now().strftime("%Y-%m-%d_%H_%M"))
#dir_str = "Certification_%druns" % (len(self.run_list))
self.certify_dir = os.path.join(self.rate_mon_dir, dir_str)
if os.path.exists(self.certify_dir):
shutil.rmtree(self.certify_dir)
print "\tRemoving existing directory: %s " % (self.certify_dir)
print "\tCreating directory: %s " % (self.certify_dir)
os.mkdir(self.certify_dir)
os.chdir(self.certify_dir)
for run in self.run_list:
run_str = "run%d" % run
run_dir = os.path.join(self.certify_dir, run_str)
print "\tCreating directory: %s " % (run_dir)
os.mkdir(run_dir)
os.chdir(run_dir)
os.mkdir("png")
os.chdir(self.certify_dir)
return
else:
if os.path.exists(self.save_dir):
shutil.rmtree(self.save_dir)
print "\tRemoving existing directory: %s " % (self.save_dir)
os.mkdir(self.save_dir)
os.chdir(self.save_dir)
print "\tCreating directory: %s " % (self.save_dir)
os.mkdir("png")
if self.use_grouping:
for grp_dir in self.group_map.keys():
os.mkdir(grp_dir)
print "\tCreating directory: %s " % (os.path.join(
self.save_dir, grp_dir))
os.chdir("../")
return
# Stiching function that interfaces with the plotter object
def makePlots(self, plot_list):
# type: (List[str]) -> List[str]
if not self.use_grouping:
print "Making plots..."
plotted_objects = []
counter = 1
prog_counter = 0
for _object in sorted(plot_list):
if prog_counter % max(1, math.floor(len(plot_list) / 10.)) == 0:
print "\tProgress: %.0f%% (%d/%d)" % (
100. * prog_counter / len(plot_list), prog_counter,
len(plot_list))
prog_counter += 1
if not self.plotter.plotting_data.has_key(_object):
# No valid data points could be found for _object in any of the runs
print "\tWARNING: Unknown object - %s" % _object
continue
self.formatLabels(_object)
if self.plotter.plotAllData(_object):
plotted_objects.append(_object)
counter += 1
return plotted_objects
# Formats the plot labels based on the type of object being plotted
# TODO: Might want to move this (along with makePlots() into PlotMaker.py),
# would require specifying a type_map within the plotter object
def formatLabels(self, _object):
# type: (str) -> None
x_axis_label = ""
y_axis_label = ""
x_axis_var = ""
y_axis_var = ""
x_units = ""
y_units = "[Hz]"
if self.use_pileup: # plot PU vs. rate
x_axis_var = "< PU >"
elif self.use_lumi: # plot iLumi vs. rate
x_axis_var = "instLumi"
else: # plot LS vs. rate
x_axis_var = "lumisection"
if self.data_parser.type_map[_object] == "trigger":
if self.data_parser.correct_for_DT == True:
y_axis_var = "pre-deadtime "
if self.data_parser.use_prescaled_rate:
y_axis_var += "prescaled rate"
else:
y_axis_var += "unprescaled rate"
elif self.data_parser.type_map[_object] == "stream":
if self.use_stream_size or self.use_stream_bandwidth:
y_units = "[bytes]"
y_axis_var = "stream rate"
elif self.data_parser.type_map[_object] == "dataset":
y_axis_var = "dataset rate"
elif self.data_parser.type_map[_object] == "L1A":
y_axis_var = "L1A rate"
x_axis_label += x_axis_var
y_axis_label += y_axis_var
# Format the y_axis_label denominator
if self.data_parser.normalize_bunches and self.data_parser.use_cross_section:
y_axis_label += " / (num bx*iLumi)"
elif self.data_parser.normalize_bunches:
y_axis_label += " / num colliding bx"
elif self.data_parser.use_cross_section:
y_axis_label += " / iLumi"
y_axis_label += " " + y_units
self.plotter.var_X = x_axis_var
self.plotter.var_Y = y_axis_var
self.plotter.label_X = x_axis_label
self.plotter.label_Y = y_axis_label
self.plotter.units_X = x_units
self.plotter.units_Y = y_units
def updateOnlineFits(self, plot_data, normalization):
# type: (Dict[str,Dict[int,object]]) -> None
# NOTE: self.object_list, contains *ONLY* the list of triggers from 'monitorlist_COLLISIONS.list'
mon_trg_dir = os.path.join(self.online_fits_dir, "Monitor_Triggers")
all_trg_dir = os.path.join(self.online_fits_dir, "All_Triggers")
all_triggers = set()
for obj in self.data_parser.getNameList():
all_triggers.add(obj)
all_triggers = list(all_triggers)
self.plotter.plot_dir = "plots"
# Plots the monitored paths
print "Updating monitored trigger fits..."
print "Total Triggers: %d" % (len(self.object_list))
self.plotter.save_dir = mon_trg_dir
#fits = self.fitter.makeFits(plot_data,self.object_list,normalization)
#self.plotter.setFits(fits)
#self.fitter.saveFits(fits,"FOG.pkl",mon_trg_dir)
#fit_info = self.fitter.makeFits(plot_data,self.object_list,normalization)
fit_info = {
'run_groups':
copy.deepcopy(self.fitter.data_dict),
'triggers':
self.fitter.makeFits(plot_data, plot_data.keys(), normalization)
}
self.plotter.setFits(fit_info)
self.fitter.saveFits(self.plotter.fit_info, "FOG.pkl", mon_trg_dir)
plotted_objects = self.makePlots(self.object_list)
# Plots all trigger paths
print "Updating all trigger fits..."
print "Total Triggers: %d" % (len(all_triggers))
self.plotter.save_dir = all_trg_dir
#fits = self.fitter.makeFits(plot_data,all_triggers,normalization)
#self.plotter.setFits(fits)
#self.fitter.saveFits(fits,"FOG.pkl",all_trg_dir)
#fit_info = self.fitter.makeFits(plot_data,all_triggers,normalization)
fit_info = {
'run_groups':
copy.deepcopy(self.fitter.data_dict),
'triggers':
self.fitter.makeFits(plot_data, plot_data.keys(), normalization)
}
self.plotter.setFits(fit_info)
self.fitter.saveFits(selfl.plotter.fit_info, "FOG.pkl", all_trg_dir)
plotted_objects = self.makePlots(all_triggers)
command_line_str = "Results produced with:\n"
command_line_str += "python plotTriggerRates.py "
for tup in self.ops:
#if tup[0].find('--updateOnlineFits') > -1:
# # never record when we update online fits
# continue
#elif tup[0].find('--lsVeto') > -1:
# continue
if len(tup[1]) == 0:
command_line_str += "%s " % (tup[0])
else:
command_line_str += "%s=%s " % (tup[0], tup[1])
for run in self.run_list:
command_line_str += "%d " % (run)
command_line_str += "\n"
command_line_file_name = os.path.join(mon_trg_dir, "command_line.txt")
log_file_mon = open(command_line_file_name, "w")
log_file_mon.write(command_line_str)
log_file_mon.close()
command_line_file_name = os.path.join(all_trg_dir, "command_line.txt")
log_file_all = open(command_line_file_name, "w")
log_file_all.write(command_line_str)
log_file_all.close()
def certifyRuns(self, plot_data):
# type: (Dict[str,Dict[int,object]]) -> None
#self.plotter.save_dir = self.certify_dir
#self.plotter.root_file_name = "CertificationSumaries.root"
# {'name': {run: [ (LS,pred,err) ] } }
lumi_info = self.data_parser.getLumiInfo()
sorted_run_list = sorted(self.run_list)
log_file_name = "CertificationSummary_run" + str(
sorted_run_list[0]) + "_run" + str(sorted_run_list[-1]) + ".txt"
log_file = open(self.certify_dir + "/" + log_file_name, 'w')
for run in self.run_list:
log_file.write("Run Number: %s\n" % (run))
self.plotter.save_dir = self.certify_dir
self.plotter.root_file_name = "CertificationSummaries.root"
#pred_data = self.getPredictionData(run) # {'trg name': { 'group name': [ (LS,pred,err) ] } }
pred_data = self.getPredictionData(
run
) # {'trg name': { 'group name': { 'fit_type': [ (LS,pred,err) ] } } }
## Check if there are multiple fit types to plot
#multi_fit_types = False
#for trg in pred_data:
# for grp in pred_data[trg]:
# if len(pred_data[trg][grp].keys()) > 1:
# multi_fit_types = True
for group in self.plotter.run_groups:
## We have multiple fit types per trg: separate histograms and summary text file by fit type
#if self.plotter.use_multi_fit and multi_fit_types:
# for fit_type in self.fitter.fits_to_try:
# log_file.write("\n")
# log_file.write("Group: %s\n" % (group))
# log_file.write("Fit type: %s\n" % (fit_type))
# self.plotter.makeCertifySummary(run,pred_data,log_file,group,multi_fit_types,fit_type)
# We have only one fit type per trg: do not separate histograms and summary text file by fit type
log_file.write("\n")
log_file.write("Group: %s\n" % (group))
self.plotter.makeCertifySummary(run, pred_data, log_file,
group)
print "Making certification plots for run %d..." % run
run_dir = os.path.join(self.certify_dir, "run%d" % run)
self.plotter.save_dir = run_dir
self.plotter.plot_dir = "png"
self.plotter.root_file_name = "HLT_LS_vs_rawRate_Fitted_Run%d_CERTIFICATION.root" % run
plotted_objects = []
for obj in self.object_list:
if not obj in self.data_parser.name_list:
print "Skipping missing trigger: %s" % (obj)
continue
self.formatLabels(obj)
if self.plotter.makeCertifyPlot(obj, run, lumi_info[run]):
print "Plotting %s..." % obj
plotted_objects.append(obj)
self.printHtml(png_list=plotted_objects,
save_dir=run_dir,
index_dir=self.save_dir,
png_dir=".")
# We create a prediction dictionary on a per run basis, which covers all triggers in that run
# TODO: Should move this to DataParser.py
def getPredictionData(self, run):
# UNFINISHED
# We need to disable converting the output
prev_state = self.data_parser.convert_output
self.data_parser.convert_output = False
#lumi_info = self.data_parser.parser.getLumiInfo(runNumber=run) # {run_number: [ (LS,ilum,psi,phys,cms_ready) ] }
lumi_info = self.data_parser.getLumiInfo(
) # {run_number: [ (LS,ilum,psi,phys,cms_ready) ] }
ls_data = self.data_parser.getLSData(
) # {'name': { run_number: [ LS ] } }
pu_data = self.data_parser.getPUData(
) # {'name': { run_number: { LS: PU } } }
bunch_map = self.data_parser.getBunchMap() # {run_number: bunches}
plotter_sigmas = self.plotter.sigmas
#pred_dict = {} # {'name': [ (LS,pred,err) ] }
#pred_dict = {} # {'trg name': {'group name': [ (LS,pred,err) ] } }
pred_dict = {
} # {'trg name': {'group name': { 'fit_type': [ (LS,pred,err) ] } } }
for obj in self.plotter.fits:
if not pu_data.has_key(obj):
continue
elif not pu_data[obj].has_key(run):
continue
pred_dict[obj] = {}
for group in self.plotter.fits[obj]:
pred_dict[obj][group] = {}
# Find the best fit
best_fit_type, best_fit = self.fitter.getBestFit(
self.plotter.fits[obj][group])
lsVals = []
puVals = []
for LS, ilum, psi, phys, cms_ready in lumi_info[run]:
if not ilum is None and phys:
if not pu_data[obj][run].has_key(LS):
continue
lsVals.append(LS)
puVals.append(pu_data[obj][run][LS])
lumisecs, predictions, ls_error, pred_error = self.fitter.getPredictionPoints(
best_fit, lsVals, puVals, bunch_map[run], 0)
pred_dict[obj][group][best_fit_type] = zip(
lumisecs, predictions, pred_error)
##############################################################################################
# --- 13 TeV constant values ---
#ppInelXsec = 80000.
#orbitsPerSec = 11246.
## Initialize our point arrays
#lumisecs = array.array('f')
#predictions = array.array('f')
#ls_error = array.array('f')
#pred_error = array.array('f')
## Unpack values
#fit_type, X0, X1, X2, X3, sigma, meanraw, X0err, X1err, X2err, X3err, ChiSqr = best_fit
## Create our point arrays
#for LS, ilum, psi, phys, cms_ready in lumi_info[run]:
# if not ilum is None and phys:
# if not pu_data[obj][run].has_key(LS):
# continue
# lumisecs.append(LS)
# #pu = (ilum * ppInelXsec) / ( self.bunch_map[run] * orbitsPerSec )
# pu = pu_data[obj][run][LS]
# # Either we have an exponential fit, or a polynomial fit
# if fit_type == "exp":
# rr = bunch_map[run] * (X0 + X1*math.exp(X2+X3*pu))
# elif fit_type == "sinh":
# #val = 0
# #val += math.pow(X0*pu,11)/39916800.
# #val += math.pow(X0*pu,9)/362880.
# #val += math.pow(X0*pu,7)/5040.
# #val += math.pow(X0*pu,5)/120.
# #val += math.pow(X0*pu,3)/6.
# #val += math.pow(X0*pu,1)
# #val = X1*val + X2
# #rr = bunch_map[run] * (val)
# rr = bunch_map[run] * (X1*math.sinh(X0*pu) + X2) # ???
# else:
# rr = bunch_map[run] * (X0 + pu*X1 + (pu**2)*X2 + (pu**3)*X3)
# if rr < 0: rr = 0 # Make sure prediction is non negative
# predictions.append(rr)
# ls_error.append(0)
# pred_error.append(bunch_map[run]*plotter_sigmas*sigma)
##############################################################################################
# Revert back to the previous convert_output setting
self.data_parser.convert_output = prev_state
return pred_dict
## NOTE1: This requires the .png file to be in the proper directory, as specified by self.group_map
## NOTE2: This function assumes that the sub-directory where the plots are located is named 'png'
##def printHtml(self,png_list,save_dir):
## # type: (List[str],str) -> None
## try:
## htmlFile = open(save_dir+"/index.html", "wb")
## htmlFile.write("<!DOCTYPE html>\n")
## htmlFile.write("<html>\n")
## htmlFile.write("<style>.image { float:right; margin: 5px; clear:justify; font-size: 6px; font-family: Verdana, Arial, sans-serif; text-align: center;}</style>\n")
## for path_name in sorted(png_list): # This controls the order that the images will be displayed in
## file_name = "%s/png/%s.png" % (save_dir,path_name)
## if os.access(file_name,os.F_OK):
## htmlFile.write("<div class=image><a href=\'png/%s.png\' target='_blank'><img width=398 height=229 border=0 src=\'png/%s.png\'></a><div style=\'width:398px\'>%s</div></div>\n" % (path_name,path_name,path_name))
## htmlFile.write("</html>\n")
## htmlFile.close
## except:
## print "Unable to write index.html file"
# For this we want to be able to specify where the images are located, relative to the index.html file
def printHtml(self, png_list, save_dir, index_dir, png_dir="."):
# save_dir: The full path to the save directory
# index_dir: The full path to the index.html file
# png_dir: The relative path from the index.html file to the png_dir
try:
htmlFile = open(index_dir + "/index.html", "wb")
htmlFile.write("<!DOCTYPE html>\n")
htmlFile.write("<html>\n")
htmlFile.write(
"<style>.image { float:left; margin: 5px; clear:justify; font-size: 6px; font-family: Verdana, Arial, sans-serif; text-align: center;}</style>\n"
)
for path_name in sorted(
png_list
): # This controls the order that the images will be displayed in
file_name = "%s/png/%s.png" % (save_dir, path_name)
if os.access(file_name, os.F_OK):
rel_dir = os.path.join(png_dir, "png/%s.png" % path_name)
html_str = ""
html_str += "<div class=image>"
html_str += "<a href=\'%s\' target='_blank'>" % rel_dir
html_str += "<img width=398 height=229 border=0 src=\'%s\'>" % rel_dir
html_str += "</a><div style=\'width:398px\'>%s</div></div>\n" % path_name
htmlFile.write(html_str)
htmlFile.write("</html>\n")
except:
print "Unable to write index.html file"
# Returns {'object_name': { run_number: ( [x_vals], [y_vals], [det_status] , [phys_status] ) } }
def getData(self, x_vals, y_vals, det_status, phys_status, runs=[]):
data = {}
for name in self.data_parser.getNameList():
if not data.has_key(name):
data[name] = {}
for run in sorted(self.data_parser.getRunsUsed()):
if not x_vals[name].has_key(run):
continue
if len(runs) > 0 and run not in runs:
continue
data[name][run] = [
x_vals[name][run], y_vals[name][run],
det_status[name][run], phys_status[name][run]
]
return data
import pygame.camera
import time
from ImageGetter import *
from AzureRequest import *
from DataParser import *
from DataAnalyzer import *
stop = False
pygame.camera.init()
cam = pygame.camera.Camera(pygame.camera.list_cameras()[0])
cam.start()
ar = AzureRequest()
data_parser = DataParser()
while not stop:
img_getter = ImageGetter()
capture = img_getter.get_image(cam)
data_parser.set_data(ar.request('{url: \'http://newsrescue.com/wp-content/uploads/2015/04/happy-person.jpg\'}'))
print data_parser.parse()
time.sleep(3)
pygame.camera.quit()
def run(self, input_socket):
f_in = input_socket.makefile()
# amount bet after flop / 200 - percentage of hand/board
# cut off for accepting my bets: my bets/200 - percentage of hand/board
# preflop bet /200 - percentage of starting hand
# global variables
d = DataParser()
minOppPercent = 100 # minimum percept opp goes all in on
discardRound = 0 # 0 if preflop, 1 if preturn, 2 if preriver
quitWhileAheadMode = False
winMode = False
onEdgeMode = False
quitRank = 60 # max rank to call in quitwhileaheadmode
myStack = 200
myBank = 0
handID = 1
numHands = 1000
bc = BetCalc()
while True:
data = f_in.readline().strip()
if not data:
print "Gameover, engine disconnected."
break
print data
# QUIT WHILE YOU'RE AHEAD FUNCTION ! **************
quitWhileAheadMode = myBank > myStack
#winMode = myBank > 1.5*(numHands-handID)
winMode = myBank > 10 * (numHands - handID)
onEdgeMode = myBank < handID - numHands
#print quitWhileAheadMode
if winMode:
s.send("CHECK\n")
continue
d.parse(data)
word = d.word
if word == "NEWGAME":
myStack = d.stackSize
numHands = d.numHands
elif word == "NEWHAND":
myBank = d.myBank
startingHandRank = d.startingHandRank
handID = d.handID
totalOppBet = 0
totalMyBets = 0
elif word == "GETACTION":
actionType = d.actionType
handRank = d.handRank
# opponent checked
if actionType == "CHECK BET/RAISE":
if len(d.board) == 0: # range for preflop: 78 to 26
bettingNums = [
100, # preflop, go all in
100, # preflop, large bet/raise
100, # preflop, medium bet/raise
52
] # preflop, small bet/raise
else:
bettingNums = [
101, # postflop, go all in
101, # postflop, large bet/raise
80, # postflop, medium bet/raise
70
] # postflop, small bet/raise
if (quitWhileAheadMode
or onEdgeMode) and handRank < quitRank and len(
d.board) == 0:
bet = 0
s.send("FOLD\n")
elif len(
d.board
) == 5 and totalOppBet <= 10 and totalMyBets <= 30:
bet = d.maxBet
s.send(d.betOrRaise + ":" + str(bet) + "\n")
print "all in bb"
elif len(
d.board
) >= 3 and d.lastOppAction == "CHECK" and handRank < 50.0:
bet = d.maxBet
s.send(d.betOrRaise + ":" + str(bet) + "\n")
print "all in bb"
elif handRank >= bettingNums[0]: # all in
bet = d.maxBet
s.send(d.betOrRaise + ":" + str(bet) + "\n")
print "all in bb"
elif handRank >= bettingNums[1]:
bet = bc.getBetAmount("LARGE", d.maxBet, d.minBet)
s.send(d.betOrRaise + ":" + str(bet) + "\n")
elif handRank >= bettingNums[2]:
bet = bc.getBetAmount("MED", d.maxBet, d.minBet)
s.send(d.betOrRaise + ":" + str(bet) + "\n")
elif handRank >= bettingNums[3]:
bet = bc.getBetAmount("SMALL", d.maxBet, d.minBet)
s.send(d.betOrRaise + ":" + str(bet) + "\n")
else:
bet = d.minBet
s.send(d.betOrRaise + ":" + str(bet) + "\n")
totalMyBets += bet
# discard round
elif actionType == "CHECK DISCARD DISCARD":
s.send(d.shouldDiscard)
# opponent bet and it wasnt all-in
elif actionType == "FOLD CALL RAISE" or actionType == "FOLD CALL":
totalOppBet += d.oppBet
if len(d.board) == 0:
bettingNums = [
100, # preflop, rank to raise on a bet
52, # preflop, rank to call a bet
53,
52
]
else:
bettingNums = [
93, # postflop, rank to raise on a bet
90, # postflop, rank to call a bet
80,
70
]
# quit while youre ahead!
if (quitWhileAheadMode
or onEdgeMode) and handRank < quitRank and len(
d.board) == 0:
s.send("FOLD\n")
elif d.oppBet >= 150 and handRank < 99:
print d.oppBet, "oppbet"
s.send("FOLD\n")
# raise!
elif handRank >= bettingNums[
0] and actionType != "FOLD CALL":
raiseAmount = d.maxRaise / 2.0 + d.minRaise / 2.0 # good amount?
s.send("RAISE:" + str(raiseAmount) + "\n")
# call no matter what
elif handRank >= bettingNums[1]:
print "call 1"
s.send("CALL\n")
print "handRank threshold call"
elif d.oppBet <= myStack / 4.0 and len(d.board) == 0:
print "call 2"
s.send("CALL\n")
# call big bet
elif len(d.board) == 0:
s.send("FOLD\n")
elif handRank >= bettingNums[
2] and d.oppBet <= myStack / 2.0:
print "call 3"
s.send("CALL\n")
elif handRank >= bettingNums[
3] and d.oppBet <= myStack / 4.0:
print "call 4"
s.send("CALL\n")
else:
# FOLD MORE OFTEN TO DEFEAT FORD
# FOLD LESS OFTEN TO DEFEAT BLUFFBOT
# could be 3, could be 2
if d.oppBet <= d.handRank / 2:
print "betsize 1 call", d.oppBet
s.send("CALL\n")
elif d.potSize - d.oppBet >= d.oppBet:
print "betsize 2 call"
s.send("CALL\n")
else:
s.send("FOLD\n")
# shitty hand (30% ish)
# raise on pre flop
# what to do?
# -> usually a good hand
#pokerbots
# protect card for hopeful straight (50% chance) in discard method
# also, dont raise a raise unless hand is REAL good >97
# no big bets on third round tho
elif word == "HANDOVER":
pass
# if d.cardsShown and d.winner == d.oppName:
# differnt amounts of overall bets
# near / all-in
# if d.winnersPot >= 2.0 * d.stackSize - 5.0:
# oppPercent = d.pw.getWinPercentage()
# minOppPercent = min(oppPercent,minOppPercent)
elif word == "REQUESTKEYVALUES":
s.send("FINISH\n")
s.close()
def runDemo():
print("\n")
animatedprint("Hi, welcome to our short demo.")
printsleep(1.0)
animatedprint("Here we'll be doing a recompile and quick runthrough of all of our code.")
animatedprint("---------------------------------------------------------------------------\n")
printsleep(3.0)
mapfile = "map2.txt"
qt_threshold = 5
actual_to_max_children_ratio = 0.75
max_num_adjacent = 30
os.chdir(os.path.dirname(os.path.abspath(__file__)))
os.chdir("QTData")
animatedprint("We'll start by compiling CreateQT.exe...")
os.system("gcc -std=c99 -o CreateQT CreateQT.c")
print("Done!\n")
printsleep(2.0)
animatedprint("Now we create the quadtree of %s with parameters"%mapfile)
print("\tqt_threshold = %d"%qt_threshold)
printsleep(0.1)
print("\tactual_to_max_children_ratio = %f"%actual_to_max_children_ratio)
printsleep(0.1)
print("\tmax_num_adjacent = %d"%max_num_adjacent)
printsleep(2.0)
os.system("CreateQT.exe {0} {1} {2} {3}".format(mapfile,qt_threshold,actual_to_max_children_ratio,max_num_adjacent))
print("Done!\n")
printsleep(2.0)
os.chdir("..")
animatedprint("Now we'll find the shortest path between all nodes...")
printsleep(2.0)
import QTPathFinder as qt
pack,pdict = qt.findAllPaths()
mngr = plt.get_current_fig_manager()
geom = mngr.window.geometry()
x, y, dx, dy = geom.getRect()
mngr.window.setGeometry(100, 100, dx, dy)
plt.show()
plt.pause(0.5)
print("Done!\n")
printsleep(1.0)
animatedprint("Now let's find some paths. Bring up the figure if you don't already see it.")
import DataParser as dp
import HelperMethods as hm
#To get legal points
limits, rectVects = dp.parseQT(justTheRectangles=True)
xrng = limits[1]-limits[0]
yrng = limits[3]-limits[2]
numPaths = 150
startPoints = [(xrng*(random.random()),yrng*(random.random())) for i in range(numPaths)]
endPoints = [(xrng*(random.random()),yrng*(random.random())) for i in range(numPaths)]
removal = []
for p in startPoints:
if hm.isInsideRect(p,rectVects):
removal.append(p)
pass
pass
for r in removal:
startPoints.remove(r)
pass
removal = []
for p in endPoints:
if hm.isInsideRect(p,rectVects):
removal.append(p)
pass
pass
for r in removal:
endPoints.remove(r)
pass
printsleep(3.0)
animatedprint("Begin plotting...")
printsleep(1.0)
demoPoints = dict(zip(startPoints,endPoints))
line = lines.Line2D([], [], lw=2, c='red')
ax = pack[2]
ax.add_line(line)
for s,e in demoPoints.items():
line = qt.plotPathWithResults(pack,pdict,s,e,line)
print("Start: ({:f},{:f}) | End: ({:f},{:f})".format(s[0],s[1],e[0],e[1]))
sys.stdout.flush()
plt.show()
plt.pause(0.001)
pass
print("...Done!")
printsleep(1.0)
animatedprint("\nThis ends the demo. Thanks for watching!")
plt.pause(5.0)
def main():
"""
Add Event Demo
"""
credentials = get_credentials()
http = credentials.authorize(httplib2.Http())
service = discovery.build('calendar', 'v3', http=http)
now = datetime.datetime.utcnow().isoformat(
) + 'Z' # 'Z' indicates UTC time
# Refer to the Python quickstart on how to setup the environment:
# https://developers.google.com/google-apps/calendar/quickstart/python
# Change the scope to 'https://www.googleapis.com/auth/calendar' and delete any
# stored credentials.
event = {
'summary': '這是summary',
'location': '某個地點',
'description': '敘述文字~\n敘述',
'start': {
'dateTime': '2017-11-06T09:00:00+08:00',
'timeZone': 'Asia/Taipei',
},
'end': {
'dateTime': '2017-11-06T11:00:00+08:00',
'timeZone': 'Asia/Taipei',
},
'recurrence': [],
'attendees': [],
'reminders': {
'useDefault': False,
'overrides': [],
},
}
# get calendarId
page_token = None
while True:
calendar_list = service.calendarList().list(
pageToken=page_token).execute()
print(calendar_list)
for calendar_list_entry in calendar_list['items']:
print(calendar_list_entry['summary'])
page_token = calendar_list.get('nextPageToken')
if not page_token:
break
#event = service.events().insert(calendarId='*****@*****.**', body=event).execute()
#print(event)
#print("Event created:{} ".format(event.get('htmlLink')))
import DataParser
for item in DataParser.getEvents():
event = {
'summary':
item['country'] + "-" + item['title'],
'description':
'前值：{}\n預測：{}\n'.format(item['history'], item['prediction']),
'start': {
'dateTime': item['datetime'],
'timeZone': 'Asia/Taipei',
},
'end': {
'dateTime': item['datetime'],
'timeZone': 'Asia/Taipei',
},
}
import DataParser
import DataUtils
import NetworkAccess
import GetUrl
if __name__ == "__main__":
CreateTable = True
for i in range(0, 249, 25):
url = GetUrl.Get_Data_Url(i)
html_data = NetworkAccess.GetDataUrl(url)
infoes = DataParser.Get_page_content(url, 0, html_data)
if infoes == 0:
pass
else:
item_list = DataUtils.ItemUtils()
for i in range(len(infoes)):
item_list.append(infoes[i]["name"], infoes[i]["content"], infoes[i]["score"], infoes[i]["say"])
pass
# item_list.clear()
pass
pass
item_list.save_excel(sheet="шби", file="list.xlsx")
item_list.save_database(CreateTable)
pass
import pygame.camera
import time
from ImageGetter import *
from AzureRequest import *
from DataParser import *
from DataAnalyzer import *
stop = False
pygame.camera.init()
cam = pygame.camera.Camera(pygame.camera.list_cameras()[0])
cam.start()
ar = AzureRequest()
data_parser = DataParser()
while not stop:
img_getter = ImageGetter()
capture = img_getter.get_image(cam)
data_parser.set_data(
ar.request(
'{url: \'http://newsrescue.com/wp-content/uploads/2015/04/happy-person.jpg\'}'
))
print data_parser.parse()
time.sleep(3)
pygame.camera.quit()
import DataParser as dps
import DataPloter as dpl
sFile = 'RawData.csv'
tFile = 'Preprocess.csv'
tNo = [1]
xMax = 386
yMax = 30000
sRate = 1000 # sampling rate
period = 0.2
#------------------------------------------------------------------------------------------------
if __name__ == '__main__':
s = dps.Parser() # new objects
s.parseRawFile(sFile, tFile, tNo)
l = dpl.Ploter()
l.showSignal(tFile, xMax, yMax, sRate)
import DataParser
import time
import random
import csv
from collections import Counter
import pathlib
url = 'https://your-url.com/parameters'
Data = DataParser.dataparser(url)
#Get relevant sales data from the html pasrsing
Title = Data[0]
Price = Data[1]
Size = Data[2]
Brand = Data[3]
Words = Data[4]
#write sales data to file
with open('FileName.csv', mode='a') as FileName:
csv_writer = csv.writer(FileName,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow(Title)
csv_writer.writerow(Price)
csv_writer.writerow(Size)
csv_writer.writerow(Brand)
#check to see if the keyword file exists
def __init__(self):
input_lines = DataParser.parse_csv_file()
self.__products_storage = ProductsStorage(input_lines)
import DataParser
import plotly
from plotly.graph_objs import Bar, Layout
player_data = DataParser.format_data_list(printing=False)
ratings = [float(player_data[x][-1]) for x in range(0, len(player_data))]
quarterbacks = [player_data[x][1] for x in range(0, len(player_data))]
plotly.offline.plot({
"data": [Bar(x=quarterbacks, y=ratings)],
"layout": Layout(title="Passer Ratings")
})
def logic():
# initialization
db = DatabaseHandler()
db.fill_database(DataParser.fetch_data())
return
INPUT_SIZE = 1
OUTPUT_SIZE = 1
TRAIN_BATCH_SIZE = 10
CELL_NUM = 64
CELL_LAYER_NUM = None
NN_TYPE = 'LSTM'
LR = 0.01
IS_LR_DECAY = False
TRAIN_EPOCH = 1000
GLOBAL_EPOCH = 0
PRED_TIME_RANGE = 325
#-----data import----#
raw_R_peak = dp.read_data_from_txt(DATA_PATH + '103Rpeak.txt')
print(raw_R_peak.shape)
# plt.plot(raw_R_peak)
# plt.show()
time_interval = get_interval(raw_R_peak)
# time_interval = (time_interval - np.mean(time_interval)) / np.std(time_interval)
# plt.plot(time_interval)
# plt.show()
print(time_interval.shape)
train_num = int(time_interval.shape[0] * TRAIN_PROB)
# train_num = 20
test_num = int(time_interval.shape[0] * (1 - TRAIN_PROB))
train_time_interval = time_interval[:train_num]
# standardization
train_time_interval_std = (train_time_interval - np.mean(train_time_interval)
) / np.std(train_time_interval)
def get_matrix(pred, gt):
TP = np.sum(np.logical_and(pred == 2, gt == 2))
TN = np.sum(np.logical_and(pred == 0, gt == 0))
FP = np.sum(np.logical_and(pred == 2, gt == 0))
FN = np.sum(np.logical_and(pred == 0, gt == 2))
confusion_matrix = np.matrix([[TN, FP], [FN, TP]])
return confusion_matrix
if __name__ == '__main__':
args = docopt(__doc__, version='1.0')
if args['tts']:
import MLModel as ml
parser1 = dp.DataParser(args['-d'], dp.Mode.TRAIN)
all_data = parser1.parseData()
X = []
Y = []
all_names = []
all_hits = []
all_samples = []
all_valid_gt = []
for data in all_data:
sample_name = data['name']
hits = data['hits']
sample = data['sample']
ground_truth = data['ground_truth']
valid_gt = get_valid_gt(ground_truth, sample)
all_names.append(sample_name)
all_hits.append(hits)
def get_matrix(pred, gt):
TP = np.sum(np.logical_and(pred == 2, gt == 2))
TN = np.sum(np.logical_and(pred == 0, gt == 0))
FP = np.sum(np.logical_and(pred == 2, gt == 0))
FN = np.sum(np.logical_and(pred == 0, gt == 2))
confusion_matrix = np.matrix([[TN, FP], [FN, TP]])
return confusion_matrix
if __name__ == '__main__':
args = docopt(__doc__, version='1.0')
#print(args)
if args['ml']:
import MLModel as ml
if args['train']:
parser = dp.DataParser(args['-d'], dp.Mode.TRAIN)
all_data = parser.parseData()
X = []
Y = []
for data in all_data:
sample_name = data['name']
hits = data['hits']
sample = data['sample']
ground_truth = data['ground_truth']
valid_gt = get_valid_gt(ground_truth, sample)
for (peak_num, name, confidence) in valid_gt:
if confidence == 2:
gt = [0, 1]
elif confidence == 0:
gt = [1, 0]
sample_spectrum = sample[peak_num - 1]['spectrum']
import sys
from sklearn.preprocessing import Imputer
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
dirname = os.path.dirname(__file__)
sys.path.append("../data_parser")
import DataParser
#import data
subjects = [
"Religion", "Sinhala", "English", "Mathematics", "Science", "History",
"Geography", "Citizenship Education", "Health", "Tamil", "Art", "PTS"
]
marks_all = DataParser.get_marks(subjects, index='no')
#handle missing values
marks_all = DataParser.handle_missing_values(marks_all, how='-1', is_nan=True)
marks_all = DataParser.handle_missing_values(marks_all, how='-1', is_nan=False)
marks_all = marks_all.replace({-1: np.nan})
marks_all = marks_all.fillna(method='bfill', axis=1)
marks_all = marks_all.fillna(method='ffill', axis=1)
marks_all = marks_all.dropna(axis=0)
"""
PCA
"""
col_all = marks_all.columns
principalDf = pd.DataFrame()
subs_ex_ratios = []
for sub_year in range(int(len(col_all) / 3)):
def run(self, input_socket):
f_in = input_socket.makefile()
# amount bet after flop / 200 - percentage of hand/board
# cut off for accepting my bets: my bets/200 - percentage of hand/board
# preflop bet /200 - percentage of starting hand
# global variables
d = DataParser()
minOppPercent = 100 # minimum percept opp goes all in on
discardRound = 0 # 0 if preflop, 1 if preturn, 2 if preriver
quitWhileAheadMode = False
winMode = False
onEdgeMode = False
quitRank = 52 # max rank to call in quitwhileaheadmode
myStack = 200
myBank = 0
handID = 1
numHands = 1000
bc = BetCalc()
while True:
data = f_in.readline().strip()
if not data:
print "Gameover, engine disconnected."
break
print data
d.parse(data)
word = d.word
if word == "NEWGAME":
myStack = d.stackSize
numHands = d.numHands
elif word == "NEWHAND":
myBank = d.myBank
startingHandRank = d.startingHandRank
handID = d.handID
elif word == "GETACTION":
actionType = d.actionType
handRank = d.handRank
# opponent checked
if actionType == "CHECK BET/RAISE": #starting round
s.send("RAISE:200\n")
# discard round
elif actionType == "CHECK DISCARD DISCARD":
s.send(d.shouldDiscard)
# opponent bet and it wasnt all-in
elif actionType == "FOLD CALL RAISE" or actionType == "FOLD CALL": #starting round
s.send("CALL\n")
# shitty hand (30% ish)
# raise on pre flop
# what to do?
# -> usually a good hand
#pokerbots
# protect card for hopeful straight (50% chance) in discard method
# also, dont raise a raise unless hand is REAL good >97
# no big bets on third round tho
elif word == "HANDOVER":
pass
# if d.cardsShown and d.winner == d.oppName:
# differnt amounts of overall bets
# near / all-in
# if d.winnersPot >= 2.0 * d.stackSize - 5.0:
# oppPercent = d.pw.getWinPercentage()
# minOppPercent = min(oppPercent,minOppPercent)
elif word == "REQUESTKEYVALUES":
s.send("FINISH\n")
s.close()
