def main(options):
global program
program = propyte.Program(options=options,
name=name,
version=version,
logo=logo)
global log
from propyte import log
expression = options["--expression"]
word_vector_model = options["--wordvectormodel"]
model_word2vec = abstraction.load_word_vector_model(
filename=word_vector_model)
sentences = [
"What are you dirty hooers doing on my planet?", "What time is it?",
"What can you do?", "Change the color from red to black.",
"All those moments will be lost in time.",
"All of those moments will be lost in time.",
"All of those moments are to be lost in time."
]
result = most_similar_expression(expression=expression,
expressions=sentences,
model_word2vec=model_word2vec)
pyprel.print_line()
log.info(
"input expression: {expression}".format(expression=expression))
log.info("most similar expression: {expression}".format(expression=result))
pyprel.print_line()
program.terminate()
def main(options):
global program
program = propyte.Program(
options = options,
name = name,
version = version,
logo = logo
)
global log
from propyte import log
# access options and arguments
subreddits = options["--subreddits"].split(",")
number_of_utterances = options["--numberOfUtterances"]
database = options["--database"]
log.info("access exchanges")
exchanges_Reddit = abstraction.access_exchanges_Reddit(
subreddits = subreddits,
number_of_utterances = number_of_utterances
)
log.info("save exchanges to database (only those not saved previously)")
abstraction.save_exchanges_to_database(
exchanges = exchanges_Reddit,
filename = database
)
abstraction.save_database_metadata(
filename = database
)
program.terminate()
def main(options):
global program
program = propyte.Program(
options = options,
name = name,
version = version,
logo = logo
)
global log
from propyte import log
log.info("")
# access options and arguments
hypermap_filename = options["--hypermapfile"]
# load grid search map
hypermap = shijian.import_object(filename = hypermap_filename)
abstraction.analyze_hypermap(
hypermap = hypermap
)
log.info("")
program.terminate()
def execute(
self,
):
if self.name == "close":
program.terminate()
else:
log.info("execute launcher \"{name}\"".format(name = self.name))
#print(self.command.split())
#subprocess.Popen(["bash", "-c"] + self.command.split())
os.system(self.command)
def display_orientation(self, orientation=None):
if orientation in ["left", "right", "inverted", "normal"]:
log.info("change display to {orientation}".format(
orientation=orientation))
engage_command(
"xrandr -o {orientation}".format(orientation=orientation))
else:
log.error("unknown display orientation \"{orientation}\" "
"requested".format(orientation=orientation))
sys.exit()
def main(options):
global program
program = propyte.Program(options=options,
name=name,
version=version,
logo=logo)
global log
from propyte import log
print("")
filename_ROOT = options["--fileroot"]
name_tree = options["--tree"]
if not os.path.isfile(os.path.expandvars(filename_ROOT)):
log.error("file {filename} not found".format(filename=filename_ROOT))
program.terminate()
file_ROOT = abstraction.open_ROOT_file(filename_ROOT)
tree = file_ROOT.Get(name_tree)
number_entries = tree.GetEntries()
names_variables = [
variable.GetName() for variable in tree.GetListOfBranches()
]
names_variables = shijian.natural_sort(names_variables)
names_objects = [key.GetName() for key in file_ROOT.GetListOfKeys()]
names_objects = shijian.natural_sort(names_objects)
log.info(
textwrap.dedent("""
input ROOT file: {filename_ROOT}
number of entries: {number_entries}
""".format(filename_ROOT=filename_ROOT,
number_entries=number_entries)))
log.info("variables:")
print("")
for name_variable in names_variables:
log.info(" " + name_variable)
print("")
log.info("objects:")
print("")
for name_object in names_objects:
log.info(" " + name_object)
#print("")
#log.info("tree printout:")
#print("")
#tree.Print()
#print("")
program.terminate()
def stylus_proximity_control_switch(self, status=None):
if status == "on":
log.info("change stylus proximity control to on")
self.process_stylus_proximity_control = multiprocessing.Process(
target=self.stylus_proximity_control)
self.process_stylus_proximity_control.start()
elif status == "off":
log.info("change stylus proximity control to off")
self.process_stylus_proximity_control.terminate()
else:
log.error("unknown stylus proximity control status \"{status}\" "
"requested".format(status=status))
sys.exit()
def acceleration_control_switch(self, status=None):
if status == "on":
log.info("change acceleration control to on")
self.process_acceleration_control = multiprocessing.Process(
target=self.acceleration_control)
self.process_acceleration_control.start()
elif status == "off":
log.info("change acceleration control to off")
self.process_acceleration_control.terminate()
else:
log.error("unknown acceleration control status \"{status}\" "
"requested".format(status=status))
sys.exit()
def main(options):
global program
program = propyte.Program(options=options,
name=name,
version=version,
logo=logo)
global log
from propyte import log
# access options and arguments
database_filename = options["--database"]
log.info("")
database = abstraction.access_database(filename=database_filename)
for table in database.tables:
log.info("\ntable: {table}/n".format(table=table))
for entry in database[table].all():
pyprel.print_line()
for column in database[table].columns:
log.info("\n{column}: {content}".format(column=column,
content=str(
entry[column])))
pyprel.print_line()
log.info("")
program.terminate()
def main(options):
global program
program = propyte.Program(options=options,
name=name,
version=version,
logo=logo)
global log
from propyte import log
# access options and arguments
database = options["--database"]
word_vector_model = options["--wordvectormodel"]
log.info("")
log.info("load word vector model {model}".format(model=word_vector_model))
model_word2vec = abstraction.load_word_vector_model(
filename=word_vector_model)
log.info("add exchange word vectors to database {database}".format(
database=database))
abstraction.add_exchange_word_vectors_to_database(
filename=database, model_word2vec=model_word2vec)
log.info("")
program.terminate()
def main(options):
global program
program = propyte.Program(
options = options,
name = name,
version = version,
logo = logo
)
global log
from propyte import log
log.info("")
input_data_filename = options["--data"]
log.info("input data file: {filename}".format(
filename = input_data_filename
))
log.debug("start to print log messages at various levels")
log.debug("message at level DEBUG")
log.info("message at level INFO")
log.warning("message at level WARNING")
log.error("message at level ERROR")
log.critical("message at level CRITICAL")
log.debug("stop printing log messages at various levels")
function_1()
log.info("")
program.terminate()
def display_position_control_switch(self, status=None):
if status == "on":
log.info("change display position control to on")
self.process_display_position_control = multiprocessing.Process(
target=self.display_position_control)
self.process_display_position_control.start()
elif status == "off":
log.info("change display position control to off")
self.process_display_position_control.terminate()
else:
log.error(
"unknown display position control status \"{orientation}\" "
"requested".format(status=status))
sys.exit()
def main(options):
global program
program = propyte.Program(options=options,
name=name,
version=version,
logo=logo)
global log
from propyte import log
log.info("")
input_data_filename = options["--data"]
log.info(
"input data file: {filename}".format(filename=input_data_filename))
log.debug("start to print log messages at various levels")
log.debug("message at level DEBUG")
log.info("message at level INFO")
log.warning("message at level WARNING")
log.error("message at level ERROR")
log.critical("message at level CRITICAL")
log.debug("stop printing log messages at various levels")
function_1()
log.info("")
program.terminate()
def display_position_control(self):
socket_ACPI = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
socket_ACPI.connect("/var/run/acpid.socket")
log.info("display position is {display_position_status}".format(
display_position_status=self.display_position_status))
while True:
event_ACPI = socket_ACPI.recv(4096)
# Ubuntu 13.10 compatibility:
#event_ACPI_display_position_change = \
# "ibm/hotkey HKEY 00000080 000060c0\n"
# Ubuntu 14.04 compatibility:
event_ACPI_display_position_change = \
"ibm/hotkey LEN0068:00 00000080 000060c0\n"
if event_ACPI == event_ACPI_display_position_change:
log.info("display position change")
if self.display_position_status == "laptop":
self.engage_mode(mode="tablet")
self.display_position_status = "tablet"
log.info(
"display position is {display_position_status}".format(
display_position_status =\
self.display_position_status
)
)
elif self.display_position_status == "tablet":
self.engage_mode(mode="laptop")
self.display_position_status = "laptop"
log.info(
"display position is {display_position_status}".format(
display_position_status =\
self.display_position_status
)
)
time.sleep(0.15)
def main(options):
global program
program = propyte.Program(options=options,
name=name,
version=version,
logo=logo)
global log
from propyte import log
filename_database = options["--database"]
rows_limit = options["--rows"]
if rows_limit is not None:
rows_limit = int(rows_limit)
log.info("\naccess database {filename}".format(filename=filename_database))
database = dataset.connect("sqlite:///{filename_database}".format(
filename_database=filename_database))
for name_table in database.tables:
log.info("access table \"{name_table}\"".format(name_table=name_table))
table = database[name_table]
log.info("number of rows in table \"{name_table}\": {number_of_rows}".
format(name_table=name_table, number_of_rows=str(len(table))))
log.info(
"\ntable {name_table} printout:\n".format(name_table=name_table))
print(
pyprel.Table(contents=pyprel.table_dataset_database_table(
table=database[name_table], rows_limit=rows_limit)))
program.terminate()
def nipple_switch(self, status=None):
if "nipple" in self.names_devices:
status_xinput = {"on": "enable", "off": "disable"}
if status_xinput.has_key(status):
log.info("change nipple to {status}".format(status=status))
engage_command("xinput {status} \"{name_device}\"".format(
status=status_xinput[status],
name_device=self.names_devices["nipple"]))
else:
_message = "unknown nipple status \"{status}\" " +\
"requested"
log.error(_message.format(status=status))
sys.exit()
else:
log.debug("nipple status unchanged")
def main(options):
global program
program = propyte.Program(
options = options,
name = name,
version = version,
logo = logo
)
global log
from propyte import log
log.info("")
usernames = [
"AndrewYNg",
"geoff_hinton",
"SamHarrisOrg",
"ylecun"
]
tweets = abstraction.access_users_tweets(
usernames = usernames
)
log.info("\ntable of tweets:\n")
print(tweets.table())
log.info("\nmost frequently expressed calculated sentiments of users:\n")
users_sentiments_single_most_frequent = tweets.users_sentiments_single_most_frequent()
pyprel.print_dictionary(dictionary = users_sentiments_single_most_frequent)
users_sentiment_negative = []
for username, sentiment in users_sentiments_single_most_frequent.iteritems():
if sentiment == "neg":
users_sentiment_negative.append(username)
log.info("list of users expressing negative sentiments most frequently:\n\n{usernames}".format(
usernames = users_sentiment_negative
))
log.info("")
program.terminate()
def main(options):
global program
program = propyte.Program(
options = options,
name = name,
version = version,
logo = logo
)
global log
from propyte import log
log.info("")
# access options and arguments
ROOT_filename = options["--data"]
log.info("load classification model")
classifier = abstraction.Classification(
load_from_directory = "abstraction_classifier_ttH_ttbb_300000_50_200_400_50_300"
#load_from_directory = "abstraction_classifier_ttH_ttbb_300000_50_150_250_300_400"
#load_from_directory = "abstraction_classifier"
)
# Access data.
data = abstraction.load_HEP_data(
ROOT_filename = ROOT_filename,
tree_name = "nominal",
maximum_number_of_events = 5000
)
# Add class labels.
if "ttH" in ROOT_filename:
class_value = 1
if "ttbb" in ROOT_filename:
class_value = 0
for index in data.indices():
data.variable(index = index, name = "class", value = class_value)
# Preprocess all data.
data.preprocess_all()
# Convert the datavision dataset to an abstraction dataset.
dataset = abstraction.convert_HEP_datasets_from_datavision_datasets_to_abstraction_datasets(
datasets = data
)
# Classify data and add the results to the datavision dataset.
results = list(classifier._model.predict(dataset.features()))
for count, index in enumerate(data.indices()):
data.variable(index = index, name = "abstraction1", value = results[count])
log.info(data.table())
log.info("")
program.terminate()
def main(options):
global program
program = propyte.Program(
options = options,
name = name,
version = version,
logo = logo
)
global log
from propyte import log
global filename_log
filename_log = options["--logfile"]
log.info("")
keyboard = Keyboard()
keyboard.log_loop()
def send_messages(
messages = None
):
if messages:
for index, message in enumerate(messages):
try:
log.info("send message to {to}".format(
to = message["To"]
))
server = smtplib.SMTP("localhost")
server.sendmail(
message["From"],
message["To"],
message.as_string()
)
server.quit()
except smtplib.SMTPException:
print("e-mail send error")
time.sleep(5)
def main(options):
filename_log = options["--logfile"]
global program
program = propyte.Program(options=options,
name=name,
version=version,
logo=logo,
filename_log=filename_log)
global log
from propyte import log
log.debug("message at level DEBUG")
log.info("message at level INFO")
log.warning("message at level WARNING")
log.error("message at level ERROR")
log.critical("message at level CRITICAL")
program.terminate()
def engage_mode(self, mode=None):
log.info("engage mode {mode}".format(mode=mode))
if mode == "tablet":
self.display_orientation(orientation="left")
self.touchscreen_orientation(orientation="left")
self.touchpad_switch(status="off")
self.nipple_switch(status="off")
elif mode == "laptop":
self.display_orientation(orientation="normal")
self.touchscreen_orientation(orientation="normal")
self.touchscreen_switch(status="on")
self.touchpad_orientation(orientation="normal")
self.touchpad_switch(status="on")
self.nipple_switch(status="on")
elif mode in ["left", "right", "inverted", "normal"]:
self.display_orientation(orientation=mode)
self.touchscreen_orientation(orientation=mode)
self.touchpad_orientation(orientation=mode)
else:
log.error("unknown mode \"{mode}\" requested".format(mode=mode))
sys.exit()
def acceleration_control(self):
while True:
# Get the mean of recent acceleration vectors.
number_of_measurements = 3
measurements = []
for measurement in range(0, number_of_measurements):
measurements.append(Acceleration_Vector())
stable_acceleration = mean_list(lists=measurements)
log.info("stable acceleration vector: {vector}".format(
vector=stable_acceleration))
table_orientations = {
(True, True): "left",
(True, False): "right",
(False, True): "inverted",
(False, False): "normal"
}
orientation = table_orientations[(abs(
stable_acceleration[0]) > abs(stable_acceleration[1]),
stable_acceleration[0] > 0)]
self.engage_mode(mode=orientation)
time.sleep(0.15)
def stylus_proximity_control(self):
self.previous_stylus_proximity_status = None
while True:
stylus_proximity_command = "xinput query-state " +\
"\"Wacom ISDv4 EC Pen stylus\" | " +\
"grep Proximity | cut -d \" \" -f3 | " +\
" cut -d \"=\" -f2"
self.stylus_proximity_status = subprocess.check_output(
stylus_proximity_command, shell=True).lower().rstrip()
if\
(self.stylus_proximity_status == "out") and \
(self.previous_stylus_proximity_status != "out"):
log.info("stylus inactive")
self.touchscreen_switch(status="on")
elif\
(self.stylus_proximity_status == "in") and \
(self.previous_stylus_proximity_status != "in"):
log.info("stylus active")
self.touchscreen_switch(status="off")
self.previous_stylus_proximity_status = self.stylus_proximity_status
time.sleep(0.15)
def main(options):
global program
program = propyte.Program(
options = options,
name = name,
version = version,
logo = logo
)
global log
from propyte import log
interval = float(options["--interval"])
clock_restart = shijian.Clock(name = "restart")
log.info("\nrestart interval: {interval} s".format(interval = interval))
while True:
log.info("\ncurrent run time: {run_time}".format(run_time = clock_restart.time()))
log.info("restart yet? (i.e. current run time >= interval): {restart}".format(restart = clock_restart.time() >= interval))
if clock_restart.time() >= interval:
print(pyprel.center_string(text = pyprel.render_banner(text = "restart")))
propyte.restart()
time.sleep(1)
def main(options):
global program
program = propyte.Program(options=options,
name=name,
version=version,
logo=logo)
global log
from propyte import log
interval = float(options["--interval"])
clock_restart = shijian.Clock(name="restart")
log.info("\nrestart interval: {interval} s".format(interval=interval))
while True:
log.info("\ncurrent run time: {run_time}".format(
run_time=clock_restart.time()))
log.info("restart yet? (i.e. current run time >= interval): {restart}".
format(restart=clock_restart.time() >= interval))
if clock_restart.time() >= interval:
print(
pyprel.center_string(text=pyprel.render_banner(
text="restart")))
propyte.restart()
time.sleep(1)
def main(options):
global program
program = propyte.Program(options=options,
name=name,
version=version,
logo=logo)
global log
from propyte import log
log.info("")
# access options and arguments
ROOT_filename = options["--data"]
log.info("load classification model")
classifier = abstraction.Classification(
load_from_directory=
"abstraction_classifier_ttH_ttbb_300000_50_200_400_50_300"
#load_from_directory = "abstraction_classifier_ttH_ttbb_300000_50_150_250_300_400"
#load_from_directory = "abstraction_classifier"
)
# Access data.
data = abstraction.load_HEP_data(ROOT_filename=ROOT_filename,
tree_name="nominal",
maximum_number_of_events=5000)
# Add class labels.
if "ttH" in ROOT_filename:
class_value = 1
if "ttbb" in ROOT_filename:
class_value = 0
for index in data.indices():
data.variable(index=index, name="class", value=class_value)
# Preprocess all data.
data.preprocess_all()
# Convert the datavision dataset to an abstraction dataset.
dataset = abstraction.convert_HEP_datasets_from_datavision_datasets_to_abstraction_datasets(
datasets=data)
# Classify data and add the results to the datavision dataset.
results = list(classifier._model.predict(dataset.features()))
for count, index in enumerate(data.indices()):
data.variable(index=index, name="abstraction1", value=results[count])
log.info(data.table())
log.info("")
program.terminate()
def get_inputs():
log.info("audit inputs")
devices_input = subprocess.Popen(["xinput", "--list"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()[0]
devices_and_keyphrases = {
"touchscreen":
["SYNAPTICS Synaptics Touch Digitizer V04", "ELAN Touchscreen"],
"touchpad": ["PS/2 Synaptics TouchPad", "SynPS/2 Synaptics TouchPad"],
"nipple": ["TPPS/2 IBM TrackPoint"],
"stylus": ["Wacom ISDv4 EC Pen stylus"]
}
names_devices = {}
for device, keyphrases in devices_and_keyphrases.iteritems():
for keyphrase in keyphrases:
if keyphrase in devices_input:
names_devices[device] = keyphrase
for device, keyphrases in devices_and_keyphrases.iteritems():
if device in names_devices:
log.info("input {device} detected as \"{name_device}\"".format(
device=device, name_device=names_devices[device]))
else:
log.info("input {device} not detected".format(device=device))
return names_devices
def main(options):
global program
program = propyte.Program(options=options,
name=name,
version=version,
logo=logo)
global log
from propyte import log
filename_database = options["--database"]
name_table = options["--table"]
name_table_metadata = options["--tablemetadata"]
rows_limit = options["--rows"]
if rows_limit is not None:
rows_limit = int(rows_limit)
log.info("\naccess database {filename}".format(filename=filename_database))
database = dataset.connect("sqlite:///{filename_database}".format(
filename_database=filename_database))
log.info("access table \"{name_table}\"".format(name_table=name_table))
table = database[name_table]
log.info(
"number of rows in table \"{name_table}\": {number_of_rows}".format(
name_table=name_table, number_of_rows=str(len(table))))
log.info("\ntable {name_table} printout:\n".format(name_table=name_table))
print(
pyprel.Table(contents=pyprel.table_dataset_database_table(
table=database[name_table],
include_attributes=["utterance", "response", "exchangeReference"],
rows_limit=rows_limit)))
log.info("database metadata:")
print(
pyprel.Table(contents=pyprel.table_dataset_database_table(
table=database[name_table_metadata], )))
program.terminate()
def main(options):
filename_log = options["--logfile"]
global program
program = propyte.Program(
options = options,
name = name,
version = version,
logo = logo,
filename_log = filename_log
)
global log
from propyte import log
log.debug("message at level DEBUG")
log.info("message at level INFO")
log.warning("message at level WARNING")
log.error("message at level ERROR")
log.critical("message at level CRITICAL")
program.terminate()
def touchpad_orientation(self, orientation=None):
if "touchpad" in self.names_devices:
coordinate_transformation_matrix = {
"left": "0 -1 1 1 0 0 0 0 1",
"right": "0 1 0 -1 0 1 0 0 1",
"inverted": "-1 0 1 0 -1 1 0 0 1",
"normal": "1 0 0 0 1 0 0 0 1"
}
if coordinate_transformation_matrix.has_key(orientation):
log.info("change touchpad to {orientation}".format(
orientation=orientation))
engage_command(
"xinput set-prop \"{name_device}\" \"Coordinate "
"Transformation Matrix\" "
"{matrix}".format(
name_device=self.names_devices["touchpad"],
matrix=coordinate_transformation_matrix[orientation]))
else:
log.error("unknown touchpad orientation \"{orientation}\""
" requested".format(orientation=orientation))
sys.exit()
else:
log.debug("touchpad orientation unchanged")
def main(options):
global program
program = propyte.Program(options=options,
name=name,
version=version,
logo=logo)
global log
from propyte import log
# access options and arguments
subreddits = options["--subreddits"].split(",")
number_of_utterances = options["--numberOfUtterances"]
database = options["--database"]
log.info("access exchanges")
exchanges_Reddit = abstraction.access_exchanges_Reddit(
subreddits=subreddits, number_of_utterances=number_of_utterances)
log.info("save exchanges to database (only those not saved previously)")
abstraction.save_exchanges_to_database(exchanges=exchanges_Reddit,
filename=database)
abstraction.save_database_metadata(filename=database)
program.terminate()
def main(options):
global program
program = propyte.Program(
options = options,
name = name,
version = version,
logo = logo
)
global log
from propyte import log
# access options and arguments
database_filename = options["--database"]
log.info("")
database = abstraction.access_database(
filename = database_filename
)
for table in database.tables:
log.info("\ntable: {table}/n".format(
table = table
))
for entry in database[table].all():
pyprel.print_line()
for column in database[table].columns:
log.info("\n{column}: {content}".format(
column = column,
content = str(entry[column])
))
pyprel.print_line()
log.info("")
program.terminate()
def main(options):
global program
program = propyte.Program(options=options,
name=name,
version=version,
logo=logo)
global log
from propyte import log
print("")
filename_CSV_input = options["--infile"]
filename_CSV_output = options["--outfile"]
if not os.path.isfile(os.path.expandvars(filename_CSV_input)):
log.error(
"file {filename} not found".format(filename=filename_CSV_input))
program.terminate()
log.info("read CSV from {filename}".format(filename=filename_CSV_input))
data = pd.read_csv(filename_CSV_input)
scaler = sklearn.preprocessing.MinMaxScaler(feature_range=(-1, 1))
number_of_columns = data.shape[1]
indices_of_feature_columns = range(0, number_of_columns - 1)
# scale feature columns
log.info("scale features")
data[indices_of_feature_columns] = scaler.fit_transform(
data[indices_of_feature_columns])
log.info(
"save scaled CSV to {filename}".format(filename=filename_CSV_output))
data.to_csv(
filename_CSV_output,
index=False,
#header = False
)
print("")
program.terminate()
def main(options):
global program
program = propyte.Program(options=options,
name=name,
version=version,
logo=logo)
global log
from propyte import log
# access options and arguments
expression = options["--expression"]
word_vector_model = options["--wordvectormodel"]
model_word2vec = abstraction.load_word_vector_model(
filename=word_vector_model)
# Convert the expression to a word vector.
expression_word_vector =\
abstraction.convert_sentence_string_to_word_vector(
sentence_string = expression,
model_word2vec = model_word2vec
)
log.info("word vector representation of expression \"{expression}\":"
"\n{expression_word_vector}".format(
expression=expression,
expression_word_vector=expression_word_vector))
log.info("")
log.info(
"word vector representation of expression \"{expression}\" as NumPy "
"array:\n{expression_NumPy_array}".format(
expression=expression,
expression_NumPy_array=numpy.array_repr(expression_word_vector)))
program.terminate()
def main(options):
global program
program = propyte.Program(options=options,
name=name,
version=version,
logo=logo)
global log
from propyte import log
# access options and arguments
database = options["--inputdatabase"]
database_out = options["--outputdatabase"]
# Access database.
database = abstraction.access_database(filename=database)
log.info("database metadata:")
abstraction.log_database_metadata(filename=database)
# Print the tables in the database.
log.info("tables in database: {tables}".format(tables=database.tables))
# Access the exchanges table.
tablename = "exchanges"
log.info("access table \"{tablename}\"".format(tablename=tablename))
# Print the columns of the table.
log.info("columns in table \"{tablename}\": {columns}".format(
tablename=tablename, columns=database[tablename].columns))
# Print the number of rows of the table.
log.info(
"number of rows in table \"{tablename}\": {number_of_rows}".format(
tablename=tablename, number_of_rows=str(len(database[tablename]))))
# Build a list of unique exchanges.
exchanges = []
for entry in database[tablename].all():
# Create a new exchange object for the existing exchange data, check its
# utterance data against existing utterance data in the new list of
# exchanges and append it to the new list of exchanges if it does not
# exist in the list.
exchange = abstraction.Exchange(
utterance=entry["utterance"],
response=entry["response"],
utterance_time_UNIX=entry["utteranceTimeUNIX"],
response_time_UNIX=entry["responseTimeUNIX"],
utterance_reference=entry["utteranceReference"],
response_reference=entry["responseReference"],
exchange_reference=entry["exchangeReference"])
# Check new exchange against exchanges in new list.
append_flag = True
for exchange_in_new_list in exchanges:
if exchange.utterance == exchange_in_new_list.utterance:
append_flag = False
if append_flag is True:
log.debug("keep exchange \"{utterance}\"".format(
utterance=exchange.utterance))
exchanges.append(exchange)
else:
log.debug("skip exchange \"{utterance}\"".format(
utterance=exchange.utterance))
# Save the exchanges to the new database.
log.info("save exchanges to database (only those not saved previously)")
abstraction.save_exchanges_to_database(exchanges=exchanges,
filename=database_out)
# Save metadata to the new database.
abstraction.save_database_metadata(filename=database_out)
program.terminate()
def main(options):
global program
program = propyte.Program(
options = options,
name = name,
version = version,
logo = logo
)
global log
from propyte import log
log.info("")
# access options and arguments
input_data_filename = options["--data"]
# define dataset
# Load the SUSY dataset (https://archive.ics.uci.edu/ml/datasets/SUSY).
# The first column is the class label (1 for signal, 0 for background),
# followed by 18 features (8 low-level features and 10 high-level features):
#
# - lepton 1 pT
# - lepton 1 eta
# - lepton 1 phi
# - lepton 2 pT
# - lepton 2 eta
# - lepton 2 phi
# - missing energy magnitude
# - missing energy phi
# - MET_rel
# - axial MET
# - M_R
# - M_TR_2
# - R
# - MT2
# - S_R
# - M_Delta_R
# - dPhi_r_b
# - cos(theta_r1)
data = abstraction.access_SUSY_dataset_format_file(input_data_filename)
dataset = abstraction.Dataset(
data = data
)
# define data
log.info("split data for cross-validation")
features_train, features_test, targets_train, targets_test =\
cross_validation.train_test_split(
dataset.features(),
dataset.targets(),
train_size = 0.7
)
# grid search
import itertools
epochs = [10, 100, 500, 1000]
architecture = [200, 300, 300, 300, 200]
grid_search_map = {}
grid_search_map["epoch"] = []
grid_search_map["hidden_nodes"] = []
grid_search_map["score_training"] = []
grid_search_map["score_test"] = []
# define progress
count_total = 0
for epoch in epochs:
for nodes_count in xrange(1, len(architecture) + 1):
combinations = itertools.product(architecture, repeat = nodes_count)
for combination in combinations:
count_total += 1
count = 0
progress = shijian.Progress()
progress.engage_quick_calculation_mode()
for epoch in epochs:
for nodes_count in xrange(1, len(architecture) + 1):
combinations = itertools.product(architecture, repeat = nodes_count)
for combination in combinations:
hidden_nodes = list(combination)
# define model
log.info("define classification model")
classifier = abstraction.Classification(
number_of_classes = 2,
hidden_nodes = hidden_nodes,
epochs = epoch
)
# train model
log.info("fit model to dataset features and targets")
classifier._model.fit(features_train, targets_train)
#classifier.save()
# predict and cross-validate training
log.info("test trained model on training dataset")
score_training = metrics.accuracy_score(
classifier._model.predict(features_train),
targets_train
)
score_test = metrics.accuracy_score(
classifier._model.predict(features_test),
targets_test
)
log.info("\ntraining-testing instance complete:")
log.info("epoch: {epoch}".format(
epoch = epoch
))
log.info("architecture: {architecture}".format(
architecture = hidden_nodes
))
log.info("score training: {score_training}".format(
score_training = 100 * score_training
))
log.info("score test: {score_test}".format(
score_test = 100 * score_test
))
pyprel.print_line()
grid_search_map["epoch"].append(epoch)
grid_search_map["hidden_nodes"].append(hidden_nodes)
grid_search_map["score_training"].append(score_training)
grid_search_map["score_test"].append(score_test)
# save current grid search map
shijian.export_object(
grid_search_map,
filename = "grid_search_map.pkl",
overwrite = True
)
count += 1
print(progress.add_datum(fraction = (count + 1) / count_total))
number_of_entries = len(grid_search_map["epoch"])
# table
table_contents = [
["epoch", "architecture", "score training", "score testing"]
]
for index in range(0, number_of_entries):
table_contents.append(
[
str(grid_search_map["epoch"][index]),
str(grid_search_map["hidden_nodes"][index]),
str(grid_search_map["score_training"][index]),
str(grid_search_map["score_test"][index])
]
)
print("\ngrid search map:\n")
print(
pyprel.Table(
contents = table_contents,
)
)
# plot
architectures = shijian.unique_list_elements(grid_search_map["hidden_nodes"])
architecture_epoch_score = {}
for architecture in architectures:
architecture_epoch_score[str(architecture)] = []
for index in range(0, number_of_entries):
if grid_search_map["hidden_nodes"][index] == architecture:
architecture_epoch_score[str(architecture)].append(
[
grid_search_map["epoch"][index],
grid_search_map["score_test"][index]
]
)
figure = matplotlib.pyplot.figure()
figure.set_size_inches(10, 10)
axes = figure.add_subplot(1, 1, 1)
axes.set_xscale("log")
figure.suptitle("hyperparameter map", fontsize = 20)
matplotlib.pyplot.xlabel("epochs")
matplotlib.pyplot.ylabel("training test score")
for key, value in architecture_epoch_score.iteritems():
epochs = [element[0] for element in value]
score_test = [element[1] for element in value]
matplotlib.pyplot.plot(epochs, score_test, label = key)
matplotlib.pyplot.legend(loc = "center right")
matplotlib.pyplot.savefig(
"hyperparameter_map.eps",
bbox_inches = "tight",
format = "eps"
)
# find best-scoring models
# Find the 3 best scores.
best_models = sorted(zip(
grid_search_map["score_test"],
grid_search_map["hidden_nodes"]),
reverse = True
)[:3]
# table
table_contents = [["architecture", "score testing"]]
for model in best_models:
table_contents.append([str(model[1]), str(model[0])])
print("\nbest-scoring models:\n")
print(
pyprel.Table(
contents = table_contents,
)
)
log.info("")
program.terminate()
def main(options):
global program
program = propyte.Program(options=options,
name=name,
version=version,
logo=logo)
global log
from propyte import log
log.info("")
# access options and arguments
grid_search_filename = options["--gridsearchfile"]
# load grid search map
grid_search_map = shijian.import_object(filename=grid_search_filename)
number_of_entries = len(grid_search_map["epoch"])
log.info("number of entries: {number_of_entries}".format(
number_of_entries=number_of_entries))
# table
table_contents = [[
"epoch", "architecture", "score training", "score testing"
]]
for index in range(0, number_of_entries):
table_contents.append([
str(grid_search_map["epoch"][index]),
str(grid_search_map["hidden_nodes"][index]),
str(grid_search_map["score_training"][index]),
str(grid_search_map["score_test"][index])
])
log.info("\ngrid search map:\n")
log.info(pyprel.Table(contents=table_contents, ))
# parallel coordinates plot
number_of_entries = len(grid_search_map["epoch"])
datasets = []
for index in range(0, number_of_entries):
row = []
architecture_padded = grid_search_map["hidden_nodes"][index] + [0] * (
5 - len(grid_search_map["hidden_nodes"][index]))
row.append(grid_search_map["epoch"][index])
row.extend(architecture_padded)
row.append(grid_search_map["score_training"][index])
row.append(grid_search_map["score_test"][index])
datasets.append(row)
datavision.save_parallel_coordinates_matplotlib(
datasets[::-1], filename="parallel_coordinates.png")
# plot
architectures = shijian.unique_list_elements(
grid_search_map["hidden_nodes"])
architecture_epoch_score = {}
for architecture in architectures:
architecture_epoch_score[str(architecture)] = []
for index in range(0, number_of_entries):
if grid_search_map["hidden_nodes"][index] == architecture:
architecture_epoch_score[str(architecture)].append([
grid_search_map["epoch"][index],
grid_search_map["score_test"][index]
])
figure = matplotlib.pyplot.figure()
figure.set_size_inches(10, 10)
axes = figure.add_subplot(1, 1, 1)
axes.set_xscale("log")
figure.suptitle("hyperparameter map", fontsize=20)
matplotlib.pyplot.xlabel("epochs")
matplotlib.pyplot.ylabel("training test score")
for key, value in architecture_epoch_score.iteritems():
epochs = [element[0] for element in value]
score_test = [element[1] for element in value]
matplotlib.pyplot.plot(epochs, score_test, label=key)
matplotlib.pyplot.legend(loc="center left",
bbox_to_anchor=(1, 0.5),
fontsize=10)
matplotlib.pyplot.savefig("hyperparameter_map.eps",
bbox_inches="tight",
format="eps")
# find best-scoring models
# Find the 15 best scores and plot them using parallel coordinates.
best_models = sorted(zip(grid_search_map["score_test"],
grid_search_map["score_training"],
grid_search_map["hidden_nodes"]),
reverse=True)[:15]
datasets = []
for model in best_models:
row = []
architecture_padded = model[2] + [0] * (5 - len(model[2]))
row.extend(architecture_padded)
row.append(model[1])
row.append(model[0])
datasets.append(row)
datavision.save_parallel_coordinates_matplotlib(
datasets, filename="15_best_models_parallel_coordinates.png")
# Find the 3 best scores.
best_models = sorted(zip(grid_search_map["score_test"],
grid_search_map["hidden_nodes"]),
reverse=True)[:3]
# table
table_contents = [["architecture", "score testing"]]
for model in best_models:
table_contents.append([str(model[1]), str(model[0])])
log.info("\nbest-scoring models:\n")
log.info(pyprel.Table(contents=table_contents, ))
log.info("")
program.terminate()
def __init__(
self,
):
super(interface, self).__init__()
self.text_panel = QtGui.QLabel(program.panel_text)
if program.power:
self.indicator_percentage_power = QtGui.QLabel(self)
self.indicator_clock = QtGui.QLabel(self)
# Loop over all launchers specified in the configuration, building a
# list of launchers.
launchers = []
for name, attributes in list(program.configuration["launchers"].items()):
log.info("load launcher \"{name}\"".format(name = name))
# If a launcher has a "desktop entry" file specification, accept it
# in preference to other specifications of the launcher.
if "desktop entry" not in attributes:
# Cope with specification or no specification of the icon. If an
# icon is specified, set no button text.
if "icon" in attributes:
icon = attributes["icon"]
button = QtGui.QPushButton(self)
else:
icon = ""
button = QtGui.QPushButton(name, self)
# Parse the command.
command = attributes["command"]
else:
filepath_desktop_entry = attributes["desktop entry"]
file_desktop_entry = open(filepath_desktop_entry, "r")
for line in file_desktop_entry:
if "Icon=" in line:
icon = line.split("Icon=")[1].rstrip("\n")
if "Exec=" in line:
command = line.split("Exec=")[1].rstrip("\n")
# Cope with specification or no specification of the icon. If an
# icon is specified, set no button text.
if icon is not None:
button = QtGui.QPushButton(self)
else:
icon = ""
button = QtGui.QPushButton(name, self)
# Create the launcher.
launcher = Launcher(
name = name,
command = command,
icon = icon,
button = button
)
launchers.append(launcher)
# Add an close launcher.
if program.close_button:
name = "close"
launcher = Launcher(
name = name,
button = QtGui.QPushButton(name, self)
)
launchers.append(launcher)
# Set the layout.
vbox = QtGui.QVBoxLayout()
vbox.addStretch(1)
if program.panel_text != "":
vbox.addWidget(self.text_panel)
# Loop over all launchers, adding the launcher buttons to the layout.
for launcher in launchers:
# add button widget
vbox.addWidget(launcher.button)
vbox.addStretch(1)
if program.power:
vbox.addStretch(1)
vbox.addWidget(self.indicator_percentage_power)
vbox.addStretch(1)
vbox.addWidget(self.indicator_clock)
self.setLayout(vbox)
self.font = QtGui.QFont("Arial", 8)
self.setStyleSheet(
"""
color: #{color_1};
background-color: #{color_2}
""".format(
color_1 = program.color_1,
color_2 = program.color_2
)
)
self.text_panel.setStyleSheet(
"""
QLabel{{
color: #{color_1};
background-color: #{color_2};
border: 1px solid #{color_1};
}}
""".format(
color_1 = program.color_1,
color_2 = program.color_2
)
)
#self.text_panel.setFont(self.font)
self.text_panel.setAlignment(QtCore.Qt.AlignCenter)
if len(program.panel_text) <= 7:
self.text_panel.setFixedSize(60, 20)
else:
self.text_panel.setFixedSize(60, 60)
if program.power:
self.indicator_percentage_power.setStyleSheet(
"""
QLabel{{
color: #{color_1};
background-color: #{color_2};
border: 1px solid #{color_1};
}}
""".format(
color_1 = program.color_1,
color_2 = program.color_2
)
)
#self.indicator_percentage_power.setFont(self.font)
self.indicator_percentage_power.setAlignment(QtCore.Qt.AlignCenter)
self.indicator_percentage_power.setFixedSize(60, 60)
self.indicator_clock.setStyleSheet(
"""
QLabel{{
color: #{color_1};
background-color: #{color_2};
border: 1px solid #{color_1};
}}
""".format(
color_1 = program.color_1,
color_2 = program.color_2
)
)
self.indicator_clock.setFont(self.font)
self.indicator_clock.setAlignment(QtCore.Qt.AlignCenter)
self.indicator_clock.setFixedSize(60, 60)
self.setWindowTitle(program.name)
if program.set_always_on_top is True:
self.setWindowFlags(QtCore.Qt.WindowStaysOnTopHint)
if program.window_frame is False:
self.setWindowFlags(QtCore.Qt.FramelessWindowHint)
if program.set_position is True:
self.move(0, 0)
if program.power:
thread_percentage_power = threading.Thread(target = self.percentage_power)
thread_percentage_power.daemon = True
thread_percentage_power.start()
thread_clock = threading.Thread(target = self.clock)
thread_clock.daemon = True
thread_clock.start()
self.show()
def main(options):
global program
program = propyte.Program(
options = options,
name = name,
version = version,
logo = logo
)
global log
from propyte import log
log.info("")
# define dataset
dataset = abstraction.Dataset(
data =\
[
[5.1, 3.5, 1.4, 0.2], [0],
[4.9, 3.0, 1.4, 0.2], [0],
[4.7, 3.2, 1.3, 0.2], [0],
[4.6, 3.1, 1.5, 0.2], [0],
[5.0, 3.6, 1.4, 0.2], [0],
[5.4, 3.9, 1.7, 0.4], [0],
[4.6, 3.4, 1.4, 0.3], [0],
[5.0, 3.4, 1.5, 0.2], [0],
[4.4, 2.9, 1.4, 0.2], [0],
[4.9, 3.1, 1.5, 0.1], [0],
[5.4, 3.7, 1.5, 0.2], [0],
[4.8, 3.4, 1.6, 0.2], [0],
[4.8, 3.0, 1.4, 0.1], [0],
[4.3, 3.0, 1.1, 0.1], [0],
[5.8, 4.0, 1.2, 0.2], [0],
[5.7, 4.4, 1.5, 0.4], [0],
[5.4, 3.9, 1.3, 0.4], [0],
[5.1, 3.5, 1.4, 0.3], [0],
[5.7, 3.8, 1.7, 0.3], [0],
[5.1, 3.8, 1.5, 0.3], [0],
[5.4, 3.4, 1.7, 0.2], [0],
[5.1, 3.7, 1.5, 0.4], [0],
[4.6, 3.6, 1.0, 0.2], [0],
[5.1, 3.3, 1.7, 0.5], [0],
[4.8, 3.4, 1.9, 0.2], [0],
[5.0, 3.0, 1.6, 0.2], [0],
[5.0, 3.4, 1.6, 0.4], [0],
[5.2, 3.5, 1.5, 0.2], [0],
[5.2, 3.4, 1.4, 0.2], [0],
[4.7, 3.2, 1.6, 0.2], [0],
[4.8, 3.1, 1.6, 0.2], [0],
[5.4, 3.4, 1.5, 0.4], [0],
[5.2, 4.1, 1.5, 0.1], [0],
[5.5, 4.2, 1.4, 0.2], [0],
[4.9, 3.1, 1.5, 0.1], [0],
[5.0, 3.2, 1.2, 0.2], [0],
[5.5, 3.5, 1.3, 0.2], [0],
[4.9, 3.1, 1.5, 0.1], [0],
[4.4, 3.0, 1.3, 0.2], [0],
[5.1, 3.4, 1.5, 0.2], [0],
[5.0, 3.5, 1.3, 0.3], [0],
[4.5, 2.3, 1.3, 0.3], [0],
[4.4, 3.2, 1.3, 0.2], [0],
[5.0, 3.5, 1.6, 0.6], [0],
[5.1, 3.8, 1.9, 0.4], [0],
[4.8, 3.0, 1.4, 0.3], [0],
[5.1, 3.8, 1.6, 0.2], [0],
[4.6, 3.2, 1.4, 0.2], [0],
[5.3, 3.7, 1.5, 0.2], [0],
[5.0, 3.3, 1.4, 0.2], [0],
[7.0, 3.2, 4.7, 1.4], [1],
[6.4, 3.2, 4.5, 1.5], [1],
[6.9, 3.1, 4.9, 1.5], [1],
[5.5, 2.3, 4.0, 1.3], [1],
[6.5, 2.8, 4.6, 1.5], [1],
[5.7, 2.8, 4.5, 1.3], [1],
[6.3, 3.3, 4.7, 1.6], [1],
[4.9, 2.4, 3.3, 1.0], [1],
[6.6, 2.9, 4.6, 1.3], [1],
[5.2, 2.7, 3.9, 1.4], [1],
[5.0, 2.0, 3.5, 1.0], [1],
[5.9, 3.0, 4.2, 1.5], [1],
[6.0, 2.2, 4.0, 1.0], [1],
[6.1, 2.9, 4.7, 1.4], [1],
[5.6, 2.9, 3.6, 1.3], [1],
[6.7, 3.1, 4.4, 1.4], [1],
[5.6, 3.0, 4.5, 1.5], [1],
[5.8, 2.7, 4.1, 1.0], [1],
[6.2, 2.2, 4.5, 1.5], [1],
[5.6, 2.5, 3.9, 1.1], [1],
[5.9, 3.2, 4.8, 1.8], [1],
[6.1, 2.8, 4.0, 1.3], [1],
[6.3, 2.5, 4.9, 1.5], [1],
[6.1, 2.8, 4.7, 1.2], [1],
[6.4, 2.9, 4.3, 1.3], [1],
[6.6, 3.0, 4.4, 1.4], [1],
[6.8, 2.8, 4.8, 1.4], [1],
[6.7, 3.0, 5.0, 1.7], [1],
[6.0, 2.9, 4.5, 1.5], [1],
[5.7, 2.6, 3.5, 1.0], [1],
[5.5, 2.4, 3.8, 1.1], [1],
[5.5, 2.4, 3.7, 1.0], [1],
[5.8, 2.7, 3.9, 1.2], [1],
[6.0, 2.7, 5.1, 1.6], [1],
[5.4, 3.0, 4.5, 1.5], [1],
[6.0, 3.4, 4.5, 1.6], [1],
[6.7, 3.1, 4.7, 1.5], [1],
[6.3, 2.3, 4.4, 1.3], [1],
[5.6, 3.0, 4.1, 1.3], [1],
[5.5, 2.5, 4.0, 1.3], [1],
[5.5, 2.6, 4.4, 1.2], [1],
[6.1, 3.0, 4.6, 1.4], [1],
[5.8, 2.6, 4.0, 1.2], [1],
[5.0, 2.3, 3.3, 1.0], [1],
[5.6, 2.7, 4.2, 1.3], [1],
[5.7, 3.0, 4.2, 1.2], [1],
[5.7, 2.9, 4.2, 1.3], [1],
[6.2, 2.9, 4.3, 1.3], [1],
[5.1, 2.5, 3.0, 1.1], [1],
[5.7, 2.8, 4.1, 1.3], [1],
[6.3, 3.3, 6.0, 2.5], [2],
[5.8, 2.7, 5.1, 1.9], [2],
[7.1, 3.0, 5.9, 2.1], [2],
[6.3, 2.9, 5.6, 1.8], [2],
[6.5, 3.0, 5.8, 2.2], [2],
[7.6, 3.0, 6.6, 2.1], [2],
[4.9, 2.5, 4.5, 1.7], [2],
[7.3, 2.9, 6.3, 1.8], [2],
[6.7, 2.5, 5.8, 1.8], [2],
[7.2, 3.6, 6.1, 2.5], [2],
[6.5, 3.2, 5.1, 2.0], [2],
[6.4, 2.7, 5.3, 1.9], [2],
[6.8, 3.0, 5.5, 2.1], [2],
[5.7, 2.5, 5.0, 2.0], [2],
[5.8, 2.8, 5.1, 2.4], [2],
[6.4, 3.2, 5.3, 2.3], [2],
[6.5, 3.0, 5.5, 1.8], [2],
[7.7, 3.8, 6.7, 2.2], [2],
[7.7, 2.6, 6.9, 2.3], [2],
[6.0, 2.2, 5.0, 1.5], [2],
[6.9, 3.2, 5.7, 2.3], [2],
[5.6, 2.8, 4.9, 2.0], [2],
[7.7, 2.8, 6.7, 2.0], [2],
[6.3, 2.7, 4.9, 1.8], [2],
[6.7, 3.3, 5.7, 2.1], [2],
[7.2, 3.2, 6.0, 1.8], [2],
[6.2, 2.8, 4.8, 1.8], [2],
[6.1, 3.0, 4.9, 1.8], [2],
[6.4, 2.8, 5.6, 2.1], [2],
[7.2, 3.0, 5.8, 1.6], [2],
[7.4, 2.8, 6.1, 1.9], [2],
[7.9, 3.8, 6.4, 2.0], [2],
[6.4, 2.8, 5.6, 2.2], [2],
[6.3, 2.8, 5.1, 1.5], [2],
[6.1, 2.6, 5.6, 1.4], [2],
[7.7, 3.0, 6.1, 2.3], [2],
[6.3, 3.4, 5.6, 2.4], [2],
[6.4, 3.1, 5.5, 1.8], [2],
[6.0, 3.0, 4.8, 1.8], [2],
[6.9, 3.1, 5.4, 2.1], [2],
[6.7, 3.1, 5.6, 2.4], [2],
[6.9, 3.1, 5.1, 2.3], [2],
[5.8, 2.7, 5.1, 1.9], [2],
[6.8, 3.2, 5.9, 2.3], [2],
[6.7, 3.3, 5.7, 2.5], [2],
[6.7, 3.0, 5.2, 2.3], [2],
[6.3, 2.5, 5.0, 1.9], [2],
[6.5, 3.0, 5.2, 2.0], [2],
[6.2, 3.4, 5.4, 2.3], [2],
[5.9, 3.0, 5.1, 1.8], [2]
]
)
# define data
log.info("split data for cross-validation")
features_train, features_test, targets_train, targets_test =\
cross_validation.train_test_split(
dataset.features(),
dataset.targets(),
train_size = 0.7
)
log.info("define classification model")
# define model
classifier = abstraction.Classification(
number_of_classes = 3,
hidden_nodes = [10, 20, 10],
epochs = 500
)
# train model
log.info("fit classification model to dataset features and targets")
classifier._model.fit(features_train, targets_train)
#classifier.save()
# predict and cross-validate training
log.info("test trained classification model on training dataset")
score = metrics.accuracy_score(
classifier._model.predict(features_train),
targets_train
)
log.info("prediction accuracy on training dataset: {percentage}".format(
percentage = 100 * score
))
log.info("accuracy of classifier on test dataset:")
score = metrics.accuracy_score(
classifier._model.predict(features_test),
targets_test
)
log.info("prediction accuracy on test dataset: {percentage}".format(
percentage = 100 * score
))
log.info("")
program.terminate()
def main(options):
global program
program = propyte.Program(
options = options,
name = name,
version = version,
logo = logo
)
global log
from propyte import log
log.info("")
# access options and arguments
ROOT_filename_ttH = options["--datattH"]
ROOT_filename_ttbb = options["--datattbb"]
engage_plotting = string_to_bool(options["--plot"])
engage_correlations_analysis = string_to_bool(options["--analyzecorrelations"])
log.info("ttH data file: {filename}".format(
filename = ROOT_filename_ttH
))
log.info("ttbb data file: {filename}".format(
filename = ROOT_filename_ttbb
))
# Access data for event classes ttbb and ttH.
data_ttbb = abstraction.load_HEP_data(
ROOT_filename = ROOT_filename_ttbb,
tree_name = "nominal",
maximum_number_of_events = None
)
data_ttH = abstraction.load_HEP_data(
ROOT_filename = ROOT_filename_ttH,
tree_name = "nominal",
maximum_number_of_events = None
)
log.info("\nnumber of ttbb and ttH events: {number_of_events}\n".format(
number_of_events = len(data_ttbb.indices()) + len(data_ttH.indices())
))
# Plot comparisons of variables of the two datasets.
if engage_plotting is True:
for variable_name in data_ttbb.variables():
log.info("plot ttbb versus ttH comparison of {variable_name}".format(
variable_name = variable_name
))
datavision.save_histogram_comparison_matplotlib(
values_1 = data_ttbb.values(name = variable_name),
values_2 = data_ttH.values(name = variable_name),
label_1 = variable_name + "_ttbb",
label_2 = variable_name + "_ttH",
normalize = True,
label_ratio_x = "frequency",
label_y = "",
title = variable_name + "_ttbb_ttH",
filename = variable_name + "_ttbb_ttH.png",
directory = "variables_comparisons"
)
# Analyse variable correlations.
if engage_correlations_analysis is True:
variables_names = data_ttH.variables()
variables_values = []
for variable_name in variables_names:
variables_values.append(data_ttH.values(name = variable_name))
datavision.analyze_correlations(
variables = variables_values,
variables_names = variables_names,
table_order_variable = "p_value"
)
# Add class labels to the data sets, 0 for ttbb and 1 for ttH.
for index in data_ttbb.indices():
data_ttbb.variable(index = index, name = "class", value = 0)
for index in data_ttH.indices():
data_ttH.variable(index = index, name = "class", value = 1)
# With classes now defined, combine the datasets before preprocessing them.
data_ttbb.add(dataset = data_ttH)
# Preprocess all data: standardize the dataset by centering its variables to
# mean and scaling its variables to unit variance.
data_ttbb.preprocess_all()
# Convert the data sets to a simple list format with the first column
# containing the class label.
dataset = abstraction.convert_HEP_datasets_from_datavision_datasets_to_abstraction_datasets(
datasets = [data_ttbb]
)
log.info("")
# search
# Define search parameters: epochs, architectures
#elements_specification = [[300000], [50, 100], [100, 150], [200, 250], [300, 350], [400, 450]]
elements_specification = [
[300],
[1000, 10000],
[1000, 10000],
[1000, 10000],
[1000, 10000],
[1000, 10000]
]
log.info("hyperpoints specification: {elements_secification}".format(
elements_specification = elements_specification
))
hyperpoints = datavision.list_element_combinations_variadic(
elements_specification
)
# Remove hyperpoints with undefined architectures.
hyperpoints = [hyperpoint for hyperpoint in hyperpoints if len(hyperpoint) > 1]
# Remove variadic hyperpoints.
hyperpoints = [hyperpoint for hyperpoint in hyperpoints if len(hyperpoint) == len(elements_specification)]
abstraction.hypersearch(
hyperpoints = hyperpoints,
dataset = dataset
)
log.info("")
program.terminate()
def main(options):
global program
program = propyte.Program(options=options,
name=name,
version=version,
logo=logo)
global log
from propyte import log
# access options and arguments
input_image_filename = options["--inputImage"]
output_image_filename = options["--outputImage"]
recursive_zoom = bool(options["--recursiveZoom"])
GPU_mode = bool(options["--gpu"])
log.info("")
if GPU_mode is True:
log.info("engage GPU mode")
caffe.set_mode_gpu()
# If multiple devices exist, select GPU.
caffe.set_device(0)
else:
log.info("engage CPU mode")
# Load GoogLeNet model trained on ImageNet dataset.
log.info("load model")
username = os.getenv("USER")
model_path = "/home/{username}/caffe/models/bvlc_googlenet/".format(
username=username)
net_fn = model_path + "deploy.prototxt"
param_fn = model_path + "bvlc_googlenet.caffemodel"
# Patch the model to enable computation of gradients.
model = caffe.io.caffe_pb2.NetParameter()
text_format.Merge(open(net_fn).read(), model)
model.force_backward = True
open("tmp.prototxt", "w").write(str(model))
net = caffe.Classifier("tmp.prototxt",
param_fn,
mean=np.float32([104.0, 116.0, 122.0]),
channel_swap=(2, 1, 0))
## Save an image of the network.
#net.name = "network"
#caffe.draw.draw_net_to_file(
# net,
# "LR"
#)
log.info("access {filename}".format(filename=input_image_filename, ))
input_image = np.float32(PIL.Image.open(input_image_filename))
log.info("generate")
if recursive_zoom is not True:
output_image = deepdream(net, input_image, end="inception_4c/output")
output_image = np.uint8(output_image)
log.info("save {filename}".format(filename=output_image_filename, ))
PIL.Image.fromarray(output_image, "RGB").save(output_image_filename,
"PNG")
else:
os.makedirs("frames")
frame = input_image
frame_i = 0
h, w = frame.shape[:2]
s = 0.05 # scale coefficient
for i in xrange(100):
frame = deepdream(net, frame, end="inception_4c/output")
output_filename = "frames/{index}.jpg".format(index=frame_i)
PIL.Image.fromarray(np.uint8(frame)).save(output_filename)
frame = nd.affine_transform(frame, [1 - s, 1 - s, 1],
[h * s / 2, w * s / 2, 0],
order=1)
frame_i += 1
log.info("")
program.terminate()
def main(options):
global program
program = propyte.Program(
options = options,
name = name,
version = version,
logo = logo
)
global log
from propyte import log
timeout_main = 604800 # 1 week
timeout_warning = 3600 # 1 hour
log.info("")
"""
|human time representation|time in seconds|
|-------------------------|---------------|
|10 minutes |600 |
|30 minutes |1800 |
|1 hour |3600 |
|8 hours |28800 |
|10 hours |36000 |
|1 day |86400 |
|2 days |172800 |
|3 days |259200 |
|4 days |345600 |
|1 week |604800 |
|1 month |2629740 |
|1 year |31556900 |
"""
response = ""
while response is not None:
response = propyte.get_input_time_limited(
prompt = "Enter some text to restart the interaction loop: ",
timeout = timeout_main
)
log.info("\nstart warning procedures\n")
# warning messages
send_messages(messages = payload.messages_warning)
log.info("")
response = ""
while response is not None:
response = propyte.get_input_time_limited(
prompt = "WARNING: {timeout} seconds remaining -- Enter some text to restart the interaction loop (then reset monitor): ".format(
timeout = timeout_warning
),
timeout = timeout_warning
)
log.info("\nstart non-response procedures\n")
# messages
send_messages(messages = payload.messages)
log.info("\ngoodbye\n")
program.terminate()
def main(options):
global program
program = propyte.Program(
options = options,
name = name,
version = version,
logo = logo
)
global log
from propyte import log
# access options and arguments
input_image_filename = options["--inputimage"]
output_pixels_filename = options["--outputpixels"]
input_pixels_filename = options["--inputpixels"]
output_image_filename = options["--outputimage"]
output_image_width = int(options["--outputimagewidth"])
mode_convert_image_to_pixels = bool(options["--convertimagetopixels"])
mode_validate_pixels = bool(options["--validatepixels"])
mode_convert_pixels_to_image = bool(options["--convertPixelsToImage"])
log.info("")
if mode_convert_image_to_pixels is True:
log.info("convert image to pixels")
# Access the input image.
log.info("access input image")
input_image = PIL.Image.open(input_image_filename)
log.info("input image mode: {image_mode}".format(image_mode = image.mode))
log.info("input image size: {image_size}".format(image_size = image.size))
pixels = list(input_image.getdata())
pixels_text = str(pixels)
# Create and save the output pixels.
output_pixels_file = open(output_pixels_filename, "w")
output_pixels_file.truncate()
log.info("save output pixels {output_pixels_filename}".format(
output_pixels_filename = output_pixels_filename
))
output_pixels_file.write(pixels_text)
output_pixels_file.close()
elif mode_validate_pixels is True:
log.info("validate pixels")
# Access input pixels.
log.info("access input pixels")
with open(input_pixels_filename) as input_pixels_file:
text = input_pixels_file.read()
parts = text[2:-2].split("), (")
log.info("validate pixels")
for n, part in enumerate(parts):
if not re.match(r"^\d+, \d+, \d+, \d+$", part):
print("tuple {tuple_index} malformed: {tuple}".format(
tuple_index = n,
tuple = part
))
elif mode_convert_pixels_to_image is True:
log.info("convert pixels to image")
# Access input pixels.
log.info("access input pixels")
input_pixels_file = open(input_pixels_filename)
pixels = input_pixels_file.read()
pixels = ast.literal_eval(pixels)
# Determine the image height by determining the maximum number of image
# widths are possible with the available pixel data.
log.info("determine output image dimensions")
image_mode = "RGBA"
image_width = output_image_width # e.g. 2379
image_height = int(len(pixels)/image_width) # e.g. 2196
image_size = (image_width, image_height)
print("output image mode: {image_mode}".format(image_mode = image_mode))
print("output image size: {image_size}".format(image_size = image_size))
# Create and save the output image.
log.info("create output image")
output_image_file = Image.new(image_width, image_height)
output_image_file.putdata(pixels)
log.info("save output image {output_image_filename}".format(
output_image_filename = output_image_filename
))
output_image_file.save(output_image_filename)
else:
log.info("no operation selected\n")
print(__doc__)
log.info("")
program.terminate()
def main(options):
global program
program = propyte.Program(
options = options,
name = name,
version = version,
logo = logo
)
global log
from propyte import log
log.info("")
# access options and arguments
ROOT_filename_ttH = options["--datattH"]
ROOT_filename_ttbb = options["--datattbb"]
engage_plotting = shijian.string_to_bool(options["--plot"])
engage_correlations_analysis = shijian.string_to_bool(options["--analyzecorrelations"])
log.info("ttH data file: {filename}".format(
filename = ROOT_filename_ttH
))
log.info("ttbb data file: {filename}".format(
filename = ROOT_filename_ttbb
))
# Access data for event classes ttbb and ttH.
data_ttbb = abstraction.load_HEP_data(
ROOT_filename = ROOT_filename_ttbb,
tree_name = "nominal",
maximum_number_of_events = None
)
data_ttH = abstraction.load_HEP_data(
ROOT_filename = ROOT_filename_ttH,
tree_name = "nominal",
maximum_number_of_events = None
)
log.info("\nnumber of ttbb and ttH events: {number_of_events}\n".format(
number_of_events = len(data_ttbb.indices()) + len(data_ttH.indices())
))
# Plot comparisons of variables of the two datasets.
if engage_plotting is True:
for variable_name in data_ttbb.variables():
log.info("plot ttbb versus ttH comparison of {variable_name}".format(
variable_name = variable_name
))
datavision.save_histogram_comparison_matplotlib(
values_1 = data_ttbb.values(name = variable_name),
values_2 = data_ttH.values(name = variable_name),
label_1 = variable_name + "_ttbb",
label_2 = variable_name + "_ttH",
normalize = True,
label_ratio_x = "frequency",
label_y = "",
title = variable_name + "_ttbb_ttH",
filename = variable_name + "_ttbb_ttH.png",
directory = "variables_comparisons"
)
# Analyse variable correlations.
if engage_correlations_analysis is True:
variables_names = data_ttH.variables()
variables_values = []
for variable_name in variables_names:
variables_values.append(data_ttH.values(name = variable_name))
datavision.analyze_correlations(
variables = variables_values,
variables_names = variables_names,
table_order_variable = "p_value"
)
# Add class labels to the data sets, 0 for ttbb and 1 for ttH.
for index in data_ttbb.indices():
data_ttbb.variable(index = index, name = "class", value = 0)
for index in data_ttH.indices():
data_ttH.variable(index = index, name = "class", value = 1)
# With classes now defined, combine the datasets before preprocessing them.
data_ttbb.add(dataset = data_ttH)
# Preprocess all data: standardize the dataset by centering its variables to
# mean and scaling its variables to unit variance.
data_ttbb.preprocess_all()
# Convert the data sets to a simple list format with the first column
# containing the class label.
dataset = abstraction.convert_HEP_datasets_from_datavision_datasets_to_abstraction_datasets(
datasets = [data_ttbb]
)
log.info("")
# define data
log.info("split data for cross-validation")
features_train, features_test, targets_train, targets_test =\
sklearn.cross_validation.train_test_split(
dataset.features(),
dataset.targets(),
train_size = 0.7
)
log.info("define classification model")
# define model
#architecture = [10000, 10000, 1000, 10000, 1000]
architecture = [10000, 10000, 10000, 10000, 1000]
#architecture = [50, 150, 250, 300, 400]
classifier = abstraction.Classification(
number_of_classes = 2,
hidden_nodes = architecture,
epochs = 30
)
# train model
log.info("fit classification model to dataset features and targets")
classifier._model.fit(
features_train,
targets_train,
#logdir = "log"
)
# predict and cross-validate training
log.info("test trained classification model on training dataset")
score = sklearn.metrics.accuracy_score(
classifier._model.predict(features_train),
targets_train
)
log.info("prediction accuracy on training dataset: {percentage}".format(
percentage = 100 * score
))
log.info("accuracy of classifier on test dataset:")
score = sklearn.metrics.accuracy_score(
classifier._model.predict(features_test),
targets_test
)
log.info("prediction accuracy on test dataset: {percentage}".format(
percentage = 100 * score
))
classifier.save(
directory = "abstraction_classifier_ttH_ttbb_{architecture}".format(
architecture = str(architecture).replace(" ", "_")
)
)
log.info("")
program.terminate()
def main(options):
global program
program = propyte.Program(
options = options,
name = name,
version = version,
logo = logo
)
global log
from propyte import log
# access options and arguments
input_image_filename = options["--inputimage"]
output_pixels_filename = options["--outputpixels"]
input_pixels_filename = options["--inputpixels"]
output_image_filename = options["--outputimage"]
output_image_width = int(options["--outputimagewidth"])
mode_convert_image_to_pixels = bool(options["--convertimagetopixels"])
mode_convert_pixels_to_NL_format = bool(options["--convertpixelstoNLformat"])
mode_validate_NL_pixels = bool(options["--validateNLpixels"])
mode_validate_pixels = bool(options["--validatepixels"])
mode_convert_pixels_to_image = bool(options["--convertpixelstoimage"])
mode_convert_NL_pixels_to_image = bool(options["--convertNLpixelstoimage"])
log.info("")
if mode_convert_image_to_pixels is True:
log.info("convert image to pixels")
# Access the input image.
log.info("access input image")
input_image = PIL.Image.open(input_image_filename)
log.info("input image mode: {image_mode}".format(image_mode = image.mode))
log.info("input image size: {image_size}".format(image_size = image.size))
pixels = list(input_image.getdata())
pixels_text = str(pixels)
# Create and save the output pixels.
output_pixels_file = open(output_pixels_filename, "w")
output_pixels_file.truncate()
log.info("save output pixels {output_pixels_filename}".format(
output_pixels_filename = output_pixels_filename
))
output_pixels_file.write(pixels_text)
output_pixels_file.close()
elif mode_convert_pixels_to_NL_format is True:
log.info("convert pixels to pixels with newlines")
# Access input pixels.
log.info("access input pixels")
with open(input_pixels_filename) as input_pixels_file:
text = input_pixels_file.read()
# Add newlines.
pixels_text = text.replace("), (", "),\n(")
# Save the pixels.
output_pixels_file = open(output_pixels_filename, "w")
output_pixels_file.truncate()
log.info("save output pixels {output_pixels_filename}".format(
output_pixels_filename = output_pixels_filename
))
output_pixels_file.write(pixels_text)
output_pixels_file.close()
elif mode_validate_NL_pixels:
log.info("validate pixels with newlines")
# Access input pixels.
log.info("access input pixels")
with open(input_pixels_filename) as input_pixels_file:
text = input_pixels_file.read()
parts = text.split("\n")
log.info("replace invalidate pixels")
for n, part in enumerate(parts):
# Create a temporary part for regex examination.
tmp_part = part.strip("),").strip("(")
if not re.match(r"^\d+, \d+, \d+, \d+$", tmp_part):
log.info("tuple {tuple_index} malformed: {tuple} -- replacing with (0, 0, 0, 255)".format(
tuple_index = n,
tuple = tmp_part
))
parts[n] = "(0, 0, 0, 255)"
# Remove trailing commas.
parts = [part.strip(",") for part in parts]
pixels_text = str(parts)
pixels_text = pixels_text.replace(")', '(", ")',\n'(")
# Save the pixels.
output_pixels_file = open(output_pixels_filename, "w")
output_pixels_file.truncate()
log.info("save output pixels {output_pixels_filename}".format(
output_pixels_filename = output_pixels_filename
))
output_pixels_file.write(pixels_text)
output_pixels_file.close()
elif mode_validate_pixels is True:
log.info("validate pixels")
# Access input pixels.
log.info("access input pixels")
with open(input_pixels_filename) as input_pixels_file:
text = input_pixels_file.read()
parts = text[2:-2].split("), (")
log.info("validate pixels")
for n, part in enumerate(parts):
if not re.match(r"^\d+, \d+, \d+, \d+$", part):
print("tuple {tuple_index} malformed: {tuple}".format(
tuple_index = n,
tuple = part
))
elif mode_convert_pixels_to_image is True:
log.info("convert pixels to image")
# Access input pixels.
log.info("access input pixels")
input_pixels_file = open(input_pixels_filename)
pixels = input_pixels_file.read()
pixels = ast.literal_eval(pixels)
pixels = pixels[:10000]
# Determine the image height by determining the maximum number of image
# widths that are possible with the available pixel data.
log.info("determine output image dimensions")
image_mode = "RGBA"
image_width = output_image_width # e.g. 2379
image_height = int(len(pixels)/image_width) # e.g. 2196
image_size = (image_width, image_height)
print type(image_size)
print("output image mode: {image_mode}".format(image_mode = image_mode))
print("output image size: {image_size}".format(image_size = image_size))
pixels = [pixel.replace("))", ")").replace("((", "(").replace("),)", "), ") for pixel in pixels]
# Convert list of pixel strings to list of pixel tuples.
pixels = [ast.literal_eval(re.sub(r'\b0+\B', '', pixel)) for pixel in pixels]
# Use only the number of pixels to make the image of the defined
# dimensions.
number_of_pixels = image_width * image_height
pixels = pixels[:number_of_pixels]
# Create and save the output image.
log.info("create output image")
#output_image_file = Image.new(image_mode, image_size)
output_image_file = Image.new("RGBA", (2379, 2069))
for pixel in pixels:
if str(type(pixel)) is not "<type 'tuple'>":
print "not tuple ", pixel
output_image_file.putdata(pixels)
log.info("save output image {output_image_filename}".format(
output_image_filename = output_image_filename
))
output_image_file.save(output_image_filename)
elif mode_convert_NL_pixels_to_image is True:
log.info("convert pixels to image")
# Access input pixels.
log.info("access input pixels")
input_pixels_file = open(input_pixels_filename)
pixels = input_pixels_file.read()
pixels = ast.literal_eval(pixels)
#pixels = pixels[:10000]
# Determine the image height by determining the maximum number of image
# widths that are possible with the available pixel data.
log.info("determine output image dimensions")
image_mode = "RGBA"
image_width = output_image_width # e.g. 2379
image_height = int(len(pixels)/image_width) # e.g. 2196
image_size = (image_width, image_height)
print type(image_size)
print("output image mode: {image_mode}".format(image_mode = image_mode))
print("output image size: {image_size}".format(image_size = image_size))
pixels = [pixel.replace("))", ")").replace("((", "(").replace("),)", "), ") for pixel in pixels]
# Convert list of pixel strings to list of pixel tuples.
pixels = [ast.literal_eval(re.sub(r'\b0+\B', '', pixel)) for pixel in pixels]
# Use only the number of pixels to make the image of the defined
# dimensions.
number_of_pixels = image_width * image_height
pixels = pixels[:number_of_pixels]
# Create and save the output image.
log.info("create output image")
#output_image_file = Image.new(image_mode, image_size)
output_image_file = Image.new("RGBA", (2379, 2069))
log.info("number of pixels: {number_of_pixels}".format(
number_of_pixels = len(pixels))
)
#for pixel in pixels:
# if str(type(pixel)) is not "<type 'tuple'>":
# print("not tuple: {pixel}".format(pixel = pixel))
pixels = pixels[:4922151]
output_image_file.putdata(pixels)
log.info("save output image {output_image_filename}".format(
output_image_filename = output_image_filename
))
output_image_file.save(output_image_filename)
else:
log.info("no operation selected\n")
print(__doc__)
log.info("")
program.terminate()
def main(options):
global program
program = propyte.Program(
options = options,
name = name,
version = version,
logo = logo
)
global log
from propyte import log
host = "127.0.0.1" # 10.0.0.41
port_number = 2718
filename_peers = options["--peersfile"]
log.info("")
# Read the local peers list.
if not os.path.exists(filename_peers):
log.error("file {filename} not found".format(
filename = filename_peers
))
program.terminate()
peers_list_local = [line.rstrip("\n") for line in open(filename_peers)]
peers_consensus = shijian.List_Consensus()
peers_consensus.append(tuple(peers_list_local))
log.debug("peers list local: {peers}".format(
peers = peers_list_local
))
log.debug("peers list consensus: {peers}".format(
peers = peers_consensus.consensus()
))
port = int(str(port_number), 10)
address_remote = (host, port)
# Create a datagram socket for UDP.
socket_UDP = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
# Set the socket to be reusable.
socket_UDP.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# Set the socket to accept incoming broadcasts.
socket_UDP.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
# Disengage socket blocking.
socket_UDP.setblocking(False)
# Set the socket to accept connections on the port.
socket_UDP.bind(("", port))
# Communicate data in a loop.
log.info("Ctrl c to exit")
while True:
# receive
try:
# buffer size: 8192
message, address = socket_UDP.recvfrom(8192)
message = message.rstrip("\n")
if message:
log.debug("{address}:{port_number}> {message}".format(
address = address[0],
port_number = port_number,
message = message
))
# upcoming: message accept/reject procedure
# Record the sender ID in order to limit the number of senders
# such that scalability problems do not manifest.
# Record the message ID in order to avoid parsing it again.
# If a peers list is detected, parse it and add it to the
# consensus.
if "peers =" in message:
peers_list_remote = eval(message.lstrip("peers =").rstrip(";"))
peers_consensus.append(tuple(peers_list_remote))
if len(peers_consensus) >= 3:
peers_consensus_list = list(peers_consensus.consensus())
log.debug("consensus peers list: {peers_consensus_list}".format(
peers_consensus_list = peers_consensus_list
))
log.debug("update local peers list with consensus peers list")
peers_list_local = peers_consensus_list
# upcoming functionality
# If a heartbeat is detected, send the local peers list.
if "heartbeat" in message:
message_send = "peers = {peers_list_local};".format(
peers_list_local = peers_list_local
)
socket_UDP.sendto(message_send, address)
except:
pass
# send
message_send =\
"message_text = heartbeat; message_ID = {message_ID};".format(
message_ID = shijian.UID()
)
socket_UDP.sendto(message_send, address_remote)
time.sleep(1)
log.info("")
program.terminate()
def main(options):
global program
program = propyte.Program(
options = options,
name = name,
version = version,
logo = logo
)
global log
from propyte import log
log.info("")
# access options and arguments
grid_search_filename = options["--gridsearchfile"]
# load grid search map
grid_search_map = shijian.import_object(filename = grid_search_filename)
number_of_entries = len(grid_search_map["epoch"])
log.info("number of entries: {number_of_entries}".format(
number_of_entries = number_of_entries
))
# table
table_contents = [
["epoch", "architecture", "score training", "score testing"]
]
for index in range(0, number_of_entries):
table_contents.append(
[
str(grid_search_map["epoch"][index]),
str(grid_search_map["hidden_nodes"][index]),
str(grid_search_map["score_training"][index]),
str(grid_search_map["score_test"][index])
]
)
log.info("\ngrid search map:\n")
log.info(
pyprel.Table(
contents = table_contents,
)
)
# parallel coordinates plot
number_of_entries = len(grid_search_map["epoch"])
datasets = []
for index in range(0, number_of_entries):
row = []
architecture_padded = grid_search_map["hidden_nodes"][index] + [0] * (5 - len(grid_search_map["hidden_nodes"][index]))
row.append(grid_search_map["epoch"][index])
row.extend(architecture_padded)
row.append(grid_search_map["score_training"][index])
row.append(grid_search_map["score_test"][index])
datasets.append(row)
datavision.save_parallel_coordinates_matplotlib(
datasets[::-1],
filename = "parallel_coordinates.png"
)
# plot
architectures = shijian.unique_list_elements(grid_search_map["hidden_nodes"])
architecture_epoch_score = {}
for architecture in architectures:
architecture_epoch_score[str(architecture)] = []
for index in range(0, number_of_entries):
if grid_search_map["hidden_nodes"][index] == architecture:
architecture_epoch_score[str(architecture)].append(
[
grid_search_map["epoch"][index],
grid_search_map["score_test"][index]
]
)
figure = matplotlib.pyplot.figure()
figure.set_size_inches(10, 10)
axes = figure.add_subplot(1, 1, 1)
axes.set_xscale("log")
figure.suptitle("hyperparameter map", fontsize = 20)
matplotlib.pyplot.xlabel("epochs")
matplotlib.pyplot.ylabel("training test score")
for key, value in architecture_epoch_score.iteritems():
epochs = [element[0] for element in value]
score_test = [element[1] for element in value]
matplotlib.pyplot.plot(epochs, score_test, label = key)
matplotlib.pyplot.legend(
loc = "center left",
bbox_to_anchor = (1, 0.5),
fontsize = 10
)
matplotlib.pyplot.savefig(
"hyperparameter_map.eps",
bbox_inches = "tight",
format = "eps"
)
# find best-scoring models
# Find the 15 best scores and plot them using parallel coordinates.
best_models = sorted(zip(
grid_search_map["score_test"],
grid_search_map["score_training"],
grid_search_map["hidden_nodes"]),
reverse = True
)[:15]
datasets = []
for model in best_models:
row = []
architecture_padded = model[2] + [0] * (5 - len(model[2]))
row.extend(architecture_padded)
row.append(model[1])
row.append(model[0])
datasets.append(row)
datavision.save_parallel_coordinates_matplotlib(
datasets,
filename = "15_best_models_parallel_coordinates.png"
)
# Find the 3 best scores.
best_models = sorted(zip(
grid_search_map["score_test"],
grid_search_map["hidden_nodes"]),
reverse = True
)[:3]
# table
table_contents = [["architecture", "score testing"]]
for model in best_models:
table_contents.append([str(model[1]), str(model[0])])
log.info("\nbest-scoring models:\n")
log.info(
pyprel.Table(
contents = table_contents,
)
)
log.info("")
program.terminate()
def main(options):
global program
program = propyte.Program(
options = options,
name = name,
version = version,
logo = logo
)
global log
from propyte import log
log.info("")
# access options and arguments
ROOT_filename_ttH = options["--datattH"]
ROOT_filename_ttbb = options["--datattbb"]
engage_plotting = string_to_bool(options["--plot"])
log.info("ttH data file: {filename}".format(
filename = ROOT_filename_ttH
))
log.info("ttbb data file: {filename}".format(
filename = ROOT_filename_ttbb
))
# Access data for event classes ttbb and ttH.
data_ttbb = abstraction.load_HEP_data(
ROOT_filename = ROOT_filename_ttbb,
tree_name = "nominal",
maximum_number_of_events = None
)
data_ttH = abstraction.load_HEP_data(
ROOT_filename = ROOT_filename_ttH,
tree_name = "nominal",
maximum_number_of_events = None
)
if engage_plotting is True:
# Plot the loaded datasets.
for variable_name in data_ttbb.variables():
log.info("plot ttbb versus ttH comparison of {variable_name}".format(
variable_name = variable_name
))
datavision.save_histogram_comparison_matplotlib(
values_1 = data_ttbb.values(name = variable_name),
values_2 = data_ttH.values(name = variable_name),
label_1 = variable_name + "_ttbb",
label_2 = variable_name + "_ttH",
normalize = True,
label_ratio_x = "frequency",
label_y = "",
title = variable_name + "_ttbb_ttH",
filename = variable_name + "_ttbb_ttH.png"
)
# upcoming: consider data ordering
# Preprocess all data (to be updated).
data_ttbb.preprocess_all()
data_ttH.preprocess_all()
# Add class labels to the data sets, 0 for ttbb and 1 for ttH.
for index in data_ttbb.indices():
data_ttbb.variable(index = index, name = "class", value = 0)
for index in data_ttH.indices():
data_ttH.variable(index = index, name = "class", value = 1)
# Convert the data sets to a simple list format with the first column
# containing the class label.
_data = []
for index in data_ttbb.indices():
_data.append([
data_ttbb.variable(index = index, name = "el_1_pt"),
data_ttbb.variable(index = index, name = "el_1_eta"),
data_ttbb.variable(index = index, name = "el_1_phi"),
data_ttbb.variable(index = index, name = "jet_1_pt"),
data_ttbb.variable(index = index, name = "jet_1_eta"),
data_ttbb.variable(index = index, name = "jet_1_phi"),
data_ttbb.variable(index = index, name = "jet_1_e"),
data_ttbb.variable(index = index, name = "jet_2_pt"),
data_ttbb.variable(index = index, name = "jet_2_eta"),
data_ttbb.variable(index = index, name = "jet_2_phi"),
data_ttbb.variable(index = index, name = "jet_2_e"),
data_ttbb.variable(index = index, name = "met"),
data_ttbb.variable(index = index, name = "met_phi"),
data_ttbb.variable(index = index, name = "nJets"),
data_ttbb.variable(index = index, name = "Centrality_all"),
#data_ttbb.variable(index = index, name = "Mbb_MindR")
])
_data.append([
data_ttbb.variable(name = "class")
])
for index in data_ttH.indices():
_data.append([
data_ttH.variable(index = index, name = "el_1_pt"),
data_ttH.variable(index = index, name = "el_1_eta"),
data_ttH.variable(index = index, name = "el_1_phi"),
data_ttH.variable(index = index, name = "jet_1_pt"),
data_ttH.variable(index = index, name = "jet_1_eta"),
data_ttH.variable(index = index, name = "jet_1_phi"),
data_ttH.variable(index = index, name = "jet_1_e"),
data_ttH.variable(index = index, name = "jet_2_pt"),
data_ttH.variable(index = index, name = "jet_2_eta"),
data_ttH.variable(index = index, name = "jet_2_phi"),
data_ttH.variable(index = index, name = "jet_2_e"),
data_ttH.variable(index = index, name = "met"),
data_ttH.variable(index = index, name = "met_phi"),
data_ttH.variable(index = index, name = "nJets"),
data_ttH.variable(index = index, name = "Centrality_all"),
#data_ttH.variable(index = index, name = "Mbb_MindR")
])
_data.append([
data_ttH.variable(name = "class")
])
dataset = abstraction.Dataset(data = _data)
log.info("")
# define data
log.info("split data for cross-validation")
features_train, features_test, targets_train, targets_test =\
cross_validation.train_test_split(
dataset.features(),
dataset.targets(),
train_size = 0.7
)
# grid search
import itertools
epochs = [100, 100000]
architecture = [200, 300, 300, 200]
grid_search_map = {}
grid_search_map["epoch"] = []
grid_search_map["hidden_nodes"] = []
grid_search_map["score_training"] = []
grid_search_map["score_test"] = []
# define progress
count_total = 0
for epoch in epochs:
for nodes_count in xrange(1, len(architecture) + 1):
combinations = itertools.product(architecture, repeat = nodes_count)
for combination in combinations:
count_total += 1
count = 0
progress = shijian.Progress()
progress.engage_quick_calculation_mode()
for epoch in epochs:
for nodes_count in xrange(1, len(architecture) + 1):
combinations = itertools.product(architecture, repeat = nodes_count)
for combination in combinations:
hidden_nodes = list(combination)
# define model
log.info("define classification model")
classifier = abstraction.Classification(
number_of_classes = 2,
hidden_nodes = hidden_nodes,
epochs = epoch
)
# train model
log.info("fit model to dataset features and targets")
classifier._model.fit(features_train, targets_train)
#classifier.save()
# predict and cross-validate training
log.info("test trained model on training dataset")
score_training = metrics.accuracy_score(
classifier._model.predict(features_train),
targets_train
)
score_test = metrics.accuracy_score(
classifier._model.predict(features_test),
targets_test
)
log.info("\ntraining-testing instance complete:")
log.info("epoch: {epoch}".format(
epoch = epoch
))
log.info("architecture: {architecture}".format(
architecture = hidden_nodes
))
log.info("score training: {score_training}".format(
score_training = 100 * score_training
))
log.info("score test: {score_test}".format(
score_test = 100 * score_test
))
pyprel.print_line()
grid_search_map["epoch"].append(epoch)
grid_search_map["hidden_nodes"].append(hidden_nodes)
grid_search_map["score_training"].append(score_training)
grid_search_map["score_test"].append(score_test)
# save current grid search map
shijian.export_object(
grid_search_map,
filename = "grid_search_map.pkl",
overwrite = True
)
count += 1
print(progress.add_datum(fraction = (count + 1) / count_total))
number_of_entries = len(grid_search_map["epoch"])
# table
table_contents = [
["epoch", "architecture", "score training", "score testing"]
]
for index in range(0, number_of_entries):
table_contents.append(
[
str(grid_search_map["epoch"][index]),
str(grid_search_map["hidden_nodes"][index]),
str(grid_search_map["score_training"][index]),
str(grid_search_map["score_test"][index])
]
)
print("\ngrid search map:\n")
print(
pyprel.Table(
contents = table_contents,
)
)
# plot
architectures = shijian.unique_list_elements(grid_search_map["hidden_nodes"])
architecture_epoch_score = {}
for architecture in architectures:
architecture_epoch_score[str(architecture)] = []
for index in range(0, number_of_entries):
if grid_search_map["hidden_nodes"][index] == architecture:
architecture_epoch_score[str(architecture)].append(
[
grid_search_map["epoch"][index],
grid_search_map["score_test"][index]
]
)
figure = matplotlib.pyplot.figure()
figure.set_size_inches(10, 10)
axes = figure.add_subplot(1, 1, 1)
axes.set_xscale("log")
figure.suptitle("hyperparameter map", fontsize = 20)
matplotlib.pyplot.xlabel("epochs")
matplotlib.pyplot.ylabel("training test score")
for key, value in architecture_epoch_score.iteritems():
epochs = [element[0] for element in value]
score_test = [element[1] for element in value]
matplotlib.pyplot.plot(epochs, score_test, label = key)
matplotlib.pyplot.legend(loc = "center right")
matplotlib.pyplot.savefig(
"hyperparameter_map.eps",
bbox_inches = "tight",
format = "eps"
)
# find best-scoring models
# Find the 3 best scores.
best_models = sorted(zip(
grid_search_map["score_test"],
grid_search_map["hidden_nodes"]),
reverse = True
)[:3]
# table
table_contents = [["architecture", "score testing"]]
for model in best_models:
table_contents.append([str(model[1]), str(model[0])])
print("\nbest-scoring models:\n")
print(
pyprel.Table(
contents = table_contents,
)
)
log.info("")
program.terminate()
def main(options):
global program
program = propyte.Program(options=options,
name=name,
version=version,
logo=logo)
global log
from propyte import log
# access options and arguments
input_image_filename = options["--inputimage"]
output_pixels_filename = options["--outputpixels"]
input_pixels_filename = options["--inputpixels"]
output_image_filename = options["--outputimage"]
output_image_width = int(options["--outputimagewidth"])
mode_convert_image_to_pixels = bool(options["--convertimagetopixels"])
mode_validate_pixels = bool(options["--validatepixels"])
mode_convert_pixels_to_image = bool(options["--convertPixelsToImage"])
log.info("")
if mode_convert_image_to_pixels is True:
log.info("convert image to pixels")
# Access the input image.
log.info("access input image")
input_image = PIL.Image.open(input_image_filename)
log.info(
"input image mode: {image_mode}".format(image_mode=image.mode))
log.info(
"input image size: {image_size}".format(image_size=image.size))
pixels = list(input_image.getdata())
pixels_text = str(pixels)
# Create and save the output pixels.
output_pixels_file = open(output_pixels_filename, "w")
output_pixels_file.truncate()
log.info("save output pixels {output_pixels_filename}".format(
output_pixels_filename=output_pixels_filename))
output_pixels_file.write(pixels_text)
output_pixels_file.close()
elif mode_validate_pixels is True:
log.info("validate pixels")
# Access input pixels.
log.info("access input pixels")
with open(input_pixels_filename) as input_pixels_file:
text = input_pixels_file.read()
parts = text[2:-2].split("), (")
log.info("validate pixels")
for n, part in enumerate(parts):
if not re.match(r"^\d+, \d+, \d+, \d+$", part):
print("tuple {tuple_index} malformed: {tuple}".format(
tuple_index=n, tuple=part))
elif mode_convert_pixels_to_image is True:
log.info("convert pixels to image")
# Access input pixels.
log.info("access input pixels")
input_pixels_file = open(input_pixels_filename)
pixels = input_pixels_file.read()
pixels = ast.literal_eval(pixels)
# Determine the image height by determining the maximum number of image
# widths are possible with the available pixel data.
log.info("determine output image dimensions")
image_mode = "RGBA"
image_width = output_image_width # e.g. 2379
image_height = int(len(pixels) / image_width) # e.g. 2196
image_size = (image_width, image_height)
print("output image mode: {image_mode}".format(image_mode=image_mode))
print("output image size: {image_size}".format(image_size=image_size))
# Create and save the output image.
log.info("create output image")
output_image_file = Image.new(image_width, image_height)
output_image_file.putdata(pixels)
log.info("save output image {output_image_filename}".format(
output_image_filename=output_image_filename))
output_image_file.save(output_image_filename)
else:
log.info("no operation selected\n")
print(__doc__)
log.info("")
program.terminate()
def main(options):
global program
program = propyte.Program(options=options,
name=name,
version=version,
logo=logo)
global log
from propyte import log
# access options and arguments
expression = options["--expression"]
word_vector_model = options["--wordvectormodel"]
# Define a dictionary of natural language expressions and word vectors.
stored_expressions = {
"This is a test.":
numpy.array([
-0.3828682, -0.36397889, 0.46676171, 0.32530552, 0.20376287,
-0.41326976, -0.58228827, 0.05073506, -0.29834735, 0.62523258,
0.48247468, 0.63565594, 0.61466146, -0.05790123, 0.49383548,
0.17871667, 0.26640224, -0.05172781, -0.43991241, 0.8027305,
0.13174312, -0.70332521, -0.56575418, -0.21705133, -0.93002945,
0.04151381, -0.15113404, 0.06264834, 0.03022593, -0.00822711,
-0.23755306, -0.9215641, 0.21348992, 0.38396335, 0.3020944,
-0.08034055, -0.36891997, -0.86551458, -1.02402425, 0.03633916,
0.34436008, 0.43058148, -0.32728755, 0.50974292, -0.31518513,
-0.63085675, -0.40564051, 0.30009648, -0.06426927, -0.6588546,
0.06724164, 0.08611558, -0.13476974, 0.43107161, -0.26038069,
0.03187743, 0.05931987, 0.28155532, 0.3636784, -0.76867509,
-0.2253349, -0.77433741, 0.01924273, 0.63751495, 0.03874384,
0.28651205, 0.14867969, -0.2256701, 0.23747981, 0.12383705,
0.27097231, -0.06902695, 0.06664967, 0.05863822, -0.06882346,
0.59539717, 0.08472043, -0.13579898, -0.31311297, -0.68136102,
0.33296993, 0.26578408, -0.55723149, 0.38583612, -0.18033087,
-0.50730389, 0.39173275, 0.57567608, -0.42063141, 0.22387385,
0.473548, 0.41959459, 0.34881225, 0.1939103, -0.54997987,
0.30737191, -0.6659264, 0.0437102, -0.11230323, -0.13493723
],
dtype=numpy.float32),
"All those moments will be lost in time.":
numpy.array([
-1.19203818e+00, -2.22961619e-01, 6.69643760e-01, 3.70975524e-01,
-6.15832031e-01, -4.36573088e-01, -6.77924156e-01, 6.26985192e-01,
1.36510044e-01, 1.09196387e-01, 7.61598766e-01, 7.17226386e-01,
-1.08178332e-01, -1.00655735e+00, 7.45964348e-01, 1.64966106e-01,
5.85332870e-01, -3.83911550e-01, -6.85201228e-01, 1.31213856e+00,
8.04567218e-01, -1.28810382e+00, -2.52677381e-01, -9.27993536e-01,
-4.17307138e-01, -4.56952095e-01, -7.27599859e-01, 7.54008472e-01,
6.67124987e-04, 2.75971144e-01, 2.75658131e-01, -6.79417193e-01,
-1.73686996e-01, 8.78942013e-01, 4.39480424e-01, -6.37802243e-01,
-6.99860230e-02, -7.99779966e-02, -7.58146644e-02, 8.09784770e-01,
-3.71645451e-01, 1.04973994e-01, -1.34749603e+00, 2.96185315e-01,
5.85593104e-01, -1.40544206e-01, -3.77467513e-01, 3.46597135e-01,
2.56733745e-01, 4.04421866e-01, 1.57907709e-01, 3.00843865e-01,
-5.41967154e-01, 5.51929235e-01, -1.69145897e-01, 4.42785203e-01,
-2.69805342e-02, 1.31654418e+00, 3.19460958e-01, 5.08862257e-01,
3.44371676e-01, -6.95496798e-01, 4.88163918e-01, 2.55316138e-01,
5.03436685e-01, 9.24195647e-02, -2.38671958e-01, -8.97032142e-01,
-3.73697281e-03, 2.99875826e-01, 1.65674359e-01, 2.01489821e-01,
1.58179402e-02, 1.30668238e-01, -1.56954467e-01, -2.88258016e-01,
6.76668346e-01, -3.77742261e-01, 2.20978767e-01, -6.34561360e-01,
8.33457410e-01, -2.13193640e-01, -6.35235757e-02, 1.89480215e-01,
6.02166615e-02, -6.64785147e-01, 1.07347333e+00, 6.22629285e-01,
-4.63467717e-01, -1.13483839e-01, 3.43968630e-01, 2.75979757e-01,
-1.28710240e-01, 1.50670230e+00, -3.10248852e-01, 3.29222828e-01,
1.64443821e-01, -7.78683364e-01, -9.80837345e-02, -1.07415296e-01
],
dtype=numpy.float32),
"All those moments were lost in time.":
numpy.array([
-0.94025505, -0.45476836, 0.41891485, 1.06683254, -0.49607083,
-0.60043317, -0.55656326, 0.05368682, 0.20896676, 0.19261286,
0.51067233, 0.01298623, -0.67276001, -0.51130211, 0.61433661,
0.03579944, 0.4515644, -0.19222273, -0.3919456, 0.65209424,
0.98329031, -0.78390068, -0.0611292, -0.88086104, 0.25153416,
-0.16051427, -0.33223695, 0.86147106, -0.19569418, -0.21456225,
0.27583197, -0.65764415, -0.76533222, 0.78306556, 0.84534264,
-0.26408321, 0.04312199, -0.00636051, 0.1322974, 0.72321951,
-0.01186696, 0.40505514, -0.87730938, 0.58147532, 0.89738142,
-0.16748536, -0.38406748, -0.12007161, 0.49123141, 0.48998365,
0.15616624, 0.52637529, -0.66329396, 0.10376941, -0.33025965,
0.04188792, 0.30536407, 0.38240519, 0.01627355, 1.23012972,
0.46352714, -0.74617827, 0.43505573, -0.16246299, 0.34668511,
-0.02247265, -0.34742412, -0.64483654, -0.2243523, 0.04222834,
0.42057285, 0.22310457, 0.36833102, -0.05716853, -0.44688487,
-0.51298815, 0.61859602, -0.21154809, -0.08168469, -0.15004104,
0.21371906, 0.21713886, 0.21935812, 0.04912762, 0.02854752,
-0.55747426, 0.70036995, 0.20306921, -0.46556181, -0.10637223,
0.60909081, 0.55366743, -0.22907487, 1.13089538, 0.34430629,
0.35133895, 0.085365, -0.58662325, -0.13062993, -0.04200239
],
dtype=numpy.float32),
"All those moments are lost in time.":
numpy.array([
-0.78943789, -0.30322614, 0.3780162, 0.80896467, -0.42042252,
-0.64176518, -0.51211309, -0.1537444, -0.04233316, 0.07710438,
0.66949254, 0.37771451, -0.74869132, -0.55132926, 0.53695548,
-0.11229508, 0.6673997, -0.34724045, -0.42173663, 0.7451877,
1.01433206, -0.85418928, -0.31583607, -0.6812892, 0.42722669,
-0.43322188, -0.35293943, 0.7662127, -0.30090365, -0.13694993,
-0.04172039, -0.65059775, -0.62617165, 0.71341687, 0.82349646,
-0.31194365, 0.00356466, -0.32218212, 0.15857732, 0.82880032,
0.0566355, 0.43106011, -1.01921201, 0.51658779, 0.8068108,
-0.09396499, -0.37920368, -0.08726061, 0.29975161, 0.25999272,
0.23571083, 0.24800834, -0.73045135, 0.19150458, -0.19696848,
-0.11186107, 0.1336731, 0.33246318, 0.22474274, 1.15420532,
0.39482915, -0.70385826, 0.54841375, -0.03638301, 0.54499787,
0.02484709, -0.2070619, -0.69282937, -0.21465099, 0.11578664,
0.22713676, 0.21237181, 0.2007356, 0.14489903, -0.37357002,
-0.50091666, 0.59818357, -0.36113665, 0.06037673, -0.26377741,
0.31544513, -0.23714744, -0.01429842, 0.17592101, -0.16280818,
-0.58340323, 0.63590413, 0.31803992, -0.47035503, -0.17544734,
0.66008455, 0.77849454, -0.04235193, 1.29202402, 0.12573826,
0.20377615, -0.08164676, -0.41151166, -0.1280518, 0.02905136
],
dtype=numpy.float32),
}
model_word2vec = abstraction.load_word_vector_model(
filename=word_vector_model)
working_expression_NL = expression
# Convert the expression to a word vector.
working_expression_WV =\
abstraction.convert_sentence_string_to_word_vector(
sentence_string = working_expression_NL,
model_word2vec = model_word2vec
)
log.info(
"word vector representation of expression \"{working_expression_NL}\":"
"\n{working_expression_WV}".format(
working_expression_NL=working_expression_NL,
working_expression_WV=working_expression_WV))
# Define table headings.
table_contents = [[
"working expression natural language",
"stored expression natural language",
"absolute magnitude difference between working amd stored expression "
"word vectors",
"angle between working and stored expression word vectors"
]]
# Compare the expression word vector representation to existing word
# vectors.
magnitude_differences = []
angles = []
stored_expressions_NL_list = []
magnitude_working_expression_WV = datavision.magnitude(
working_expression_WV)
for stored_expression_NL in stored_expressions:
stored_expression_WV = stored_expressions[stored_expression_NL]
magnitude_stored_expression_WV = datavision.magnitude(
stored_expression_WV)
magnitude_difference_working_expression_WV_stored_expression_WV = abs(
magnitude_working_expression_WV - magnitude_stored_expression_WV)
angle_working_expression_WV_stored_expression_WV = datavision.angle(
working_expression_WV, stored_expression_WV)
# Store comparison results in lists.
magnitude_differences.append(
magnitude_difference_working_expression_WV_stored_expression_WV)
angles.append(angle_working_expression_WV_stored_expression_WV)
stored_expressions_NL_list.append(stored_expression_NL)
# Build table.
table_contents.append([
str(working_expression_NL),
str(stored_expression_NL),
str(magnitude_difference_working_expression_WV_stored_expression_WV
),
str(angle_working_expression_WV_stored_expression_WV)
])
# Record table.
print(pyprel.Table(contents=table_contents))
log.info("")
index_minimum_magnitude_differences =\
magnitude_differences.index(min(magnitude_differences))
index_minimum_angles = angles.index(min(angles))
index_minimum_match_width = len(angles) / 4
if abs(index_minimum_magnitude_differences -
index_minimum_angles) < index_minimum_match_width:
log.info("translation: {translation_expression_NL}".format(
translation_expression_NL =\
stored_expressions_NL_list[index_minimum_angles]
))
else:
log.error("unable to translate")
log.info("")
program.terminate()
def function_1():
log.info("log message of function1")
