def random(self, N):
CLOSED = []
CLOSED.append(self.boardArr)
cars = self.getCarsFromBoard()
cloneBoard = Board()
#for i in range(6):
#for j in range(6):
#bitch.boardArr[i][j] = self.boardArr[i][j]
cloneBoard.clone(self)
while True:
if cloneBoard.isDone() == True:
misc.printCLOSED(CLOSED)
return
if len(CLOSED) == N:
misc.printCLOSED(CLOSED)
return
nextBoards = cloneBoard.next()
boardIndex = randint(0,len(nextBoards)-1)
nextBoard = nextBoards[boardIndex]
if nextBoard not in CLOSED:
CLOSED.append(nextBoard)
cloneBoard.boardArr = nextBoard
# The Code commented out underneath is for me to remember what an idiot i am.
'''
def ExtractPlotScores(classification_file, score_file):
# Determines the separators for both files.
sep1 = mf.determine_separator(classification_file)
sep2 = mf.determine_separator(score_file)
# Creates dataframes from both files.
pathogenicity_dataframe = pd.read_csv(classification_file, sep=sep1, header=0)
score_dataframe = pd.read_csv(score_file, sep=sep2, header=0)
# Inquires about which column in the curated file contains pathogenicity data. Sets the variant
# identifier to the first column.
mf.print_columns_with_index(pathogenicity_dataframe)
pathogenicity_index = mf.get_int_answer('What column states the pathogenicity? ')
pathogenicity_column = list(pathogenicity_dataframe.columns)[pathogenicity_index - 1]
coordinate_index_1 = mf.get_int_answer('What column identifies the variant? ')
genomic_coordinate_column1 = list(pathogenicity_dataframe.columns)[coordinate_index_1 - 1]
# Inquires about which column in the output file of the predictor method contains pathogenicity
# scores. Sets the variant identifier to the first column, which must match a value in the first
# column of the curated file.
mf.print_columns_with_index(score_dataframe)
score_index = mf.get_int_answer('What column gives the scores? ')
score_column = list(score_dataframe.columns)[score_index - 1]
coordinate_index_2 = mf.get_int_answer('What column identifies the variant? ')
genomic_coordinate_column2 = list(score_dataframe.columns)[coordinate_index_2 - 1]
# Creates a dictionary to store each variant's pathogenicity and pathogenicity score. Creates
# a list that stores each variant identifier associated with a reported score.
scores = {'pathogenicity': [], 'scores': []}
chromosome_list = []
for index, row in score_dataframe.iterrows(): # Iterates through each variant in the output file.
# Determines if the given variant in the outfile has a score. If so, the variable
# 'after' contains the variant's score. Otherwise, 'after' is empty.
to_parse = score_dataframe[score_column].iloc[index]
phrase = 'REVEL_score='
before, at, after = to_parse.partition(phrase)
if ( after != ''): # If the variant reports a score.
# And the variant has no previously reported score.
if ( score_dataframe[genomic_coordinate_column2].iloc[index] not in chromosome_list ):
# Add the variant to the list of scored variants. Then append the score to the list
# of scores and append the variants pathogenicity to the list of pathogencities.
chromosome_list.append(score_dataframe[genomic_coordinate_column2].iloc[index])
scores['scores'].append(float(after))
genomic_coordinate = chromosome_list[-1]
pathogenicity = pathogenicity_dataframe[pathogenicity_dataframe[genomic_coordinate_column1] == genomic_coordinate][pathogenicity_column]
if ( (pathogenicity.values[0] == 'Benign') | (pathogenicity.values[0] == 'Likely_benign') ):
scores['pathogenicity'].append("Benign")
elif ( (pathogenicity.values[0] == 'Pathogenic') | (pathogenicity.values[0] == 'Likely_pathogenic') ):
scores['pathogenicity'].append('Pathogenic')
# Construct a dataframe from the dictionary, and then we create a boxplot from the data.
df = pd.DataFrame.from_dict(scores)
if ( len(sys.argv) == 4 ):
df.to_csv(output_file)
print('variants scored:', df.shape[0], '/', pathogenicity_dataframe.shape[0])
# , '/', len((pathogenicity_dataframe.shape())[1])
boxplt = df.boxplot(by='pathogenicity')
plt.show()
def minimax_pruning(self):
self.player1 = Player.RandomPlayer(self.board.boardString, 'x')
self.player2 = Player.MinimaxAlphaBetaPlayer(self.board.boardString,
'y')
CLOSED = []
CLOSED.append(self.board)
cloneBoard = Board.Board()
cloneBoard.clone(self.board)
while True:
w = self.board.winner()
if w != None:
Misc.printCLOSED(CLOSED)
print w
return w
if self.activeTurn == 'x':
self.board.createBoard(self.player1.random().stringifyBoard())
self.changeTurns()
elif self.activeTurn == 'y':
self.board.createBoard(
self.player2.minimaxDecisionABPruning().stringifyBoard())
self.changeTurns()
CLOSED.append(self.board.boardArr)
self.board.printBoard()
self.player1.modifyBoard(self.board.stringifyBoard())
self.player2.modifyBoard(self.board.stringifyBoard())
async def del_toon(self, ctx, *, msg):
if MiscFunctions.role_name_has_access(
admin_role_names,
ctx.author.roles) or MiscFunctions.role_id_has_access(
admin_role_ids, ctx.author.roles):
if ctx.message.mentions:
num = User.toon_delete_for_user(str(
ctx.message.mentions[0].id))
if num > 0:
await ctx.message.channel.send(
"Deleted: " + str(num) + " characters for: <@!" +
str(ctx.message.mentions[0].id) + ">")
else:
await ctx.message.channel.send(
"Could not find/delete any characters for: <@!" +
str(ctx.message.mentions[0].id) + ">")
else:
if User.toon_delete(msg.strip()) > 0:
await ctx.message.channel.send("Character: \"" +
msg.strip() +
"\" Deleted.")
else:
await ctx.message.channel.send(
"Character: \"" + msg.strip() +
"\" could not be found/deleted.")
def __init__(self, boardArr, carCh): # takes the board and the car character
if misc.isCarInBoard(boardArr, carCh) == False:
print 'Car Constructer __init__() ', CarNotIn
return
self.carChar = carCh
self.allCoordinates = misc.findAll(boardArr, carCh)
self.end1 = self.allCoordinates[0] # end1 comes before end2
self.end2 = self.allCoordinates[len(self.allCoordinates)-1]
self.length = len(self.allCoordinates)
if self.end1[1] == self.end2[1]: # checking for orientation # if y coordinate is same
self.orientation = 'vertical'
elif self.end1[0] == self.end2[0]:
self.orientation = 'horizontal'
def ExtractPlotScores(classification_file, score_file):
# Separators for both file types.
sep1 = ','
sep2 = '\t'
# Creates two dataframes, one to hold the classifications, the other to hold the scores.
pathogenicity_dataframe = pd.read_csv(classification_file, sep=sep1, header=0)
score_dataframe = pd.read_csv(score_file, sep=sep2, header=0)
# Prepares the dataframes for merging, then merges them.
ID_INFO_dataframe = score_dataframe[['#Uploaded_variation', 'Extra']]
ID_INFO_dataframe = ID_INFO_dataframe.rename(index=str, columns={'#Uploaded_variation' : 'ID', 'Extra' : 'Extra'})
pathogenicity_dataframe = pathogenicity_dataframe.rename(index=str, columns={'#Coordinate' : 'ID', 'Pathogenicity' : 'Pathogenicity'})
merged = pathogenicity_dataframe.merge(ID_INFO_dataframe, on='ID')
# Strips the INFO field of the merged data from of everything but the scores of the variants
# scored by EA. The field is then renamed to "Score".
for index,row in merged.iterrows():
to_check = row.values[-1].split(',')[-1]
phrase = 'REVEL_score='
before, at, after = to_check.partition(phrase)
if ( mf.check_float(after) ):
merged.iloc[index, 2] = float(after)
merged = merged.loc[ ((merged['Extra'].apply(lambda x: len(str(x)))) <= 5) ]
merged = merged.rename(index=str, columns={'ID': 'Variant', 'Pathogenicity' : 'Pathogenicity', 'Extra' : 'Score'})
merged = merged.drop_duplicates()
# Construct a dataframe from the dictionary, and then we create a boxplot from the data.
#df = pd.DataFrame.from_dict(scores)
if ( len(sys.argv) == 4 ):
merged.to_csv(sys.argv[3], index=False, index_label=False)
def ExtractTestData(read_file, write_file):
# Creates a dataframe from read_file and opens a new file for writing.
df = pd.read_csv(read_file, sep='\t', header=0)
write_file = open(write_file, 'w+')
# Asks the user if they want to add a column for pathogenicity in write_file.
extract_pathogenicity = mf.get_yes_no("Extract the pathogenicity (y/n)? ")
answer_yes = (extract_pathogenicity == 'y') | (extract_pathogenicity
== 'Y')
# Adds the column names.
write_file.write('#Variant')
if (answer_yes):
write_file.write(',Pathogenicity')
write_file.write('\n')
# Adds all the variants that are single subsitutions and are not VUS's to write_file. Also,
# adds the pathogenicity of the variant if the user replied yes.
for index, row in df.iterrows():
if (('>' in row.values[1]) & (not int(row.values[-1]) == 3)):
if (row.values[1] == 'BRCA1'):
write_file.write('NM_007294.3:' + str(row.values[1]))
else:
write_file.write('NM_000059.3:' + str(row.values[1]))
if (answer_yes):
if (int(row.values[-1]) > 3):
write_file.write(',' + 'Pathogenic')
elif (int(row.values[-1]) < 3):
write_file.write(',' + 'Benign')
write_file.write('\n')
write_file.close()
async def toon(self, ctx, *, msg):
if not ctx.message.mentions:
user_toon = msg.strip()
user = ctx.message.author.id
if "," in user_toon:
toons = user_toon.split(",")
for x in toons:
User.toon_add(user, ctx.message.guild.id, x.strip(),
timestamp())
await ctx.message.channel.send(
ctx.message.author.display_name +
", I have added characters: " + user_toon +
" to your profile.")
else:
User.toon_add(user, ctx.message.guild.id, user_toon,
timestamp())
await ctx.message.channel.send(
ctx.message.author.display_name +
", I have added character: " + user_toon +
" to your profile.")
else:
user = str(ctx.message.mentions[0].id)
user_toon = msg.split(' ', 1)[1].strip()
if MiscFunctions.role_name_has_access(admin_role_names, ctx.author.roles) or \
MiscFunctions.role_id_has_access(admin_role_ids, ctx.author.roles):
if "," in user_toon:
toons = user_toon.split(",")
for x in toons:
User.toon_add(user, ctx.message.guild.id, x.strip(),
timestamp())
await ctx.message.channel.send(
"Added characters: " + user_toon +
" to user: <@!" + str(ctx.message.mentions[0].id) +
">")
else:
User.toon_add(user, ctx.message.guild.id, user_toon,
timestamp())
await ctx.message.channel.send(
"Added character: " + user_toon + " to user: <@!" +
str(ctx.message.mentions[0].id) + ">")
else:
await ctx.message.channel.send(
ctx.message.author.name +
" Sorry, but you do not have access to this "
"function. Contact a bot admin.")
def SingleByteXORCipher(cipher):
#cipher = bytes.fromhex(
# '1b37373331363f78151b7f2b783431333d78397828372d363c78373e783a393b3736')
potential_messages = []
for key_value in range(256):
message = MiscFunctions.singleCharacterXOR(cipher, key_value)
score = MiscFunctions.getEnglishScore(message)
data = {
'message': message,
'score': score,
'key': key_value
}
potential_messages.append(data)
return sorted(potential_messages,
key=lambda x: x['score'], reverse=True)[0]
def RocPlot(data_file):
# Determines the separator type associated with the file format of the data file.
sep = mf.determine_separator(data_file)
# Creates a partition of the interval [0,1]. Creates two empty lists; one will store the true
# positive rate and the other will store the false positive rates.
thresholds = [float(i)/100 for i in range(101)]
x = []
y = []
df = pd.read_csv(sys.argv[1], sep=',')
# Determines what term will be considered positive and which will be negative.
positive_term = 'Pathogenic'
negative_term = 'Benign'
positives = df[df['pathogenicity'] == positive_term].shape[0]
negatives = df[df['pathogenicity'] == negative_term].shape[0]
# For each threshold in the partition, determines the true positive rate and the true negative
# rate for the data.
for threshold in thresholds:
true_positives = 0
false_positives = 0
for index, row in df.iterrows():
score = float(row.values[2])
if ( (score >= threshold) & (row.values[1] == positive_term) ):
true_positives += 1
elif ( (score >= threshold) & (row.values[1] == negative_term) ):
false_positives += 1
true_positive_rate = true_positives/positives
false_positive_rate = false_positives/negatives
x.append(false_positive_rate)
y.append(true_positive_rate)
#------------------------------------------------------------------------------------------------
# Creates an empty figure, with total area of 1.
figure = plt.figure(figsize=(6, 6))
ax = figure.add_subplot(1, 1, 1)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.xaxis.set_major_locator(MultipleLocator(0.250))
ax.yaxis.set_major_locator(MultipleLocator(0.250))
ax.xaxis.set_minor_locator(MultipleLocator(0.125))
ax.yaxis.set_minor_locator(MultipleLocator(0.125))
ax.tick_params(which='major', length=10.0)
ax.tick_params(which='minor', length=5.0)
ax.set_ylabel('True Positive Rate')
ax.set_xlabel('False Positive Rate')
# Draws the grid and plots the ROC curce and the baseline curve.
ax.grid(linestyle=':', linewidth=0.5, color='black')
ax.plot(x, y, color='blue', linewidth='3.0')
ax.plot([0,1], [0,1], color='orange', linestyle='--')
plt.show()
def CountUniqueElementsColumn(read_file):
# Creates a csv file from read_file.
sep = mf.determine_separator(read_file)
df = pd.read_csv(read_file, sep=sep, header=0)
# Print the columns of read_file to the console. Gets the name of the column to be printed
# from the user.
mf.print_columns_with_index(df)
column_number = mf.get_int_answer('What column should be accumulated? ')
column_name = list(df.columns)[column_number - 1]
# Creates a list of the unique values from the specified column. Prints its length, and
# optionally each item and its frequency.
unique_values = pd.unique(df[column_name])
print('Number of unique items: ' + str(len(unique_values)))
show_items = mf.get_yes_no('Show item/counts (y/n)? ')
if ((show_items == 'y') | (show_items == 'Y')):
for item in unique_values:
print(str(item) + ': ' + str(df[df[column_name] == item].shape[0]))
def replaceCarPos(self, carInit, carFin): # to be only used in moveCar()
#print 'Car INIT'
#carInit.printDetails()
#print 'Car FIN'
#carFin.printDetails()
initPos = misc.findAll(self.boardArr, carInit.carChar)
finPos = carFin.allCoordinates
for i in initPos:
self.boardArr[i[0]][i[1]] = ' '
for j in finPos:
self.boardArr[j[0]][j[1]] = carFin.carChar
def Optimization(data_file):
# Creates a dataframe from data_file.
sep = mf.determine_separator(sys.argv[1])
df = pd.read_csv(sys.argv[1], sep=',')
# Creates a partition of the interval [0,1]. Creates two empty lists; one will store the accuracy
# and the other will store the threshold.
thresholds = [float(i) / 100 for i in range(101)]
x = []
y = []
# Determines what term will be considered positive and which will be negative.
positive_term = 'Pathogenic'
negative_term = 'Benign'
total = df['pathogenicity'].shape[0]
# For each threshold in the partition, determines the accuracy of the predictor.
for threshold in thresholds:
true_positives = 0
true_negatives = 0
for index, row in df.iterrows():
score = float(row.values[2])
if ((score >= threshold) & (row.values[1] == positive_term)):
true_positives += 1
elif ((score < threshold) & (row.values[1] == negative_term)):
true_negatives += 1
accuracy = (true_positives + true_negatives) / total
x.append(threshold)
y.append(accuracy)
print('Max : ' + str(max(y)) + '\nAt : ', end='')
print(*[x[index] for index, value in enumerate(y) if value == max(y)],
sep=', ')
#------------------------------------------------------------------------------------------------
# Creates an empty figure, with total area of 1.
figure = plt.figure(figsize=(6, 6))
ax = figure.add_subplot(1, 1, 1)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.xaxis.set_major_locator(MultipleLocator(0.250))
ax.yaxis.set_major_locator(MultipleLocator(0.250))
ax.xaxis.set_minor_locator(MultipleLocator(0.125))
ax.yaxis.set_minor_locator(MultipleLocator(0.125))
ax.tick_params(which='major', length=10.0)
ax.tick_params(which='minor', length=5.0)
ax.set_ylabel('Accuracy')
ax.set_xlabel('Threshold')
# Draws the grid and plots the response curve.
ax.grid(linestyle=':', linewidth=0.5, color='black')
ax.plot(x, y, color='blue', linewidth='3.0')
plt.show()
def DataExtraction(read_file, write_file):
# Creates a dataframe from read_file and opens a new file for writing.
sep = mf.determine_separator(read_file)
df = pd.read_csv(read_file, sep=sep, header=0)
write_file = open(write_file, 'w+')
# Prints the columns of the file to the console. Gets the name of the column to extract
# variant identifiers from. Optionally writes the pathogenicity into a second column.
mf.print_columns_with_index(df)
col1 = mf.get_int_answer('What is the number of the column to sort by? ')
coordinate_column = list(df.columns)[col1 - 1]
df.sort_values(by=df.columns[col1 - 1])
first_question = mf.get_yes_no('Write pathogenicity (y/n)? ')
if ((first_question == 'y') | (first_question == "Y")):
second_question = mf.get_int_answer('Pathogenicity\'s column number? ')
pathogenicity_column = list(df.columns)[second_question - 1]
# Writes the variant identifier and the pathogencity to output_file.
for index, row in df.iterrows():
write_file.write(df[coordinate_column].iloc[index] + ',')
if ((first_question == 'y') | (first_question == "Y")):
write_file.write(df[pathogenicity_column].iloc[index])
write_file.write('\n')
write_file.close()
def next_for_car(self, carCh): # can take arguments as character
CLOSED = [] # list to store the states that have already been achieved
if misc.isCarInBoard(self.boardArr, carCh) == False:
print 'next_for_car(): ', CarNotIn
car = Car.Car(self.boardArr, carCh)
cloneBoard = Board()
#car.printDetails()
if car.orientation == 'horizontal':
directions = ['left', 'right']
for direction in directions:
for i in range(1, 5):
cloneBoard.clone(self)
newBoardArr = cloneBoard.moveCar(carCh, direction, i)
#cloneBoard.printBoard()
if newBoardArr not in CLOSED and newBoardArr is not None:
#print newBoardArr
#CLOSED.append(cloneBoard)
CLOSED.append(newBoardArr)
elif car.orientation == 'vertical':
directions = ['down', 'up']
for direction in directions:
for i in range(1, 5):
cloneBoard.clone(self)
newBoardArr = cloneBoard.moveCar(carCh, direction, i)
#cloneBoard.printBoard()
if newBoardArr not in CLOSED and newBoardArr is not None:
#print newBoardArr
#CLOSED.append(cloneBoard)
CLOSED.append(newBoardArr)
#CLOSED.remove(None)
#print CLOSED
#misc.printCLOSED(CLOSED)
for board in CLOSED:
#print board
#board.printBoard()
misc.printBoardArr(board)
#for row in board:
#print row
#print
return CLOSED
def submit_to_queue(folder_script_pairs, max_on_queue=600):
qsub = sh.Command("qsub")
for run_line in folder_script_pairs:
with MiscFunctions.cd(os.path.join("./", run_line[0])):
last_submitted = qsub(run_line[1])
print last_submitted
time.sleep(5)
while get_tasks_on_queue() > max_on_queue:
print "{0} tasks on queue is greater then maximum allowed {1}," \
" waiting 20 min from {2}".format(get_tasks_on_queue(), max_on_queue, datetime.datetime.now().time())
time.sleep(20 * 60)
return int(last_submitted.split(" ")[2])
async def on_member_update(self, before, after):
new_role = Util.get_new_roles(before.roles, after.roles)
if new_role == "Academy":
await asyncio.sleep(20)
roles = self.bot.get_guild(after.guild.id).get_member(
after.id).roles
for role in roles:
# German Role Greeting
if role.name == "German":
channel = self.bot.get_channel(german_chat)
await channel.send("<@!" + str(after.id) + ">" +
german_welcome)
# French Role Greeting
if role.name == "French":
channel = self.bot.get_channel(french_chat)
await channel.send("<@!" + str(after.id) + ">" +
french_welcome)
channel = self.bot.get_channel(coalition_chat)
await channel.send("<@!" + str(after.id) + ">" + english_welcome)
# modify this table at all, as per the instruction.
# for this simulation, we will have, in the end, two tables,
# 'mailing' and 'totalcount', where 'totalcount' keeps track
# of all the emails added for each day
emailDB.resetMailingTableInDatabase()
emailDB.resetDailyCountInDatabase()
# http://tinyurl.com/ooknlll
# this part will simulate the addition of new email addresses daily
# char_set = string.ascii_lowercase + string.digits
numberOfSimulationDays = 100
MiscFunctions.addEmailAddresses(emailDB, numberOfSimulationDays)
# this part represents the email addresses that have not been included
# in the daily count. These addresses represent addresses that have been
# added today (on a 24-hour cycle). Daily count is updated every 24 hour.
# The statistics that will be generated will be up to the point the last
# email address is added.
MiscFunctions.addEmailWithoutUpdatingCount(emailDB)
## print 'Committing now'
emailDB.cnx.commit()
# total domain counts
f = open('output_count_total_top_50.txt','w')
domainNameCount = emailDB.getDomainCountTotal()
def Optimization(data_file):
# Creates a dataframe from data_file.
sep = mf.determine_separator(sys.argv[1])
df = pd.read_csv(sys.argv[1], sep=',')
scores = (df['Score'].values).copy()
scores.sort()
# Sets the resolution for the plot
granularity = 1
for index in range(len(scores) - 1):
difference = scores[index + 1] - scores[index]
if (difference < granularity):
granularity = difference
if (granularity < .001):
granularity = .001
# Creates a partition of the interval [0,1]. Creates two empty lists; one will store the accuracy
# and the other will store the threshold.
thresholds = [
float(i) * granularity for i in range(int(1 / granularity) + 1)
]
x = []
y = []
# Determines what term will be considered positive and which will be negative.
positive_term = 'Pathogenic'
negative_term = 'Benign'
total = df['Pathogenicity'].shape[0]
print(total)
# For each threshold in the partition, determines the accuracy of the predictor.
for threshold in thresholds:
true_positives = 0
true_negatives = 0
for index, row in df.iterrows():
score = float(row.values[2])
if ((score >= threshold) & (row.values[1] == positive_term)):
true_positives += 1
elif ((score < threshold) & (row.values[1] == negative_term)):
true_negatives += 1
accuracy = (true_positives + true_negatives) / total
x.append(threshold)
y.append(accuracy)
# Prints the threshold(s) that optimizes the accuracy and computes their average.
max_values = [x[index] for index, value in enumerate(y) if value == max(y)]
print('Resolution : ' + str(granularity))
print('Max : ' + str(max(y)))
print('Mean : ' + str(stat.mean(max_values)))
print('Median : ' + str(stat.median(max_values)))
#------------------------------------------------------------------------------------------------
# Creates an empty figure, with total area of 1.
figure = plt.figure(figsize=(6, 6))
ax = figure.add_subplot(1, 1, 1)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.xaxis.set_major_locator(MultipleLocator(0.250))
ax.yaxis.set_major_locator(MultipleLocator(0.250))
ax.xaxis.set_minor_locator(MultipleLocator(0.125))
ax.yaxis.set_minor_locator(MultipleLocator(0.125))
ax.tick_params(which='major', length=10.0)
ax.tick_params(which='minor', length=5.0)
ax.set_ylabel('Accuracy')
ax.set_xlabel('Threshold')
# Draws the grid and plots the response curve.
ax.grid(linestyle=':', linewidth=0.5, color='black')
ax.plot(x, y, color='blue', linewidth='2.0')
ax.plot(
[stat.median(max_values),
stat.median(max_values)],
[0, max(y)],
color='red',
linestyle='--',
)
plt.title('Accuracy of EA on Training Set as a Function of Threshold')
plt.show()
def get_configuration(self):
"""
:return:
"""
# Creating the message
msg_type = np.array([int('0002', 16)], dtype=np.uint16)
msg_id = np.array([int('0000', 16)], dtype=np.uint16)
rcm_get_config_request = Mf.typecast(
np.array(
[Mf.swap_bytes_16(msg_type[0]),
Mf.swap_bytes_16(msg_id[0])],
dtype=np.uint16), 8)
rcm_get_config_request = bytearray(rcm_get_config_request)
# send data
if self.req_ip is not None:
msg = self.send_rcv(rcm_get_config_request, 32, 0.2)
else:
print 'UWB radio has no ip set'
return -1
# Processing message
if msg is not None:
msg_type = Mf.typecast(np.array([msg[1], msg[0]], dtype=np.uint8),
16)
if msg_type != np.array([0x0102], dtype=np.uint16):
print 'Message type %s does not match RCM_GET_CONFIG_CONFIRM.' % msg_type
return
else:
self.config.msg_id = Mf.typecast(
np.array([msg[3], msg[2]], dtype=np.uint8), 16)
self.config.node_id = Mf.typecast(
np.array([msg[7], msg[6], msg[5], msg[4]], dtype=np.uint8),
32)
self.config.pii = Mf.typecast(
np.array([msg[9], msg[8]], dtype=np.uint8), 16)
self.config.ant_mode = np.array([msg[10]], dtype=np.uint8)
self.config.code_chnl = np.array([msg[11]], dtype=np.uint8)
self.config.ant_dly_a = Mf.typecast(
np.array([msg[15], msg[14], msg[13], msg[12]],
dtype=np.uint8), 32)
self.config.ant_dly_b = Mf.typecast(
np.array([msg[19], msg[18], msg[17], msg[16]],
dtype=np.uint8), 32)
self.config.flags = Mf.typecast(
np.array([msg[21], msg[20]], dtype=np.uint8), 16)
self.config.tx_pwr = np.array([msg[22]], dtype=np.uint8)
self.config.unused = np.array([msg[23]], dtype=np.uint8)
self.config.time_stamp = Mf.typecast(
np.array([msg[27], msg[26], msg[25], msg[24]],
dtype=np.uint8), 32)
self.config.stat = Mf.typecast(
np.array([msg[31], msg[30], msg[29], msg[28]],
dtype=np.uint8), 32)
self.config.persist_flag = np.array([0], dtype=np.uint8)
return
def set_conf(self):
"""
:return: the status of the response from the RCM.
"""
msg_type = Mf.swap_bytes_16(
np.array([int('0001', 16)],
dtype=np.uint16)) # rcm_set_config_request msg type.
msg_id = Mf.swap_bytes_16(np.array([int('0003', 16)], dtype=np.uint16))
if self.config.node_id is not None:
node_id = Mf.swap_bytes_32(self.config.node_id)
else:
print 'UWB radio has no ip set'
return -1
pii = Mf.swap_bytes_16(self.config.pii)
ant_mode = self.config.ant_mode
code = self.config.code_chnl
ant_delay_a = Mf.swap_bytes_32(self.config.ant_dly_a)
ant_delay_b = Mf.swap_bytes_32(self.config.ant_dly_b)
flags = Mf.swap_bytes_16(self.config.flags)
tx_gain = self.config.tx_pwr
persist = self.config.persist_flag
rcm_set_config_request = \
np.concatenate([Mf.typecast(np.array([msg_type[0], msg_id[0]], dtype=np.uint16), 8),
Mf.typecast(np.array([node_id[0]], dtype=np.uint32), 8),
Mf.typecast(np.array([pii[0]], dtype=np.uint16), 8),
ant_mode, code,
Mf.typecast(np.array([ant_delay_a[0], ant_delay_b[0]], dtype=np.uint32), 8),
Mf.typecast(np.array([flags[0]], dtype=np.uint16), 8),
tx_gain, persist])
rcm_set_config_request.dtype = np.uint8
rcm_set_config_request = bytearray(rcm_set_config_request)
# send data
if self.req_ip is not None:
msg = self.send_rcv(rcm_set_config_request, 8, 0.4)
else:
print 'Unable to parse config, check your connection with the UWB'
return -1
# processing response
if msg is not None:
msg_type = Mf.typecast(np.array([msg[1], msg[0]], dtype=np.uint8),
16)
if msg_type != np.array([int('0101', 16)], dtype=np.uint16):
print 'Message type %s does not match RCM_SET_CONFIG_CONFIRM. ' % msg_type
else:
# msg_id in confirm should be equal to msg_id in request
msg_id = Mf.typecast(
np.array([msg[3], msg[2]], dtype=np.uint8), 16)
# status = Mf.typecast(np.array([msg[7], msg[6], msg[5], msg[4]], dtype=np.uint8), 32)
return msg_id
def req_range(self, msg_id, responder_id):
"""
:param msg_id: ID of the message to send to the RCM. The id will decide what type of measurement
the RCM will perform.
:param responder_id: The id of the node to which measurements will be done.
:return: The status of the request and the confirmed message id from the RCM. This message should be the same
as the one you sent it to make sure you received what you want.
"""
status = np.array([0xFFFFFFFF], dtype=np.uint32) # return variable 1
msg_id_confirm = np.empty(0, dtype=np.uint32) # return variable 2
msg_type = np.array(
[int('0003', 16)],
dtype=np.uint16) # rcm_send_range_request message type.
msg_id = np.array([msg_id], dtype=np.uint16)
resp_id = np.array([responder_id], dtype=np.uint32)
ant_mode = np.array([0], dtype=np.uint8)
reserved = np.array([0], dtype=np.uint8)
data_size = np.array([0], dtype=np.uint16)
data = np.empty(0, dtype=np.uint8)
rcm_send_range_request = np.concatenate([
Mf.typecast(Mf.swap_bytes_16(msg_type), 8),
Mf.typecast(Mf.swap_bytes_16(msg_id), 8),
Mf.typecast(Mf.swap_bytes_32(resp_id), 8), ant_mode, reserved,
Mf.typecast(Mf.swap_bytes_16(data_size), 8), data
])
rcm_send_range_request.dtype = np.uint8
rcm_send_range_request = bytearray(rcm_send_range_request)
# send data
if self.req_ip is not None:
msg = self.send_rcv(rcm_send_range_request, 8, 0.3)
else:
print 'Unable to parse config, check your connection with the UWB'
return -1
# processing message
if msg is not None:
msg_type = Mf.typecast(np.array([msg[1], msg[0]], dtype=np.uint8),
16)
if msg_type != np.array([int('0103', 16)], dtype=np.uint16):
print 'Message type %s does not match RCM_SEND_RANGE_CONFIRM. ' % msg_type
status = np.array([1], dtype=np.uint32)
# msg_id_confirm remains empty uint32
else:
msg_id_confirm = Mf.typecast(
np.array([msg[3], msg[2]], dtype=np.uint8), 16)
status = Mf.typecast(
np.array([msg[7], msg[6], msg[5], msg[4]], dtype=np.uint8),
32)
return status, msg_id_confirm
def FixedXOR():
b1 = bytes.fromhex('1c0111001f010100061a024b53535009181c')
b2 = bytes.fromhex('686974207468652062756c6c277320657965')
print(MiscFunctions.xorCombination(b1,b2).hex())
import MiscFunctions as mf
import random # For pseudo-random numbers
import time # To set the seed
#------------------------------------------------------------------------------------------------
## Move a specified number of pathogenic or benign variants from a csv file to another file.
# this doesn't modify the original files, rather creates new ones with new data.
# @param take_from : The csv-like file to remove variants from.
# @param add_to : The csv-like file to add variants to.
# @param modified_take_from : The modified version of take_from
# @param modified_add_to : The modified version of add_to
#
def MoveVariants(take_from, add_to, modified_take_from, modified_add_to)
# Create dataframes from take_from and add_to. I will refer to the dataframes by their
# respective filenames.
sep1 = mf.determine_separator(take_from)
sep2 = mf.determine_separator(add_to)
df1 = pd.read_csv(take_from, sep=sep1, header=0)
df2 = pd.read_csv(add_to, sep=sep2, header=0)
# Gets the name of the column of the variants' pathogenicity in take_from.
mf.print_columns_with_index(df1)
df1_column_number = mf.get_int_answer('What column contains the pathogenicity in the first file? ')
df1_column_name = list(df1.columns)[df1_column_number - 1]
df1_variant_column = list(df1.columns)[0]
# Gets the name of the column of the variants' pathogenicity add_to.
mf.print_columns_with_index(df2)
df2_column_number = mf.get_int_answer('What column contains the pathogenicity in the second file? ')
df2_column_name = list(df2.columns)[df2_column_number - 1]
df2_variant_column = list(df2.columns)[0]
def BreakRepeatingKeyXOR(ciphertext):
""" Attempts to break repeating-key XOR encryption """
averages_distances = []
#Take the keysize from the suggested range
for keysize in range(1,41):
#Initialize list to store Hamming Distances for this keysize
distances = []
#Break the ciphertext into chunks the lenght of the keysize
chunks = [ciphertext[i: i +keysize] for i in range(0,len(ciphertext),keysize)]
while True:
try:
#Take the two chunks at the beginning of the list
#Get the Hamming Distance between these 2 chunks
chunk_1 = chunks[0]
chunk_2 = chunks[1]
distance = MiscFunctions.calculateHammingDistance(chunk_1,chunk_2)
#Normalize the result by dividing by the keysize
distances.append(distance/keysize)
#Remove these 2 chunks so when the loop starts over we get the next 2 parts of the cipher
del chunks[0]
del chunks[1]
#When an exception Occurs (indicating all chunks have been processed)
#Break the loop
except Exception as e:
break
result = {
'key':keysize,
'avg_distance' :sum(distances) / len(distances)
}
averages_distances.append(result)
#Take the 5 shortest average Distances
possible_key_lengths = sorted(averages_distances,key = lambda x: x['avg_distance'])[:5]
possible_plaintext = []
#Iterating through each one of the five results with the shortest
#Normalized differences
for res in possible_key_lengths:
#Will populate with a single characted as each transposed
#Block has been single byte-XOR brute forced
key = b''
for i in range(res['key']):
#Create a block made up of each nth byte , where n
#is the keysize
block = b''
for j in range(i,len(ciphertext),res['key']):
block+= bytes([ciphertext[j]])
key += bytes([SingleByteXORCipher(block)['key']])
possible_plaintext.append((MiscFunctions.XORWithRepeatingKey(ciphertext,key),key))
return max(possible_plaintext,key = lambda x :MiscFunctions.getEnglishScore(x[0]))
def ConvertHextoBase64() :
print(MiscFunctions.hexTobase64("49276d206b696c6c696e6720796f757220627261696e206c696b65206120706f69736f6e6f7573206d757368726f6f6d"))
def printPath(self):
boardArrs = []
for board in self.boards:
boardArrs.append(board.boardArr)
misc.printCLOSED(boardArrs) #or boards.boardArr
command = str(sys.argv[1]) # storing the command
board = Board.Board()
if len(sys.argv
) > 2: # checking whether there's the optional argument for input
boardString = str(sys.argv[2]) # storing the argument into an input string
board.createBoard(boardString)
if command == "print":
board.printBoard()
elif command == "done":
print board.isDone()
elif command == "next":
nextBoards = board.next()
misc.printCLOSED(nextBoards)
elif command == "random":
board.random(10)
elif command == "bfs":
board.bfs()
elif command == "astar":
board.astar()
elif command == "test":
#print misc.findAll(board.boardArr, 'q')
board.next_for_car('o')
#board.moveCar('q', 'up', 2)
#print misc.find(board.boardArr, 'p')
#print misc.findAll(board.boardArr, 'p')
#board.moveCar('p','right',2)
arrHorizontalBox = np.matmul(arrHorizontalCell,arrBasis) arrVerticalBox = np.matmul(arrVerticalCell, arrBasis) objSimulationCell = gl.SimulationCell(arrFullBox) arrGrain1 = gl.ParallelopiedGrain(arrHorizontalBox,arrGrainBasis1,ld.FCCCell,a*np.ones(3), np.zeros(3)) arrGrain2 = gl.ParallelopiedGrain(arrSmallBox,arrGrainBasis2,ld.FCCCell,a*np.ones(3), 0.5*(arrFullBox[0]+arrFullBox[1])) arrGrain3 = gl.ParallelopiedGrain(arrSmallBox,arrGrainBasis3,ld.FCCCell,a*np.ones(3), 0.5*arrFullBox[1]) strFilename = 'TJ123.dat' objSimulationCell.AddGrain(arrGrain1) objSimulationCell.AddGrain(arrGrain2) objSimulationCell.AddGrain(arrGrain3) objSimulationCell.MergeTooCloseAtoms(0.1,1) objSimulationCell.WriteLAMMPSDataFile(strDirectory + strFilename) lstNew = [strFilename, strFilename[:-3]+'dmp', strFilename[:-3]+'lst', strFilename[:-3] + 'log'] MiscFunctions.UpdateTemplate(lstOldTemplate,lstNew, strDirectory + strTemplateName, strDirectory +'Template' + strFilename[:-3] + 'in') objSimulationCell = gl.SimulationCell(arrSmallBox) arrGrain1 = gl.ParallelopiedGrain(arrSmallBox,arrGrainBasis1,ld.FCCCell,a*np.ones(3), np.zeros(3)) arrGrain2 = gl.ParallelopiedGrain(arrSmallBox,arrGrainBasis2,ld.FCCCell,a*np.ones(3), np.zeros(3)) strFilename = 'G1.dat' objSimulationCell.AddGrain(arrGrain1) objSimulationCell.WriteLAMMPSDataFile(strDirectory + strFilename) lstNew = [strFilename, strFilename[:-3]+'dmp', strFilename[:-3]+'lst', strFilename[:-3] + 'log'] MiscFunctions.UpdateTemplate(lstOldTemplate,lstNew, strDirectory + strTemplateName, strDirectory +'Template' + strFilename[:-3] + 'in') objSimulationCell.RemoveAllGrains() strFilename = 'G2.dat' objSimulationCell.AddGrain(arrGrain2) objSimulationCell.WriteLAMMPSDataFile(strDirectory + strFilename) lstNew = [strFilename, strFilename[:-3]+'dmp', strFilename[:-3]+'lst', strFilename[:-3] + 'log'] MiscFunctions.UpdateTemplate(lstOldTemplate,lstNew, strDirectory + strTemplateName, strDirectory +'Template' + strFilename[:-3] + 'in')
def MoveVariants(take_from, add_to, modified_take_from, modified_add_to):
# Create dataframes from take_from and add_to. I will refer to the dataframes by their
# respective filenames.
sep1 = mf.determine_separator(take_from)
sep2 = mf.determine_separator(add_to)
df1 = pd.read_csv(take_from, sep=sep1, header=0)
df2 = pd.read_csv(add_to, sep=sep2, header=0)
# Gets the name of the column of the variants' pathogenicity in take_from.
mf.print_columns_with_index(df1)
df1_column_number = mf.get_int_answer(
'What column contains the pathogenicity in the first file? ')
df1_column_name = list(df1.columns)[df1_column_number - 1]
df1_variant_column = list(df1.columns)[0]
# Gets the name of the column of the variants' pathogenicity add_to.
mf.print_columns_with_index(df2)
df2_column_number = mf.get_int_answer(
'What column contains the pathogenicity in the second file? ')
df2_column_name = list(df2.columns)[df2_column_number - 1]
df2_variant_column = list(df2.columns)[0]
# Gets the number of variants to transfer and of what pathogenicity.
answer = input('Move pathogenic or benign variants (p/b)? ')
number_to_move = mf.get_int_answer('Move how many variants? ')
# Splits take_from into two parts, one containing only pathogenic variants and the other only benign.
df1_pathogenic = df1[(df1[df1_column_name] == 'Pathogenic') |
(df1[df1_column_name] == 'Likely_pathogenic')]
df1_benign = df1[(df1[df1_column_name] == 'Benign') |
(df1[df1_column_name] == 'Likely_benign')]
if (answer == 'p'): # If the user wants to move pathogenic variants.
# Randomly selects number_to_move pathogenic variants from take_from's pathogenic half.
number_of_variants = df1_pathogenic.shape[0]
indices = random.sample(range(number_of_variants), number_to_move)
everything_else = [
i for i in range(number_of_variants) if i not in indices
]
# Creates a dataframe of the pathogenic variants to move. Renames the columns to match add_to's.
df1_to_move = df1_pathogenic.iloc[indices][[
df1_variant_column, df1_column_name
]]
df1_to_move = df1_to_move.rename(index=str,
columns={
df1_variant_column:
df2_variant_column,
df1_column_name: df2_column_name
})
# Generates the two modified dataframes, and generates the two new csv files.
df_out_2 = df2[[df2_variant_column,
df2_column_name]].append(df1_to_move,
ignore_index=True)
df_out_1 = df1_benign.append(df1_pathogenic.iloc[everything_else])
df_out_1.to_csv(modified_take_from, index=False)
df_out_2.to_csv(modified_add_to, index=False)
else: # If the user wants to move benign variants.
# Randomly selects number_to_move benign variants from take_from's benign half.
number_of_variants = df1_benign.shape[0]
indices = random.sample(range(number_of_variants), number_to_move)
everything_else = [
i for i in range(number_of_variants) if i not in indices
]
# Creates a dataframe of the benign variants to move. Renames the columns to match add_to's.
df1_to_move = df1_benign.iloc[indices][[
df1_variant_column, df1_column_name
]]
df1_to_move = df1_to_move.rename(index=str,
columns={
df1_variant_column:
df2_variant_column,
df1_column_name: df2_column_name
})
# Generates the two modified dataframes, and generates the two new csv files.
df_out_2 = df2[[df2_variant_column,
df2_column_name]].append(df1_to_move,
ignore_index=True)
df_out_1 = df1_pathogenic.append(df1_benign.iloc[everything_else])
df_out_1.to_csv(modified_take_from, index=False)
df_out_2.to_csv(modified_add_to, index=False)
def RocPlot(data_file):
# Determines the separator type associated with the file format of the data file.
sep = mf.determine_separator(data_file)
df = pd.read_csv(sys.argv[1], sep=sep)
scores = (df['Score'].values).copy()
binary_classifications = df['Pathogenicity'].replace({
'Pathogenic': 1,
'Benign': 0
})
AUC = roc_auc_score(list(binary_classifications.values), scores)
scores.sort()
# Sets the resolution for the plot
granularity = 1
for index in range(len(scores) - 1):
difference = scores[index + 1] - scores[index]
if (difference < granularity):
granularity = difference
if (granularity < .001):
granularity = .001
# Creates a partition of the interval [0,1]. Creates two empty lists; one will store the true
# positive rate and the other will store the false positive rates.
thresholds = [
float(i) * granularity for i in range(int(1 / granularity) + 1)
]
x = []
y = []
# Determines what term will be considered positive and which will be negative.
positive_term = 'Pathogenic'
negative_term = 'Benign'
positives = df[df['Pathogenicity'] == positive_term].shape[0]
negatives = df[df['Pathogenicity'] == negative_term].shape[0]
# For each threshold in the partition, determines the true positive rate and the true negative
# rate for the data.
for threshold in thresholds:
true_positives = 0
false_positives = 0
for index, row in df.iterrows():
score = float(row.values[2])
if ((score >= threshold) & (row.values[1] == positive_term)):
true_positives += 1
elif ((score >= threshold) & (row.values[1] == negative_term)):
false_positives += 1
true_positive_rate = true_positives / positives
false_positive_rate = false_positives / negatives
x.append(false_positive_rate)
y.append(true_positive_rate)
#------------------------------------------------------------------------------------------------
# Creates an empty figure, with total area of 1.
figure = plt.figure(figsize=(6, 6))
ax = figure.add_subplot(1, 1, 1)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.xaxis.set_major_locator(MultipleLocator(0.250))
ax.yaxis.set_major_locator(MultipleLocator(0.250))
ax.xaxis.set_minor_locator(MultipleLocator(0.125))
ax.yaxis.set_minor_locator(MultipleLocator(0.125))
ax.tick_params(which='major', length=10.0)
ax.tick_params(which='minor', length=5.0)
ax.set_ylabel('True Positive Rate')
ax.set_xlabel('False Positive Rate')
# Draws the grid and plots the ROC curce and the baseline curve.
ax.grid(linestyle=':', linewidth=0.5, color='black')
ax.plot(x, y, color='blue', linewidth='3.0')
ax.plot([0, 1], [0, 1], color='orange', linestyle='--')
plt.title('ROC Plot of REVEL on Test Set')
ax.legend(['AUC: ' + str(round(AUC, 3))], loc=4)
plt.show()
