def get_listings_information_by_url_file(filename, folder_out):
""" Open filename, read the formated links,
and write the information of each listing to json files
"""
f = open(filename, "r")
links = [l[:-1] for l in f]
files, index = 0, 0
info = []
print("\nJOB STARTING, TOTAL OF " + str(len(links)) + " LINKS LOADED")
for address in links:
index += 1
printProgressBar(index,
len(links),
prefix="Progress:",
suffix=str(index) + "/" + str(len(links)))
# when a certain number of links are scraped, store them as a file
if index % 200 == 0:
with open(folder_out + "info" + str(files) + ".json", "w") as fw:
json.dump(info, fw, indent=2)
info.clear()
files += 1
gotten = get_listing_information_by_url(address)
if gotten:
info.append(gotten)
with open(folder_out + "info" + str(files) + ".json", "w") as fw:
json.dump(info, fw, indent=2)
def marginal_outflow(p):
part_out = np.zeros(2 * p.para['nt'])
mass_out = np.zeros(2 * p.para['nt'])
# for each processor
for n in range(p.para['num_proc']):
f = ff.FortranFile(
join(p.particle, "c.outflowing_" + str(n + 1) + ".dat"))
# for each half timestep
for i in range(2 * p.para['nt']):
try:
mrat = f.readReals()
except ff.NoMoreRecords:
break
beta_p = f.readReals()
tags = f.readReals()
# Each of the arrays must have the same length
assert len(mrat) == len(beta_p) and len(mrat) == len(tags)
# If that length is zero than there was no outflow
if len(mrat) == 0:
continue
# for each macro particle
for m, b, t in zip(mrat, beta_p, tags):
if t != 0:
part_out[i] += 1 / (b * p.para['beta'])
mass_out[i] += kg_per_amu / m * 1 / (p.para['beta'] * b)
printProgressBar(n + 1, p.para['num_proc'], prefix='Outflow Files')
return part_out, mass_out
def main():
# Command Line Arguments
parser = argparse.ArgumentParser()
parser.add_argument("-i", "--input", const=str, nargs="?")
args = parser.parse_args()
print("Loading Data")
file1 = open(args.input or "preprocessorfile.txt", "r")
data = file1.read()
globaldata = ["start"]
splitdata = data.split("\n")
splitdata = splitdata[:-1]
print("Processed Pre-Processor File")
print("Converting to readable format")
for idx, itm in enumerate(splitdata):
printProgressBar(
idx, len(splitdata) - 1, prefix="Progress:", suffix="Complete", length=50
)
itm = itm.split(" ")
itm.pop(-1)
entry = itm
globaldata.append(entry)
pointstats = core.returnPointDist(globaldata)
print(pointstats)
def train_ec(self):
self.Qtable_ec = copy.deepcopy(self.Qtable)
Iterations = 100000
printProgressBar(0,
Iterations,
prefix='Training:',
suffix='Complete',
length=50)
for i in range(Iterations):
s = self.play_ec(1, i)
self.mdp_ec.init_envvironment()
printProgressBar(i + 1,
Iterations,
prefix='Training:',
suffix='Complete',
length=50)
if (i % 1000 == 0 or i == Iterations - 1):
num_success = 0
total = 0
for j in range(200):
s = self.play_ec(0, i)
total += s
self.mdp_ec.init_envvironment()
average = total / 200
# print('Average Score: ', average)
self.epoch_plot_ec.append(i)
self.avg_plot_ec.append(average)
pickle.dump(self, open('Qtable.txt', 'wb'))
def oldSolve(root_oct):
(portions, file_points) = initialisePoints(f)
start_time = time.time()
estimated_time = 0
for (i, portion) in enumerate(portions):
if (i == 6):
h = hpy()
print(h.heap())
start_time = time.time()
dt = np.dtype([('x', float), ('y', float), ('z', float)])
read = open("output/tmp/transformed/" + str(len(f.filename)) + "_port_" + str(i) + ".transformed", "rb")
arrayRead = np.frombuffer(read.read(), dtype=dt)
for c, coord in enumerate(arrayRead):
place(root_oct, {
'x': coord[0],
'y': coord[1],
'z': coord[2],
})
if (c % 10000 == 0):
prog.printProgressBar(c, len(arrayRead), 'Generating Portion ' + str(i) + "/" + str(len(portions)), 'Estimated : ' + str(estimated_time), length=50)
print()
end_time = time.time()
estimated_time = (end_time - start_time) * (len(portions) - i)
def create_show(data):
order = data.get('order')
vids_data = data.get('vids')
# print(order)
Path(OUT_FRAMES_FOLDER).mkdir(parents=True, exist_ok=True)
decor = VidDecoration()
vids = []
counter = 0
offset = SOURCE_WIDTH + HORIZONTAL_GAP
for name in order:
vid_settings = vids_data.get(name, {})
new_vid = MiniVid(name, vid_settings)
new_vid.place(offset, POS_Y)
vids.append(new_vid)
counter += 1
offset = offset + new_vid.width() + HORIZONTAL_GAP
decor.calculate_length(offset - SOURCE_WIDTH)
total_frames = int(offset / X_SPEED) + 10
bg_frames = 7392
start_show_at = 6600 - total_frames
print('total frames {}'.format(total_frames))
printProgressBar(0,
total_frames,
prefix='Progress:',
suffix='Complete',
length=50)
background = Image.open('images/green_back.png')
# background = Image.open('images/bg_real.png')
decor.place(SOURCE_WIDTH, POS_Y - (VIDS_HEIGHT * (DECOR_RATIO - 1) / 2))
for frame_number in range(1, bg_frames):
# print (frame_number)
printProgressBar(frame_number,
bg_frames,
prefix='Progress:',
suffix='Complete',
length=50)
dst = Image.new('RGB', (SOURCE_WIDTH, SOURCE_HEIGHT))
dst.paste(get_background_frame(frame_number), (0, 0))
# dst.paste(background,(0,0))
if (frame_number >= start_show_at) and (frame_number <
start_show_at + total_frames):
decor.move()
decor.draw(dst)
for vid in vids:
if vid.move():
dst.paste(vid.get_frame_image(), (vid.x(), vid.y()))
#print('x:{} y:{}'.format(vid.x(), vid.y()))
filename = 'frame{}.jpg'.format(zero_format(frame_number))
dst.save(os.path.join(OUT_FRAMES_FOLDER, filename))
def main():
filename = 'new_data'
files = os.listdir(filename)
files.sort()
folder = "sound_f"
if os.path.exists("sound_f"):
os.system("rm -r sound_f")
os.mkdir("sound_f")
ext = "/Sound"
count = 0
for i in files:
count += 1
name = filename + "/" + i + ext
file = os.listdir(name)
name_f = name + "/" + file[0]
sr, x = read_audio(name_f)
# print(x.shape)
if len(x.shape) == 2: # if channael is stereo then convert it to mono
#if wavefile is stereofile the data is returned in the form of multi dimensional
#array,if we need to convert the stereo to mono
# we need to add up the two values of the list of the index as 1 as float data
x = convert_to_mono(x)
X = band_pass(x, sr, 2000, 5001) #we consider between 2000 & 5001
i = 1
pwd = os.getcwd() #get the current working directory
os.chdir(
folder
) #folder variable actually contains the output directoy changing to it
f = open("processed_" + os.path.basename(name_f)[:-4] + "_0" +
str(count) + ".txt",
"w+") #newfile in output dir to write as text
while i * sr < len(
X
): #sampling rate X time actually i is the time to reach all the samples
x_ = X[
(i - 1) * sr:i *
sr] #considering the values of the currently working i only its equal to sr actually
# Take slice
N = 8000
win = np.hamming(
N
) #https://docs.scipy.org/doc/numpy-1.15.1/reference/generated/numpy.hamming.html
#we want the array final to have 8000 value.hamming(m):m->int->number of points in the output window
#if zero empty array is returned.The output is a normalized numpy array of 8000 values
freq, s_db = dbfft(x_, sr, win)
t = date_time(name_f, i)
date, time = t.split(" ")
f.write(date + ',' + time + "," + str(np.max(s_db)) +
'\n') # logging data timewise and the value
i += 1
printProgressBar(
i * sr,
len(X)) #print the progress of operation for each sound file
f.close()
# write_audio(X,sr,os.path.basename(name_f)[:-4]+'_filtered.wav') #saving the sounds with the honks only
os.chdir(pwd)
def getWindowFeaturesTrain(text, word_list):
sents = sent_tokenize(text)
flag = 0
counts = Counter()
progress = 0
totalen = len(sents)
for sent in sents:
parsed_sents = nlp.annotate(sent,
properties={
'annotators': 'tokenize,ssplit,pos',
'outputFormat': 'json'
})
for parsed_sent in parsed_sents['sentences']:
totalwords = len(parsed_sent['tokens'])
for item in parsed_sent['tokens']:
if (item["word"] in word_list):
#print(item)
word_index = item["index"]
features = ["None" for x in range(5)]
features[2] = item["word"]
if (word_index - 2 > 0):
#two words exists before
features[0] = parsed_sent['tokens'][word_index -
3]["pos"]
features[1] = parsed_sent['tokens'][word_index -
2]["pos"]
elif (word_index - 1 > 0):
#one word exists before
features[1] = parsed_sent['tokens'][word_index -
2]["pos"]
if (word_index + 2 <= totalwords):
#two words exists ahead
features[4] = parsed_sent['tokens'][word_index +
1]["pos"]
features[3] = parsed_sent['tokens'][word_index]["pos"]
elif (word_index + 1 <= totalwords):
#one word exists ahead
features[3] = parsed_sent['tokens'][word_index]["pos"]
counts[tuple(features)] += 1
#print(counts)
flag = 1
#break
#if(flag):
# break
progress += 1
printProgressBar(progress, totalen)
with open("window_size2_counts.pkl", 'wb') as modelfile:
pickle.dump(counts, modelfile)
def initialisePoints(f):
import os
if (os.path.exists("output/tmp/transformed") == False):
os.mkdir("output/tmp/transformed")
file_points = f.get_points()['point']
# file_points = all_file_points[0:(len(all_file_points)) // fraction]
if (fraction > 1):
print("Fraction less than 100 (Partial Export)")
count = len(file_points['X'] // fraction)
_1M = min(count, 1000000)
steps = math.ceil(count / _1M)
portions = [(i * _1M, min(count, (i + 1) * _1M)) for i in range(steps)]
X = file_points['X']
Y = file_points['Y']
Z = file_points['Z']
lenP = len(file_points)
start = time.time()
for i, portion in enumerate(portions):
if (os.path.exists("output/tmp/transformed/" + str(len(f.filename)) + "_port_" + str(i) + ".transformed") == False):
point_count = portion[1] - portion[0]
step = min(point_count, max((point_count) // 10, 100000))
indices = [i for i in range(math.ceil((point_count) / step))]
write = open("output/tmp/transformed/" + str(len(f.filename)) + "_port_" + str(i) + ".transformed", "wb")
for index in indices:
start_offset = portion[0] + index * step
num = min(step, portion[1] - start_offset)
# NEED A SCALED OFFSET TOO
x = X[start_offset:start_offset + num] * f.header.scale[0] + f.header.offset[0]
y = Y[start_offset:start_offset + num] * f.header.scale[1] + f.header.offset[1]
z = Z[start_offset:start_offset + num] * f.header.scale[2] + f.header.offset[2]
# saved.append([x, y, z])
#!! NOT CONFIRMED IF THIS REALLY OUTPUTS ALL THE DATA ::
write.write(np.vstack((x, y, z)).transpose().tobytes())
write.close()
prog.printProgressBar(i, len(portions), 'Transforming Coordinates', '%', length=100)
end = time.time()
print("Time to Transform " + str(end - start))
else:
print("Transformed Coordinates are Cached, no transformation required for : " + str(len(f.filename)) + "_port_" + str(i) + ".transformed" + " (100%)")
os.system("cls")
return (portions, file_points)
def loadFile(file):
startFullTime = time.time()
clearTemps()
(allPoints, rootBounds, spacing) = initialisePointsFixed(file)
# root_oct = node(
# {
# 'min': rootBounds[0],
# 'max': rootBounds[1]
# },
# 0,
# [],
# []
# )
time.sleep(10)
root_oct = oct_block(
None,
{
'min': rootBounds[0],
'max': rootBounds[1]
},
[],
[],
level=0,
id="root"
)
#print("Bounds : " + str(f.header.get_min()) + " - > " + str(f.header.get_max()))
count = 0
for p in allPoints:
count += 1
place(root_oct, p)
prog.printProgressBar(count, len(allPoints), 'Processing Octree')
print()
print("COMPLETED {}".format(count))
print()
import os
import linecache
root_oct.outputToPnts()
generateTileset(root_oct)
moveTilesAndTileset("")
def saveWindowFeaturesTrain(text, word_list, win_size):
sents = sent_tokenize(text)
flag = 0
traindata = []
progress = 0
totalen = len(sents)
featvocab = set({"None"})
for sent in sents:
parsed_sents = nlp.annotate(sent,
properties={
'annotators': 'tokenize,ssplit,pos',
'outputFormat': 'json'
})
for parsed_sent in parsed_sents['sentences']:
totalwords = len(parsed_sent['tokens'])
for item in parsed_sent['tokens']:
if (item["word"] in word_list):
#print(item)
word_index = item["index"]
feature = ["None" for x in range(win_size * 2 + 1)]
#first will be the word itself
feature[0] = item["word"]
for i in range(1, win_size + 1):
if (word_index - i > 0):
feature[i] = parsed_sent['tokens'][word_index - i -
1]["pos"]
featvocab.add(feature[i])
if (word_index + i <= totalwords):
feature[win_size +
i] = parsed_sent['tokens'][word_index + i -
1]["pos"]
featvocab.add(feature[win_size + i])
traindata.append(feature)
#flag += 1
#break
#if(flag==2):
# break
progress += 1
printProgressBar(progress, totalen)
#print(traindata)
#print(featvocab)
with open("dataset_window_train.pkl", 'wb') as modelfile:
pickle.dump([traindata, featvocab], modelfile)
def train(self):
Iterations = 50000
printProgressBar(0,
Iterations,
prefix='Training:',
suffix='Complete',
length=50)
for i in range(Iterations):
# print('iteration: ',i)
s = self.play(1)
self.mdp.init_envvironment()
printProgressBar(i + 1,
Iterations,
prefix='Training:',
suffix='Complete',
length=50)
def get_tokens(text):
tokens = [
word.lower() for sent in nltk.sent_tokenize(text)
for word in nltk.word_tokenize(sent)
]
filtered_tokens = [
token for token in tokens if re.search('[a-zA-Z]', token)
]
global iterations
iterations += 1
printProgressBar(iterations,
number_of_books,
prefix='Progress:',
suffix='Complete',
length=50)
return filtered_tokens
def initialisePointsFixed(filename):
data = reader.init([filename], None, None, None, fraction=100)
totalCoords = []
print("TOTAL : {}".format(data['point_count']))
print()
rotation_matrix = None
for portionData in data['portions']:
(filename, portion) = portionData
root_aabb = data['aabb'] - data['avg_min']
base_spacing = compute_spacing(root_aabb)
if base_spacing > 10:
root_scale = np.array([0.01, 0.01, 0.01])
elif base_spacing > 1:
root_scale = np.array([0.1, 0.1, 0.1])
else:
root_scale = np.array([1, 1, 1])
root_aabb = root_aabb * root_scale
root_spacing = compute_spacing(root_aabb)
offset_scale = (-data['avg_min'], root_scale, rotation_matrix[:3, :3].T if rotation_matrix is not None else None, data['color_scale'])
print(offset_scale)
coords, colors = reader.runSingle(filename, portion, offset_scale)
for (i, p) in enumerate(coords):
point = {
'x': p[0],
'y': p[1],
'z': p[2],
'r': colors[i][0],
'g': colors[i][1],
'b': colors[i][2],
}
totalCoords.append(point)
prog.printProgressBar(i, len(coords), 'Loading Data')
return (totalCoords, root_aabb, root_spacing)
def renaming(name,input_path, output_path):
printProgressBar(0, 1000, prefix='Converting', suffix='Complete', length=50)
cols = ['filename','width','height','class','xmin','ymin','xmax','ymax']
data =[]
annotations = pd.read_csv(input_path, header=None, names=cols, sep=' *, *', engine='python')
row_count = annotations.shape[0]
for idx, row in annotations.iterrows():
row['filename'] = name+row['filename'][3:10]+'.jpg'
data.append(row)
printProgressBar(idx, row_count, prefix='Converting', suffix='Complete', length=50)
output_csv = pd.DataFrame.from_dict(data)
output_csv = output_csv[cols]
output_csv.to_csv(output_path, mode='w', index=False, header=False)
def main():
# Command Line Arguments
parser = argparse.ArgumentParser()
parser.add_argument("-i", "--input", const=str, nargs="?")
args = parser.parse_args()
print("Loading Data")
file1 = open(args.input or "preprocessorfile.txt", "r")
data = file1.read()
globaldata = ["start"]
splitdata = data.split("\n")
splitdata = splitdata[:-1]
print("Processed Pre-Processor File")
print("Converting to readable format")
for idx, itm in enumerate(splitdata):
printProgressBar(idx,
len(splitdata) - 1,
prefix="Progress:",
suffix="Complete",
length=50)
itm = itm.split(" ")
itm.pop(-1)
entry = itm
globaldata.append(entry)
globaldata = core.cleanNeighbours(globaldata)
stuff = []
for idx, _ in enumerate(globaldata):
if idx > 0:
flag = core.getFlag(idx, globaldata)
if flag == 2:
ptx, pty = core.getPoint(idx, globaldata)
stuff.append([ptx, pty])
with open("outer.txt", "w") as text_file:
for item1 in stuff:
text_file.writelines(["%s " % item for item in item1])
text_file.writelines("\n")
def getDepFeaturesTrain(text, word_list):
sents = sent_tokenize(text)
flag = 0
counts = Counter()
progress = 0
totalen = len(sents)
for sent in sents:
parsed_sents = nlp.annotate(sent,
properties={
'annotators':
'tokenize,ssplit,pos,depparse',
'outputFormat': 'json'
})
for parsed_sent in parsed_sents['sentences']:
for item in parsed_sent['basicDependencies']:
if (item["dependentGloss"] in word_list):
gov_index = item["governor"]
if (gov_index == 0):
continue
dep = item["dep"]
gov = parsed_sent['tokens'][gov_index - 1]
if (gov_index != gov['index']):
raise Exception('Governor Index dont match')
gov_pos = gov['pos']
word = item["dependentGloss"]
#print(sent)
#print(word, dep, gov_pos)
counts[dep, gov_pos, word] += 1
flag = 1
#break
#params[word["originalText"]][]
#if(flag):
# break
progress += 1
printProgressBar(progress, totalen)
with open("deptree_level1_counts.pkl", 'wb') as modelfile:
pickle.dump(counts, modelfile)
def get_links_by_location_file(driver_link, r_file, w_file):
driver = prepare_driver(driver_link)
line_count = 0
with open(r_file, "r") as f:
line_count = sum([1 for line in f])
print("\nJOB STARTING, TOTAL OF " + str(line_count) +
" QUERIES LOADED")
f = open(r_file, "r")
urls = set()
current_line = 0
for line in f:
printProgressBar(current_line,
line_count,
prefix="Progress:",
suffix=str(len(urls)) + " collected")
urls |= get_link_by_search_query(line[:-1], driver)
current_line += 1
f.close()
with open(w_file, "w") as f:
for e in urls:
f.write(e + "\n")
def main():
# Command Line Arguments
parser = argparse.ArgumentParser()
parser.add_argument("-i", "--input", const=str, nargs="?")
args = parser.parse_args()
log.info("Loading Data")
log.debug("Arguments")
log.debug(args)
file1 = open(args.input or "preprocessorfile.txt", "r")
data = file1.read()
globaldata = ["start"]
splitdata = data.split("\n")
splitdata = splitdata[:-1]
log.info("Processed Pre-Processor File")
log.info("Converting to readable format")
for idx, itm in enumerate(splitdata):
printProgressBar(idx,
len(splitdata) - 1,
prefix="Progress:",
suffix="Complete",
length=50)
itm = itm.split(" ")
itm.pop(-1)
entry = itm
globaldata.append(entry)
globaldata = core.cleanNeighbours(globaldata)
wallpoints = core.getWallPointArray(globaldata)
wallpointsData = core.generateWallPolygons(wallpoints)
log.info("Finding average distance")
log.info("Done")
def get_availabilities_by_url_file(filename, date_in, date_out, driver_link,
folder_out):
""" Open filename, read the formated links,
and write the availablity of each listing to json files
"""
driver = prepare_driver(driver_link)
# read all the links that will be processed
f = open(filename, "r")
links = [l[:-1] for l in f]
# initialize how the files will be splited
files, index = 0, 0
info = []
print("\nJOB STARTING, TOTAL OF " + str(len(links)) + " LINKS LOADED")
for address in links:
index += 1
printProgressBar(index,
len(links),
prefix="Progress:",
suffix=str(index) + "/" + str(len(links)))
# when a certain number of links are scraped, store them as a file
if index % 200 == 0:
with open(folder_out + date_in + "_" + str(files) + ".json",
"w") as fw:
json.dump(info, fw, indent=2)
info.clear()
files += 1
availability = get_availabilities_by_url(address, date_in, date_out,
driver)
if availability:
info.append(availability)
f.close()
with open(folder_out + date_in + "_" + str(files) + ".json", "w") as fw:
json.dump(info, fw, indent=2)
def main():
filename = sys.argv[1]
folder = sys.argv[2]
sr, x = read_audio(filename)
# print(x.shape)
if len(x.shape) == 2: # if channael is stereo then convert it to mono
x = convert_to_mono(x)
X = band_pass(x, sr, 2000, 5001)
i = 1
pwd = os.getcwd()
os.chdir(folder)
f = open("processed_" + os.path.basename(filename)[:-4] + ".txt", "w+")
while i * sr < len(X):
x_ = X[(i - 1) * sr:i * sr]
# Take slice
N = 8000
win = np.hamming(N)
freq, s_db = dbfft(x_, sr, win)
f.write(date_time(filename, i) + ',' + str(np.max(s_db)) + '\n')
i += 1
printProgressBar(i * sr, len(X))
f.close()
write_audio(X, sr, os.path.basename(filename)[:-4] + '_filtered.wav')
os.chdir(pwd)
def spectrogram(x,
v,
mrat,
beta,
points,
orientations,
radius=None,
volume=None,
progress=False):
"""The spectrogram is built out of a spectrum for each given point in space"""
mutually_exclusive_args = [radius, volume]
if mutually_exclusive_args.count(None) != 1:
raise TypeError("Only provide one of radius or volume")
if radius is not None:
print('Warining: radius is depreciated')
volume = (4. / 3.) * np.pi * radius**3
ret = np.empty((points.shape[0], bin_edges.shape[0] - 1), dtype=np.float64)
if len(x) == 0:
ret[:, :] = 0
return ret
if progress:
printProgressBar(0, 1)
kdparts = spatial.cKDTree(x)
kdpoints = spatial.cKDTree(points)
effective_radius = volume**(1. / 3.) / 2.
radius_enhancement = 4
enhanced_radius = effective_radius * radius_enhancement
enhanced_volume = (2.0 * enhanced_radius)**3.0
local = kdpoints.query_ball_tree(kdparts, enhanced_radius, p=np.inf)
# Since we ignore particle weighting we need to correct counts by a certain factor
# assuming particles are uniformly distributed within the cell volume.
# We also assume the same cell volume throughout.
effective_volume = enhanced_volume / 8.0
for i, l in enumerate(local):
if progress:
printProgressBar(len(local) + i, 2 * len(local))
ret[i, :] = spectrum(v[l], mrat[l], 1. / (effective_volume * beta[l]),
orientations[i])
if progress:
printProgressBar(1, 1)
return ret
def main():
# Command Line Arguments
parser = argparse.ArgumentParser()
parser.add_argument("-i", "--input", const=str, nargs="?")
args = parser.parse_args()
log.info("Loading Data")
log.debug("Arguments")
log.debug(args)
file1 = open(args.input or "preprocessorfile.txt", "r")
data = file1.read()
globaldata = ["start"]
splitdata = data.split("\n")
splitdata = splitdata[:-1]
log.info("Processed Pre-Processor File")
log.info("Converting to readable format")
silentRemove("removal_points.txt")
for idx, itm in enumerate(splitdata):
printProgressBar(idx,
len(splitdata) - 1,
prefix="Progress:",
suffix="Complete",
length=50)
itm = itm.split(" ")
itm.pop(-1)
entry = itm
globaldata.append(entry)
globaldata = cleanNeighbours(globaldata)
outerpts = []
interiorpts = []
log.info("Point Classification")
for idx, itm in enumerate(globaldata):
printProgressBar(idx,
len(globaldata) - 1,
prefix="Progress:",
suffix="Complete",
length=50)
if idx > 0 and getFlag(idx, globaldata) == 2:
outerpts.append(idx)
elif idx > 0 and getFlag(idx, globaldata) == 1:
interiorpts.append(idx)
wallpts = getWallPointArray(globaldata)
for itm in wallpts:
inflatedWallPolygon(
globaldata, itm,
float(core.getConfig()["pseudowall"]["inflatedPolygonDistance"]),
interiorpts)
# print("Triangulating")
# interiorpts = convertPointToShapelyPoint(convertIndexToPoints(interiorpts,globaldata))
# interiorpts = MultiPoint(interiorpts)
# interiortriangles = triangulate(interiorpts)
# wallpts = convertPointToShapelyPoint(convertIndexToPoints(wallpts,globaldata))
# wallpts = Polygon2(wallpts)
# print("Generating Model")
# polygns = []
# fig, ax = plt.subplots()
# for idx,itm in enumerate(interiortriangles):
# printProgressBar(idx, len(interiortriangles) - 1, prefix = 'Progress:', suffix = 'Complete', length = 50)
# itm = itm.difference(wallpts)
# try:
# theshit = list(zip(*itm.exterior.xy))
# polygns.append(Polygon(theshit, True))
# except AttributeError:
# pass
# p = PatchCollection(polygns, cmap=matplotlib.cm.jet, alpha=0.4)
# colors = 100*np.random.rand(len(polygns))
# p.set_array(np.array(colors))
# ax.add_collection(p)
# print("Plotting")
# plt.show()
# xs, ys = [],[]
# mergedtriangles = cascaded_union(outertriangles)
# for triangle in outertriangles:
# xstemp,ystemp = triangle.exterior.xy
# print(xstemp,ystemp)
# xs,ys = mergedtriangles.exterior.xy
# fig, axs = plt.subplots()
# axs.fill(xs, ys, alpha=0.5, fc='r', ec='none')
# plt.show() #if not interactive.
# print("Set Flag")
# for idx,itm in enumerate(globaldata):
# if(idx > 0 and getFlag(idx,globaldata) == 1):
# globaldata = setFlags(idx,globaldata,60)
log.info("Done")
def MiniBatchGD(self, Epochs):
self.data = np.array(self.data)
N = 10000
n = 128
num_of_y = np.zeros(3)
printProgressBar(0,
Epochs - 1,
prefix='Training:',
suffix='Complete',
length=50)
for e in range(Epochs):
np.random.shuffle(self.data)
printProgressBar(e,
Epochs - 1,
prefix='Training:',
suffix='Complete',
length=50)
for j in range(0, N // n):
A = []
y = []
for i in range(j * n, j * n + n):
A.append(self.data[i][:5])
y.append(self.data[i][5])
A = np.array(A)
y = np.array(y)
loss = self.FourLayerNetwork(A, self.W1, self.W2, self.W3,
self.W4, self.b1, self.b2,
self.b3, self.b4, y, 0, n)
if (e % 25 == 0 or e == Epochs - 1):
np.random.shuffle(self.data)
correct = 0
for i in range(N):
A = self.data[i][:5]
y = self.data[i][5]
A = np.array(A)
classification = self.FourLayerNetwork(
A, self.W1, self.W2, self.W3, self.W4, self.b1,
self.b2, self.b3, self.b4, y, 1, n)
if (e == Epochs - 1):
y = int(y)
num_of_y[y] += 1
self.confusion[y][classification] += 1
if (classification == y):
correct += 1
accuracy = 100 * correct / N
self.epoch_plot.append(e)
self.accuracy_plot.append(accuracy)
self.loss_plot.append(loss)
print()
print('Accuracy: ', accuracy, '%')
#confusion matrix
print('Confusion')
for i in range(3):
for j in range(3):
conf = 100 * self.confusion[i][j] / num_of_y[i]
print(conf, end=' ')
print()
pickle.dump(self, open('DeepPong.txt', 'wb'))
def main():
# Command Line Arguments
parser = argparse.ArgumentParser()
parser.add_argument("-i", "--input", const=str, nargs="?")
args = parser.parse_args()
print("Loading Data")
file1 = open(args.input or "preprocessorfile_pointremoval.txt", "r")
data = file1.read()
globaldata = ["start"]
splitdata = data.split("\n")
splitdata = splitdata[:-1]
print("Processed Pre-Processor File")
print("Converting to readable format")
for idx, itm in enumerate(splitdata):
printProgressBar(idx,
len(splitdata) - 1,
prefix="Progress:",
suffix="Complete",
length=50)
itm = itm.split(" ") # Split the gaps
itm.pop(-1) # Remove last element
entry = itm
globaldata.append(entry)
file2 = open("removal_points2.txt", "r")
removalFlags = file2.read()
file2.close()
removalFlags = removalFlags.replace("\t", " ")
removalFlags = removalFlags.split("\n")
removalFlags.pop(-1)
removalFlags = [int(i) for i in removalFlags]
# print(1030 in removalFlags)
globaldata = cleanNeighbours(globaldata)
wallpoints = getWallPointArray(globaldata)
globaldata = addNewPoints(globaldata, removalFlags, 100, 1, wallpoints)
globaldata = cleanNeighbours(globaldata)
# The New Index (with bad points removed) || 5 --> 4
aliasArray = [0] * (len(globaldata))
# The Old Index from the new || 4 --> 5
reverseAliasArray = [0] * (len(globaldata))
count = 1
for individiualPoint in globaldata[1:]:
index = int(individiualPoint[0])
# print(index)
if index in removalFlags:
continue
else:
aliasArray[count] = index
reverseAliasArray[index] = count
count = count + 1
# print(globaldata)
# print(aliasArray)
# print(count)
newglobaldata = ["start"]
for i in range(1, count):
storage = []
aliasArrayIndex = aliasArray[i]
storage.append(i)
storage.append(globaldata[aliasArrayIndex][1])
storage.append(globaldata[aliasArrayIndex][2])
reverseAliasArrayIndex = reverseAliasArray[aliasArrayIndex]
if reverseAliasArrayIndex == 0:
left_point = 0
right_point = 0
else:
left_point = int(globaldata[aliasArrayIndex][3])
right_point = int(globaldata[aliasArrayIndex][4])
storage.append(reverseAliasArray[left_point])
storage.append(reverseAliasArray[right_point])
# The Flags
for i in range(5, 11):
storage.append(globaldata[aliasArrayIndex][i])
# The Neighbours
neighbourCount = 0
storage.append(0) # Temporary count of neighbours
# We are skipping the element that has total number of original
# neighbours
for neighbourIterate in globaldata[aliasArrayIndex][12:]:
if int(neighbourIterate) in removalFlags:
continue
else:
storage.append(reverseAliasArray[int(neighbourIterate)])
neighbourCount = neighbourCount + 1
storage[11] = neighbourCount
newglobaldata.append(storage)
# print(newglobaldata[1028:1034])
newglobaldata = cleanNeighbours(newglobaldata)
problempts = []
for individiualPoint in newglobaldata[1:]:
if int(individiualPoint[5]) != 1:
continue
index = int(individiualPoint[0])
checkConditionNumber(index, newglobaldata, aliasArray, 80, problempts)
problempts = list(dict.fromkeys(problempts))
# print(problempts)
with open("removal_points3.txt", "w") as text_file:
for item1 in problempts:
text_file.writelines(["%s " % item1])
text_file.writelines("\n")
newglobaldata.pop(0)
with open("preprocessorfile_pointremoval2.txt", "w") as text_file:
for item1 in newglobaldata:
text_file.writelines(["%s " % item for item in item1])
text_file.writelines("\n")
print("Data Converted")
size = len(data_t)
# A frame is time stamp : (startIndex,EndIndex)
for d in data_t:
if d[0].decode() == checker:
pass
else:
if checker is None:
checker = d[0].decode()
else:
frames[checker] = (last, i)
last = i - 1
checker = d[0].decode()
# input()
i += 1
printProgressBar(i, size + len(frames.keys()))
file = open('processed_' + os.path.basename(filename), 'w')
for frame in frames.keys():
start, stop = frames[frame]
now = np.array(data[start:stop])
if len(now) < 10:
now_ = now
else:
now_ = now[0:now.size:3] #down sampling
sum_sqr_z = 0
arr = []
for acc_row in now_:
r_x, r_y, r_z, x, y, z = accelerometer_correction(
acc_row[0], acc_row[1], acc_row[2])
sum_sqr_z += z * z
def main():
# Command Line Arguments
parser = argparse.ArgumentParser()
parser.add_argument("-i", "--input", const=str, nargs="?")
args = parser.parse_args()
print("Loading Data")
file1 = open(args.input or "preprocessorfile.txt", "r")
data = file1.read()
globaldata = ["start"]
splitdata = data.split("\n")
splitdata = splitdata[:-1]
print("Processed Pre-Processor File")
print("Converting to readable format")
for idx, itm in enumerate(splitdata):
printProgressBar(idx,
len(splitdata) - 1,
prefix="Progress:",
suffix="Complete",
length=50)
itm = itm.split(" ")
itm.pop(-1)
entry = itm
globaldata.append(entry)
globaldata = core.cleanNeighbours(globaldata)
wallpoints = core.getWallPointArray(globaldata)
wallpointsIndex = core.getWallPointArrayIndex(globaldata)
# result = core.findAverageWallDistance(globaldata,wallpointsIndex)
# print(result)
# temp.writeNormalsToText(globaldata)
# temp.writeConditionValues(globaldata)
temp.writeSrikanthStyle(globaldata)
exit()
wallpointsData = core.generateWallPolygons(wallpoints)
while True:
ptidx = input("Which point do you want to check? ")
if ptidx == "exit":
break
ptidx = int(ptidx)
print("Point Index:", ptidx)
print("Point Co ordinate:", core.getPointXY(ptidx, globaldata))
flag = core.getFlag(ptidx, globaldata)
flag = int(flag)
if flag == 0:
flagd = "Wall Point"
elif flag == 1:
flagd = "Interior Point"
else:
flagd = "Outer Point"
print("Point Type:", flagd)
nbhs = core.getNeighbours(ptidx, globaldata)
print("Total Number of Neighbours:", len(nbhs))
print("Neighbour Array")
print(nbhs)
if (flag == 0):
print(core.getConditionNumberLegacy(ptidx, globaldata))
xpos = core.getXPosPoints(ptidx, globaldata)
xneg = core.getXNegPoints(ptidx, globaldata)
print("xpos", len(xpos), "xneg", len(xneg))
else:
print(core.getConditionNumberLegacy(ptidx, globaldata))
xpos = core.getDXPosPointsLegacy(ptidx, globaldata)
xneg = core.getDXNegPointsLegacy(ptidx, globaldata)
ypos = core.getDYPosPointsLegacy(ptidx, globaldata)
yneg = core.getDYNegPointsLegacy(ptidx, globaldata)
print("xpos", len(xpos), "xneg", len(xneg), "ypos", len(ypos),
"yneg", len(yneg))
def saveTreeFeaturesTrain(text, word_list, depth):
sents = sent_tokenize(text)
flag = 0
counts = Counter()
progress = 0
totalen = len(sents)
features = []
ignore_deps = ["punct", "cc"]
featvocab = set({"NaN"})
for sent in sents:
parsed_sents = nlp.annotate(sent, properties={
'annotators': 'tokenize,ssplit,pos,depparse',
'outputFormat': 'json'
})
for parsed_sent in parsed_sents['sentences']:
for item in parsed_sent['basicDependencies']:
if(item["dependentGloss"] in word_list):
feature = {}
#print(sent)
#print(parsed_sent['basicDependencies'])
#the first feature is always the word itself
feature["y"] = item["dependentGloss"]
gov_index = item["governor"]
#if we reach root break
if(gov_index==0):
continue
gov = parsed_sent['tokens'][gov_index-1]
if(gov_index!=gov['index']):
raise Exception('Governor Index dont match')
gov_pos = gov['pos']
dep = "p_"+item["dep"]
feature[dep] = gov_pos
featvocab.add(gov_pos)
for node_item in parsed_sent['basicDependencies']:
if(node_item["governor"]==gov_index and node_item["dependent"]!=item['dependent']):
#print(node_item)
dep = node_item["dep"]
if(dep in ignore_deps):
continue
dep_index = node_item["dependent"]
dep_item = parsed_sent['tokens'][dep_index-1]
if(dep_index!=dep_item['index']):
raise Exception('Dependency Index dont match')
pos = dep_item["pos"]
feature[dep] = pos
featvocab.add(pos)
#features = getFeatures(1, depth, feature, item, parsed_sent)
features.append(feature)
#flag+=1
#break
#params[word["originalText"]][]
#if(flag==6):
# break
progress += 1
printProgressBar(progress, totalen)
#print(featvocab)
with open("dataset_dtree_train.pkl", 'wb') as modelfile:
pickle.dump([features, featvocab], modelfile)
f = ff.FortranFile(join(p.particle, "c.outflowing_" + str(n + 1) + ".dat"))
# for each half timestep
for i in range(2 * p.para['nt']):
try:
mrat = f.readReals()
except ff.NoMoreRecords:
break
max_i = i
beta_p = f.readReals()
tags = f.readReals()
# Each of the arrays must have the same length
assert len(mrat) == len(beta_p) and len(mrat) == len(tags)
# If that length is zero than there was no outflow
if len(mrat) == 0:
continue
# for each macro particle
for m, b, t in zip(mrat, beta_p, tags):
if t != 0:
part_out[i] += 1 / (b * p.para['beta'])
mass_out[i] += kg_per_amu / m * 1 / (p.para['beta'] * b)
printProgressBar(n + 1, p.para['num_proc'])
print(p.para['dt'])
plt.plot(part_out)
plt.show()
import sys
from passlib.hash import bcrypt
choice = input('check common passwords or whole english dictionary? (p/d): ')
if (choice != "p" and choice != "d"):
sys.exit('Invalid Option')
passwords = (choice == "p")
if (passwords):
text_file = open("passwords.txt", "r")
else:
text_file = open("words.txt", "r")
words = text_file.read().splitlines()
hash = input('hash to crack: ')
length = len(words)
correct_word = ""
for (index, word) in enumerate(words):
printProgressBar(index, length, prefix = 'Progress:', suffix = 'Complete', length = columns)
correct = bcrypt.verify(word, hash)
if (correct):
correct_word = word
print()
break
print("correct word is:", correct_word)
