def main():
# path = 'ExampleData/input_files/'
# endPath = 'ExampleData/test/'
path, endPath = getFilePath()
reader_fgi = list(csv.reader(open(path + "fGI_stats.csv", "rt", encoding="ascii"), delimiter=","))
reader_core = list(csv.reader(open(path + "Core_attfGI.csv", "rt", encoding="ascii"), delimiter=","))
genomeListing = list(open(path + "db.txt", "r"))
genomeClusterDict = pickle.load(open(path + "genomeCluster.dict", "rb"))
genomeLocusDict = pickle.load(open(path + "genomeLocus.dict", "rb"))
coreDict, fgiDict = createCoreClusterDict(reader_core)
# genome = 'E00002'
genomeIdDict = {}
index = 3
for genome2 in genomeListing:
if "\n" in genome2:
genome2 = genome2[0:-1]
genomeIdDict[genome2] = index
index += 1
for genome in genomeIdDict:
genomeDict = createfgiInsertDict(reader_fgi, genome)
referenceList = createfGIFeatures(
genomeDict, coreDict, fgiDict, genomeClusterDict, genomeLocusDict, genome, genomeIdDict[genome]
)
writeFile(endPath, genome, referenceList)
genomeDict = createfgiInsertDict(reader_fgi, genome)
referenceList = createfGIFeatures(
genomeDict, coreDict, fgiDict, genomeClusterDict, genomeLocusDict, genome, genomeIdDict[genome]
)
writeFile(endPath, genome, referenceList)
def _remove_persistent(put_queue_name, get_queue_name, pickled, pos_name):
try:
try:
with open(pos_name, 'r') as pn:
get_pos = int(pn.read())
except FileNotFoundError:
get_pos = 0
os.remove(pos_name)
method = 'ab' if pickled else 'a'
with open(put_queue_name, method) as put_queue, \
open(get_queue_name, _file_method('r', pickled)) as f:
if pickled:
for _ in range(get_pos):
pickle.load(f)
while True:
try:
pickle.dump(pickle.load(f), put_queue)
except EOFError:
break
else:
for _ in range(get_pos):
next(f)
for line in f:
put_queue.write(line.strip('\n') + '\n')
_touch(get_queue_name, pickled=pickled)
except (Empty, FileNotFoundError):
pass
def _load_data(datadir, tetrodes):
import os
import re
import pickle as pkl
filelist = os.listdir(datadir)
filelist.sort()
reg = [ re.search('(\w+)_(\w+).(\w+)', filename) for filename in filelist ]
reg = [ r for r in reg if r != None ]
ext = ['bhv', 'cls', 'syn', 'ons']
data = dict.fromkeys(ext)
data['cls'] = {}
# Load the data files into data dictionary
for r in reg:
if r.group(3) in ['bhv', 'syn', 'ons']:
file = os.path.join(datadir, r.group(0))
with open(file,'r') as f:
data[r.group(3)] = pkl.load(f)
elif r.group(3) == 'cls':
for tetrode in tetrodes:
file = '%s/%s.%s' % (datadir, r.group(0), tetrode)
with open(file,'r') as f:
data['cls'].update({tetrode:pkl.load(f)})
# Checking to make sure the data files were loaded
if None in data.viewvalues():
for key, value in data.iteritems():
if value == None:
raise Exception, '%s file wasn\'t loaded properly' % key
return data
def test_pickle_model():
tmpdir = tempfile.mkdtemp()
pickle_file = os.path.join(tmpdir, 'stanmodel.pkl')
model_code = 'parameters {real y;} model {y ~ normal(0,1);}'
m = pystan.StanModel(model_code=model_code, model_name="normal1",
save_dso=False)
module_name = m.module.__name__
with open(pickle_file, 'wb') as f:
pickle.dump(m, f)
del m
del sys.modules[module_name]
with open(pickle_file, 'rb') as f:
m = pickle.load(f)
assert m.model_name == "normal1"
m = pystan.StanModel(model_code=model_code, model_name="normal2")
module_name = m.module.__name__
module_filename = m.module.__file__
with open(pickle_file, 'wb') as f:
pickle.dump(m, f)
del m
del sys.modules[module_name]
with open(pickle_file, 'rb') as f:
m = pickle.load(f)
assert m.model_name == "normal2"
assert m.module is not None
assert module_filename != m.module.__file__
fit = m.sampling()
y = fit.extract()['y']
assert len(y) == 4000
def main():
pickle_folder = "../mir_1k/pickles"
pickle_folders_to_load = [f for f in os.listdir(pickle_folder) if "__beat_spec.pick" in f]
pickle_folders_to_load = sorted(pickle_folders_to_load)
sdr_type = "background"
fits = []
sdrs = []
for pick in pickle_folders_to_load:
pick = pick.replace("__beat_spec.pick", "")
beat_spec_path = join(pickle_folder, pick + "__beat_spec.pick")
beat_spec = pickle.load(open(beat_spec_path, "rb"))
entropy, log_mean = beat_spectrum_prediction_statistics(beat_spec)
fit_X = [entropy, log_mean]
fits.append(fit_X)
sdrs_name = join(pickle_folder, pick + "__sdrs.pick")
sdr_vals = pickle.load(open(sdrs_name, "rb"))
cur_sdr = sdr_vals[sdr_type][0]
sdrs.append(cur_sdr)
fits = np.array(fits)
sdrs = np.array(sdrs).reshape(-1, 1)
knn = neighbors.KNeighborsRegressor(5, weights="distance")
scores = cross_validation.cross_val_predict(knn, fits, sdrs, cv=10, verbose=1)
print("Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2))
def main():
t_c_matrix = open("t_c_matrix", "r")
t_set = open("t_set", "r")
word_dict = defaultdict(float)
word_context_matrix = pickle.load(t_c_matrix)
word_set = pickle.load(t_set)
word_set = list(word_set)
# print word_set.index("Spain")
# print word_set.index("England")
# print word_set.index("Athens")
vec1 = word_context_matrix[word_set.index("Spain")]
vec2 = word_context_matrix[word_set.index("England")]
vec3 = word_context_matrix[word_set.index("Athens")]
vec = vec1 - vec2 + vec3
for index, word in enumerate(word_set):
sim = cos_sim(word_context_matrix[index], vec)
word_dict[word] = sim
count = 0
for word, sim in sorted(word_dict.items(), key = lambda x:-x[1]):
print word + "\t" + str(sim)
count += 1
if count == 9:
break
def TF_IDF():
print('Doing TF_IDF', file=sys.stderr)
global TFIDF, docWeight, index
if os.path.isfile('TFIDF.dat') and os.path.isfile('docWeight.dat') and os.path.isfile('index.dat'):
f = open('TFIDF.dat', 'rb')
TFIDF = pickle.load(f)
f.close()
f = open('docWeight.dat', 'rb')
docWeight = pickle.load(f)
f.close()
f = open('index.dat', 'rb')
index = pickle.load(f)
f.close()
else:
print('.dat not exist, generating', file=sys.stderr)
TFIDF = {}
docCnt = len(docSize)
avgSize = 0
index = [[] for i in range(docCnt)]
for i in range(docCnt):
avgSize += docSize[i]
avgSize /= docCnt
docWeight = [0 for i in range(docCnt)]
para_b = 0.7 # tuning
d = [(1 - para_b + para_b*docSize[i]/avgSize) for i in range(docCnt)]
for i in invIndexUnigram: # word id
IDF = math.log( docCnt / len(invIndexUnigram[i]) )
TFIDF[i] = {}
for j in invIndexUnigram[i]: # doc id
v = (invIndexUnigram[i][j] / d[j]) * IDF
TFIDF[i][j] = v
docWeight[j] += v * v
index[j].append(i)
for i in invIndexBigram: # word id
IDF = math.log( docCnt / len(invIndexBigram[i]) )
TFIDF[i] = {}
for j in invIndexBigram[i]: # doc id
v = (invIndexBigram[i][j] / d[j]) * IDF
TFIDF[i][j] = v
docWeight[j] += v * v
index[j].append(i)
f = open('TFIDF.dat', 'wb')
pickle.dump(TFIDF, f)
f.close()
f = open('docWeight.dat', 'wb')
pickle.dump(docWeight, f)
f.close()
f = open('index.dat', 'wb')
pickle.dump(index, f)
f.close()
printTime()
def dump_file(self, content):
counter = 0
while True:
while True:
try:
fl = open(self.PATH+'.tmp', 'wb')
pickle.dump(content, fl)
fl.close()
fl = open(self.PATH+'.tmp','rb')
h2 = pickle.load(fl)
fl.close()
assert h2 == content
break
except:
#print '\nThere was an error dumping the history!\n'\
#'This happened %d times so far, trying again...'%(counter)
counter+=1
try:
if os.path.exists(self.PATH):
os.remove(self.PATH)
os.rename(self.PATH+'.tmp',self.PATH)
fl = open(self.PATH,'rb')
h2 = pickle.load(fl)
fl.close()
assert h2 == content
break
except:
pass
def sort_warheroes_list():
data_list = pickle.load(open("war_base", "rb"))
war_base_url = "http://www.warheroes.ru/hero/hero.asp?Hero_id="
session = requests.session()
for unit in data_list:
num = unit[0]
title = unit[1]
print(unit)
id = str(num)
war_url = war_base_url + id
is_get_res = False
try_count = 0
while not is_get_res and try_count < 10:
try:
res = session.get(war_url)
war_data = res.text
is_get_res = True
except:
try_count += 1
time.sleep(1)
if is_otrs_exists(war_data):
try:
print(" - otrs exists")
good_list = pickle.load(open("free_war_base", "rb"))
good_list.append(unit)
pickle.dump(good_list, open("free_war_base", "wb"))
except:
tools.debug_print("Bad pickle save3. "+id)
else:
try:
bad_list = pickle.load(open("nonfree_war_base", "rb"))
bad_list.append(unit)
pickle.dump(bad_list, open("nonfree_war_base", "wb"))
except:
tools.debug_print("Bad pickle save3. "+id)
def main():
"""loads data, trains model, tests model
Inputs:
file: binary file containing sparse numpy array with text features
file: binary file containing pandas dataframe with training labels
Outs:
print: classification report of classifier performance
"""
# Load training labels and text features
chdir("../pickles")
with open("word_counts.pkl", "rb") as f:
X = pickle.load(f)
with open("training_labels.pkl", "rb") as f:
y = pickle.load(f)
y = np.ravel(y["sponsored"])
# Create train and test splits
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
# Create and train model
clf = MultinomialNB()
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
print(classification_report(y_test, y_pred))
def test_pickleable():
interval = pybedtools.create_interval_from_list(
['chr1', '1', '100', 'asdf'])
fn = pybedtools.BedTool._tmp()
import pickle
out = open(fn, 'w')
pickle.dump(interval, out)
out.close()
new_interval = pickle.load(open(fn))
assert str(interval) == str(new_interval)
interval = pybedtools.create_interval_from_list(
['chr1', '1', '100'])
fn = pybedtools.BedTool._tmp()
import pickle
out = open(fn, 'w')
pickle.dump(interval, out)
out.close()
new_interval = pickle.load(open(fn))
assert str(interval) == str(new_interval)
interval = pybedtools.create_interval_from_list(
"chr2L . UTR 41 70 0 + . ID=mRNA:xs2:UTR:41-70;Parent=mRNA:xs2;".split('\t'))
fn = pybedtools.BedTool._tmp()
import pickle
out = open(fn, 'w')
pickle.dump(interval, out)
out.close()
new_interval = pickle.load(open(fn))
assert str(interval) == str(new_interval)
def sortFilesByPlayback(self):
originalFiles=[]
historyMovFile = self.getHistoryMovFileName()
if(os.path.exists(historyMovFile)):
infile = open(historyMovFile,"rb")
originalFiles = pickle.load(infile)
infile.close()
finalFiles = []
favoriteFiles = []
favoriteFileName = self.getFavoriteFileName()
if(os.path.exists(favoriteFileName)):
infile = open(favoriteFileName,"rb")
favoriteFiles = pickle.load(infile)
infile.close()
favoriteLen = len(favoriteFiles)
originalLen = len(originalFiles)
if favoriteLen == 0 or originalLen == 0:
return
else:
for i in range(favoriteLen):
try:
finalFiles.append(favoriteFiles[i][1])
originalFiles.remove(favoriteFiles[i][1])
except:
pass
finalFiles = finalFiles + originalFiles
## print finalFiles
outfile = open(historyMovFile,"wb")
pickle.dump(finalFiles, outfile,2)
outfile.close()
self.showAllImage()
def search_key(index, keyword):
""" search museum dictionary """
#load dictionary
dict_file = open('data.pkl', 'rb')
dictionar = pickle.load(dict_file)
dict_file.close()
#load headers
head_file = open('headers.hd', 'rb')
header = pickle.load(head_file)
head_file.close()
#find keyword
muzee = []
if keyword == "":
for i in range(len(dictionar[header[0]])):
muzee.append({'cod': dictionar[header[0]][i],
'judet': dictionar[header[2]][i].decode(encoding="UTF-8"),
'nume': dictionar[header[3]][i].decode(encoding="UTF-8"),
'lat': sub(',', '.', dictionar[header[35]][i]),
'lng': sub(',', '.', dictionar[header[36]][i])})
else:
for i in range(len(dictionar[header[index]])):
new_word = dictionar[header[index]][i].decode(encoding='UTF-8').lower()
keyword = keyword.lower()
if keyword in new_word:
muzee.append({'cod': dictionar[header[0]][i],
'judet': dictionar[header[2]][i].decode(encoding="UTF-8"),
'nume': dictionar[header[3]][i].decode(encoding="UTF-8"),
'lat': sub(',', '.', dictionar[header[35]][i]),
'lng': sub(',', '.', dictionar[header[36]][i])})
return muzee
def test_feature_extraction():
np.random.seed(33)
print "Testing implementation of feature extraction..."
import featureLearner as fl
k = 5
learner = fl.FeatureLearner(k)
learner.trained = True
image = util.loadTrainImages()[33]
# load test centroids and features
testDat = open('data/kmeans_test.npy','r')
centroids = pickle.load(testDat)
testDat.close()
testDat = open('data/features_test.npy','r')
features = pickle.load(testDat)
testDat.close()
learner.centroids = centroids
studentFeats = learner.extractFeatures(image)
assert isinstance(studentFeats,np.ndarray),"Features should be in an numpy array"
assert studentFeats.shape==features.shape,"Dimension mismatch"
studentFeatsList = studentFeats.tolist()
if np.abs(np.sum(features)-np.sum(studentFeats)) > 1e-3:
print "Feature mismatch, test failed"
return
if np.abs(np.sqrt(np.sum(features**2)) - np.sqrt(np.sum(features**2))) > 1e-3:
print "Feature mismatch, test failed"
return
print "Feature extraction test passed"
def save_settings(self, server, port, path, username, password):
# load config file or create a new one
if os.path.isfile(self._config_path):
print "Found ftp config file - saving new settings!"
with open(self._config_path) as f:
l = pickle.load(f)
l['server'] = str(server)
l['port'] = str(port)
l['path'] = str(path)
l['username'] = str(username)
l['pass'] = str(password)
with open(self._config_path, 'w') as f:
pickle.dump(l,f)
print "Saved successful ftp-config!"
else:
print "Did not found ftp config file, creating new one!"
with open(self._config_path, 'w') as f:
pickle.dump(
{
'server': server,
'port': port,
'path': path,
'username': username,
'pass': password,
}, f)
with open(self._config_path) as f:
l = pickle.load(f)
print "Saved successful ftp-config!"
def adauga_muzeu(muzeu):
"""adauga intrare noua in dictionar"""
#load dictionary
dict_file = open('data.pkl', 'rb')
dictionar = pickle.load(dict_file)
dict_file.close()
#load headers
head_file = open('headers.hd', 'rb')
header = pickle.load(head_file)
head_file.close()
request.args.get('nume')
#read info from form
if request.method == 'GET':
target_fields = {3: 'nume', 2: 'judet', 17: 'descriere', 35: 'lat', 36: 'lng'}
for i in range(len(header)):
if i in target_fields.keys():
dictionar[header[i]].append(request.args.get(target_fields[i]))
elif i == 0:
dictionar[header[i]].append(get_next_code(dictionar, header))
else:
dictionar[header[i]].append("")
output = open('data.pkl', 'wb')
pickle.dump(dictionar, output)
output.close()
return redirect("/")
def read_encrypted_logs(username, filelogs):
names = []
my_key = RSA.importKey(open(username + '.pri').read())
cipher = PKCS1_OAEP.new(my_key, SHA256.new())
for log in filelogs:
file_log_hash = SHA256.new()
with open(log, 'rb') as input:
dictlog = pickle.load(input)
sig = pickle.load(input)
owner = dictlog['owner']
with open(owner + '.dsapub', 'rb') as input:
owner_msk = pickle.load(input)
length = len(dictlog)
with open(log, 'rb') as outfile:
picklelog = outfile.read(length)
file_log_hash.update(picklelog)
if not owner_msk.verify(file_log_hash.digest(), sig):
print('invalid file')
if username in dictlog:
block = dictlog[username]
block.decrypt_permission_block(cipher)
name = decrypt_filename(block.get_file_encryption_key(), log[0:-5])
names.append(name)
return names
def rename_directory(username, old_filename, new_filename):
filelog = oldfilename + '.flog'
with open(filelog, 'rb') as input:
log = pickle.load(input)
sig = pickle.load(input)
block = log[username]
key = RSA.importKey(open(username + '.pri').read())
cipher = PKCS1_OAEP.new(key, SHA256.new())
block.decrypt_permission_block(cipher)
encrypted_new_filename = encrypt_filename(block.get_file_encryption_key(), new_filename)
log['encrypted_name'] = encrypted_new_filename
new_filelog = encrypted_new_filename + '.dlog'
with open(new_filelog, 'wb') as outfile:
pickle.dump(log)
length = len(log)
with open(new_filelog, 'rb') as infile:
pickleload = infile.read(length)
file_log_hash = SHA256.new()
file_log_hash.update(picklelog)
with open(username + '.dsa', 'rb') as infile:
owner_msk = pickle.load(infile)
k = random.StrongRandom().randint(1,owner_msk.q-1)
sig = owner_msk.sign(file_log_hash.digest(), k)
with open(new_filelog, 'a+b') as outfile:
pickle.dump(sig, outfile, -1)
return encrypted_new_filename
def main():
parser = buildArgsParser()
args = parser.parse_args()
ocr = args.ocr
validation_file = args.validation_file
train_file = args.train_file
test_file = args.test_file
is_vizu = args.is_vizu
if not os.path.isfile(ocr):
parser.error("-ocr '{0}' doit être un fichier!".format(os.path.abspath(ocr)))
if not os.path.isfile(train_file):
parser.error("-train '{0}' doit être un fichier!".format(os.path.abspath(train_file)))
if not os.path.isfile(test_file):
parser.error("-test '{0}' doit être un fichier!".format(os.path.abspath(test_file)))
if not os.path.isfile(validation_file):
parser.error("-valider '{0}' doit être un fichier!".format(os.path.abspath(validation_file)))
with open(train_file) as f:
train = pickle.load(f)
with open(test_file) as f:
test = pickle.load(f)
reseau_de_neurones = recognize_characters(ocr, train, test, validation_file)
validate_ocr(reseau_de_neurones, validation_file)
if is_vizu:
# Visualisation des prédictions sur l'ensemble de test
show_recognized_characters(reseau_de_neurones, test)
def share_directory(other_username, dlog):
with open(dlog, 'rb') as input:
log = pickle.load(input)
userList = log['users']
userList.append(other_username)
owner_block = log[owner]
owner = log['owner']
key = RSA.importKey(open(owner + '.pri').read())
cipher = PKCS1_OAEP.new(key, SHA256.new())
owner_block.decrypt_permission_block(cipher)
file_aes_key = owner_block.get_file_encryption_key()
file_dsa_key = None
user_block = AccessBlock(file_aes_key, file_dsa_key)
other_key = RSA.importKey(open(other_username + '.pub').read())
other_cipher = PKCS1_OAEP.new(other_key, SHA256.new())
user_block.encrypt_permission_block(other_cipher)
log[other_username] = user_block
file_log_hash = SHA256.new()
with open(filelog, 'wb') as infile:
pickle.dump(log, infile, -1)
length = len(log)
with open(filelog, 'rb') as outfile:
picklelog = outfile.read(length)
file_log_hash.update(picklelog)
with open(owner + '.dsa', 'rb') as infile:
owner_msk = pickle.load(infile)
k = random.StrongRandom().randint(1,owner_msk.q-1)
sig = owner_msk.sign(file_log_hash.digest(), k)
with open(filelog, 'a+b') as outfile:
pickle.dump(sig, outfile, -1)
def __init__(self, pos_doc_file = "positive.p",
neg_doc_file = "negative.p",
num_doc_pos_file = "num_doc_pos.txt",
num_doc_neg_file = "num_doc_neg.txt",
j = -1):
"""This method initializes and trains the Naive Bayes Sentiment Classifier. If a
cache of a trained classifier has been stored, it loads this cache. Otherwise,
the system will proceed through training. After running this method, the classifier
is ready to classify input text."""
self.positive_words = {}
self.negative_words = {}
#for presence
self.num_doc_pos = 0
self.num_doc_neg = 0
#If the pickled files exist, then load the dictionaries into memory.
if os.path.exists(pos_doc_file):
self.positive_words = pickle.load(open(pos_doc_file, "rb"))
if os.path.exists(num_doc_pos_file):
self.num_doc_pos = int(pickle.load(open(num_doc_pos_file, "rb")))
if os.path.exists(neg_doc_file):
self.negative_words = pickle.load(open(neg_doc_file, "rb"))
if os.path.exists(num_doc_neg_file):
self.num_doc_neg = int(pickle.load(open(num_doc_neg_file, "rb")))
#If the pickled files do not exist, then train the system.
else:
self.train(pos_doc_file, neg_doc_file, num_doc_pos_file, num_doc_neg_file, j)
def preprocess(article_file, lable_file, k):
features = pickle.load(open(article_file))
features = np.array(features)
# transform non-numerical labels (as long as they are hashable and comparable) to numerical labels
lables = pickle.load(open(lable_file))
le = preprocessing.LabelEncoder()
le.fit(lables)
lables = le.transform(lables)
# print le.inverse_transform([0])
### text vectorization--go from strings to lists of numbers
vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5, min_df=1,
stop_words='english')
features_train_transformed = vectorizer.fit_transform(features)
# selector : SelectPercentile
selector = SelectPercentile(f_classif, percentile=k)
selector.fit(features_train_transformed, lables)
# selector : chi2
# selector = SelectPercentile(score_func=chi2)
# selector.fit(features_train_transformed, lables)
features_train_transformed = selector.transform(features_train_transformed).toarray()
return features_train_transformed, lables, vectorizer, selector, le, features
def build_assigned_from_existing(assigned_dir, clusters, regions, nrand):
"""
Loads results produced from above analysis from saved file - can either be switched to
piclking or removed
assigned_dir - location of files
clusters - name of experiment
regions - list of genic regions
nrand - number of shuffled datsets to look for
"""
CLUS_regions = {}
for region in regions:
CLUS_regions[region]={}
for n in range(nrand):
CLUS_regions[region]['rand']={}
for region in regions:
bedfile = os.path.join(assigned_dir, "%s.%s.real.BED" %(clusters, region))
CLUS_regions[region]['real'] = pybedtools.BedTool(bedfile)
for n in range(nrand):
randbedfile = os.path.join(assigned_dir, "%s.%s.rand.%s.BED" %(clusters, region, str(n)))
CLUS_regions[region]['rand'][n] = pybedtools.BedTool(randbedfile)
try:
sizes = pickle.load(open(os.path.join(assigned_dir, "%s.sizes.pickle" %(clusters)), 'rb'))
except:
sizes=[1,1,1,1,1]
try:
Gsizes = pickle.load(open(os.path.join(assigned_dir, "Gsizes.pickle"), 'rb'))
except:
Gsizes=[1,1,1,1,1]
return CLUS_regions, sizes, Gsizes
def make_dewpoint(directory):
"""Makes an hourly timeseries of dewpoint temperatures."""
# open the 3-hr temperature and daily relative humidity files
tempfile = '{}/average_temperature'.format(directory)
hfile = '{}/average_humidity'.format(directory)
dewfile = '{}/average_dewpoint'.format(directory)
if not os.path.isfile(dewfile):
with open(tempfile, 'rb') as f: ts, Ts = zip(*pickle.load(f))
with open(hfile, 'rb') as f: ts, RHs = zip(*pickle.load(f))
# aggregate 3-hr temperatures to daily and get the Tmin
Tmins = [min(Ts[i:i+8]) for i in range(0, len(Ts), 8)]
Ts = [sum(Ts[i:i+8]) / 8 for i in range(0, len(Ts), 8)]
# calculate the daily dewpoint
dewpoints = calculate_dewpoint(numpy.array(Ts), numpy.array(RHs))
dewpoints = [(t, min(Tm, T)) for t, T, Tm in zip(ts, dewpoints, Tmins)]
with open(dewfile, 'wb') as f: pickle.dump(dewpoints, f)
def load_data():
"""Loads movie_data, cust_data, and answers from pickles.
Returns:
The tuple (movie_data, cust_data, answers) with the objects loaded from
their pickles.
"""
# load movie data cache
if isfile(CACHE_LOC + MOVIE_PICKLE):
with open(CACHE_LOC + MOVIE_PICKLE, 'rb') as movie_file:
movie_data = load(movie_file)
else:
movie_data = loads(urlopen(CACHE_URL + MOVIE_PICKLE).read())
# load customer data cache
if isfile(CACHE_LOC + CUSTOMER_PICKLE):
with open(CACHE_LOC + CUSTOMER_PICKLE, 'rb') as cust_file:
cust_data = load(cust_file)
else:
cust_data = loads(urlopen(CACHE_URL + CUSTOMER_PICKLE).read())
# load answers
if isfile(CACHE_LOC + ANSWER_PICKLE):
with open(CACHE_LOC + ANSWER_PICKLE, 'rb') as answer_file:
answers = load(answer_file)
else:
answers = loads(urlopen(CACHE_URL + ANSWER_PICKLE).read())
return(movie_data, cust_data, answers)
def load(filename):
"""
Restores a PPSD instance from a file.
Automatically determines whether the file was saved with compression
enabled or disabled.
:type filename: str
:param filename: Name of file containing the pickled PPSD object
"""
# identify bzip2 compressed file using bzip2's magic number
bz2_magic = b'\x42\x5a\x68'
with open(filename, 'rb') as file_:
file_start = file_.read(len(bz2_magic))
if file_start == bz2_magic:
# In theory a file containing random data could also start with the
# bzip2 magic number. However, since save() (implicitly) uses
# version "0" of the pickle protocol, the pickled data is
# guaranteed to be ASCII encoded and hence cannot start with this
# magic number.
# cf. http://docs.python.org/2/library/pickle.html
#
# due to an bug in older python version we can't use with
# http://bugs.python.org/issue8601
file_ = bz2.BZ2File(filename, 'rb')
ppsd = pickle.load(file_)
file_.close()
else:
with open(filename, 'rb') as file_:
ppsd = pickle.load(file_)
return ppsd
def line_position(): global black_line, white_line, range_col # load default values from files try: with open(file_b, 'rb') as f: black_line = pickle.load(f) except: black_line = [0] * 5 try: with open(file_w, 'rb') as f: white_line = pickle.load(f) except: white_line = [0] * 5 try: with open(file_r, 'rb') as f: range_col = pickle.load(f) except: range_col = [0] * 5 curr = get_sensorval() diff_val = list(map(operator.sub, curr, white_line)) curr_pos = 0 percent_black_line = [0] * 5 for i in range(5): percent_black_line[i] = diff_val[i] * 100 / range_col[i] curr_pos += percent_black_line[i] * multp[i] return curr_pos
def plot_TP():
with open ('p_files/ROC_table_' + str(seg) + '_pc' + str(p) + '_k' + str(k) + '.p', 'rb') as f:
ROC_table = pickle.load(f)
with open ('p_files/species_stats.p', 'rb') as f:
species_table = pickle.load(f)
with open ('p_files/species.p', 'rb') as f:
species_name = pickle.load(f)
xes = []#[item['number'] for item in ROC_table.values()]
yes = []#[item['fp'] for item in ROC_table.values()]
label = []
low_TP = []
for specie in ROC_table:
xes.append(species_table[specie])
yes.append(ROC_table[specie]['tp_rate'])
label.append(specie)
if float(ROC_table[specie]['tp_rate']) < 0.3 and species_table[specie] > 100:
#print(ROC_table[specie]['tp_rate'])
low_TP.append((specie, ROC_table[specie]['tp']))
fig, ax = plt.subplots()
plt.subplots_adjust(bottom=0.1)
ax.plot([0,max(xes)],[0.3,0.3], ls="--")
ax.scatter(xes, yes, marker = '.')
for i, txt in enumerate(label):
if txt in interesting:
#if float(yes[i]) < 0.3 and xes[i] > 100 :
#print(txt)
#ax.annotate(parser.get_specie_name('../data/train/', str(species_name[txt][0]) + '.xml'), (xes[i],yes[i]))
ax.annotate(txt, (xes[i],yes[i]))
plt.suptitle('False Positive', fontsize = 14)
# ax.set_xlabel('Principal Components')
# ax.set_ylabel('Percentage of Variance')
plt.show()
def averages_if_less(seg, p, k, n):
with open ('p_files/ROC_table_' + str(seg) + '_pc' + str(p) + '_k' + str(k) + '.p', 'rb') as f:
ROC_table = pickle.load(f)
with open ('p_files/species_stats.p', 'rb') as f:
species_table = pickle.load(f)
ROC_table = dict((k, ROC_table[k]) for k in ROC_table.keys() if species_table[k] > n)
#print(len(ROC_table))
#sys.exit(2)
recall_sum = sum(float(item['tp_rate']) for item in ROC_table.values())
recall_sum_percentage = recall_sum * 100.0
recall_average = recall_sum_percentage / len(ROC_table)
#print('Average of recall for ', seg, 'with ', p, ' PCs and ', k, ' neighbours ', recall_average)
precision_sum = sum(float(item['precision']) for item in ROC_table.values())
precision_sum_percentage = precision_sum * 100.0
precision_average = precision_sum_percentage / len(ROC_table)
#print('Average of precision for ', seg, 'with ', p, ' PCs and ', k, ' neighbours ', precision_average)
f_sum = sum(float(item['f_measure']) for item in ROC_table.values())
f_sum_percentage = f_sum * 100.0
f_average = f_sum_percentage / len(ROC_table)
#print('Average of F-measure for ', seg, 'with ', p, ' PCs and ', k, ' neighbours ', f_average)
tp_sum = sum(float(item['tp']) for item in ROC_table.values())
all_instances = sum(float(item['number']) for item in ROC_table.values())
average = tp_sum / all_instances
return recall_average, precision_average, f_average, average
def build_series(concepts, vector_dir, min_score=.5):
concept_series = defaultdict(lambda: [])
for concept in concepts:
print '\t', concept['id']
pos_concept = concept['id']
neg_concept = 'random'
pipeline = pickle.load(open(concept_pipeline(pos_concept, neg_concept), 'rb'))
classifier = pipeline.named_steps['classifier']
concept_series[pos_concept].append( ['Week', 'Signal'] )
vecfiles = os.listdir(vector_dir)
vecfiles.sort()
for vecfile in vecfiles:
X_sel = pickle.load(open(vector_dir+'/'+vecfile, 'r'))
tokens = vecfile.split('-')
week_dir = '-'.join(tokens[:3])
probs = classifier.predict_proba(X_sel)
pos_indices = [ i for i in range(X_sel.shape[0]) if probs[i,1] > min_score]
range_pct = (len(pos_indices) / float(X_sel.shape[0])) * 100.0
concept_series[pos_concept].append( [week_dir, range_pct] )
return concept_series
def prepareData(dataFile, nSample, doPlot=False):
with open(dataFile, 'rb') as file:
molDataGroupedChosen = pickle.load(file)
#nSmilesCodes = 200000
nSmilesMore = np.min([molDataGroupedChosen.shape[0], int(1.2 * nSample)])
mask = random.randint(0, molDataGroupedChosen.shape[0], size=nSmilesMore)
#mask = random.randint(0, molDataGroupedChosen.shape[0], size=nSmilesCodes)
mask = molDataGroupedChosen.index
staticFeatures = pd.DataFrame()
toBeAveraged = [
'standard_value', 'alogp', 'hba', 'hbd', 'psa', 'rtb', 'full_mwt',
'qed_weighted'
]
for quantity in toBeAveraged:
staticFeatures.loc[:, quantity] = (
molDataGroupedChosen.loc[mask, (quantity, 'min')] +
molDataGroupedChosen.loc[mask, (quantity, 'max')]) / 2
staticFeatures.loc[:, quantity].astype(float)
toBeTaken = ['aromatic_rings', 'heavy_atoms']
for quantity in toBeTaken:
staticFeatures.loc[:, quantity] = molDataGroupedChosen.loc[mask,
(quantity,
'min')]
staticFeatures.loc[:, quantity].astype(float)
staticFeatures.loc[:, 'number_of_rings'] = molDataGroupedChosen.loc[
mask, 'numberOfRings'].astype(float)
staticFeatures['full_mwt'] = staticFeatures.full_mwt.astype(float)
staticFeatures['qed_weighted'] = staticFeatures.qed_weighted.astype(float)
staticFeatures['aromatic_rings'] = staticFeatures.aromatic_rings.astype(
float)
staticFeatures['smiles_length'] = molDataGroupedChosen.loc[
staticFeatures.index, 'canonicalSmiles'].apply(lambda x: len(x))
# Remove rows with nans
staticFeatures = staticFeatures.dropna()
# Filter the smiles from given length range
staticFeatures = staticFeatures[(staticFeatures['smiles_length'] >= 40)
& (staticFeatures['smiles_length'] <= 60)]
thres = 100000
print(staticFeatures[staticFeatures['standard_value'] < thres].shape[0] /
staticFeatures['standard_value'].shape[0])
staticFeatures = staticFeatures[staticFeatures['standard_value'] < thres]
staticFeatures = staticFeatures.sample(nSample)
allDescriptors = [
'standard_value', 'alogp', 'hba', 'hbd', 'psa', 'rtb', 'full_mwt',
'qed_weighted', 'aromatic_rings', 'heavy_atoms', 'number_of_rings',
'smiles_length'
]
if (doPlot):
plotIdx = 1
nRows = np.ceil(len(allDescriptors) / 2)
fig = plt.figure(figsize=(16, 16))
for quantity in allDescriptors:
print(quantity)
plt.subplot(nRows, 2, plotIdx)
plt.hist(
staticFeatures[~staticFeatures[quantity].isnull()][quantity],
bins=10)
plt.title(quantity)
plotIdx += 1
smilesCodes = molDataGroupedChosen.loc[staticFeatures.index,
'encodedSmiles']
maxlen = -1
for code in smilesCodes:
if len(code) > maxlen:
maxlen = len(code)
maxlen
minlen = 1e6
for code in smilesCodes:
if len(code) < minlen:
minlen = len(code)
minlen
# pad the codes to the longest code
smilesCodes = smilesCodes.apply(
lambda x: pad_smile(x, max_len=maxlen, padding='right'))
chars = sorted(list(set(smilesCodes.str.cat(sep=''))))
print('total chars:', len(chars))
print(chars)
char2indices = dict((c, i) for i, c in enumerate(chars))
indices2char = dict((i, c) for i, c in enumerate(chars))
dynamicFeatures = np.zeros((len(smilesCodes), maxlen, len(chars)),
dtype=np.float)
print(dynamicFeatures.shape)
for codeidx, code in enumerate(smilesCodes):
for charidx, char in enumerate(code):
dynamicFeatures[codeidx, charidx, char2indices[char]] = 1
if (doPlot):
sums = []
for idx in range(dynamicFeatures.shape[0]):
sums.append(np.sum(dynamicFeatures[idx, :, :]))
plt.hist(sums)
return staticFeatures, dynamicFeatures, char2indices, indices2char
# -*- coding: utf-8 -*-
import os
import pickle
import numpy as np
from keras.models import Sequential
import gensim
from keras.layers.recurrent import LSTM, SimpleRNN
from sklearn.model_selection import train_test_split
import theano
theano.config.optimizer = "None"
with open('conversation.pickle', 'rb') as f:
vec_x, vec_y = pickle.load(f)
vec_x = np.array(vec_x, dtype=np.float64)
vec_y = np.array(vec_y, dtype=np.float64)
x_train, x_test, y_train, y_test = train_test_split(vec_x,
vec_y,
test_size=0.2,
random_state=1)
model = Sequential()
model.add(
LSTM(output_dim=300,
input_shape=x_train.shape[1:],
return_sequences=True,
init='glorot_normal',
inner_init='glorot_normal',
activation='sigmoid'))
import tkinter as tk
from tkinter import ttk
import pickle
import bisect
from tkinter import messagebox as msg
from student import Student
from teacher import Teacher
from course import Course
FONT = ("Times New Roman", 16)
currentUser = None
# Opening students.dat to populate students[] -- if this throws an error for you, run initialize_data.py to fix it
with open("students.dat", "rb") as fp:
students = pickle.load(fp)
# Opening teachers.dat to populate teachers[]
with open("teachers.dat", "rb") as fp:
teachers = pickle.load(fp)
courses=[]
for student in students:
for course in student.courses:
if course.title not in courses:
bisect.insort(courses, course.title)
print(courses)
class Bookworm(tk.Tk):
dicId[row[0]] = dic
dicIdTrain = {}
with open("/home/ramon/Escritorio/text/pan-ap17-bigdata/training/truth.txt",
'rt') as csvfile:
spamreader = csv.reader(csvfile, delimiter=':')
for row in spamreader:
dic = {}
if len(row) > 1:
dic["sexo"] = row[3]
dic["nacionalidad"] = row[6]
dicIdTrain[row[0]] = dic
vocabularioNacional = {}
vocabularioNacionalTest = {}
fileObject = open("tntTrained", 'rb')
b = pickle.load(fileObject)
with open("/home/ramon/Escritorio/text/frases_training.json") as json_data:
data = json.load(json_data)
lexicon = {}
for id in data.keys():
lexicon = generateBoW(data[id], lexicon, b)
bolsaPalabras = reduceBow(lexicon, 500)
dicRepeticionesNumeroDePalabras = {}
for i in sorted(bolsaPalabras.keys()):
dicRepeticionesNumeroDePalabras[math.log(i)] = math.log(
len(bolsaPalabras[i]))
pdf = pd.DataFrame(list(dicRepeticionesNumeroDePalabras.items()),
columns=["repeticiones", "numeroP"])
print(dicRepeticionesNumeroDePalabras)
def __init__(
self,
N_d,
N_s,
K,
f_s,
inp_tgt_type,
network_type,
min_snr,
max_snr,
snr_inter,
sample_dir=None,
ver='VERSION_NAME',
train_s_list=None,
train_d_list=None,
sample_size=None,
**kwargs
):
"""
Argument/s
N_d - window duration (samples).
N_s - window shift (samples).
K - number of frequency bins.
f_s - sampling frequency.
inp_tgt_type - input and target type.
network_type - network type.
min_snr - minimum SNR level for training.
max_snr - maximum SNR level for training.
stats_dir - path to save sample statistics.
ver - version name.
train_s_list - clean-speech training list to compute statistics.
train_d_list - noise training list to compute statistics.
sample_size - number of samples to compute the statistics from.
kwargs - keyword arguments.
"""
self.inp_tgt_type = inp_tgt_type
self.network_type = network_type
self.min_snr = min_snr
self.max_snr = max_snr
self.snr_levels = list(range(self.min_snr, self.max_snr + 1, snr_inter))
self.ver = ver
self.train_s_list=train_s_list
self.train_d_list=train_d_list
inp_tgt_obj_path = sample_dir + '/' + self.ver + '_inp_tgt.p'
if os.path.exists(inp_tgt_obj_path):
with open(inp_tgt_obj_path, 'rb') as f:
self.inp_tgt = pickle.load(f)
else:
self.inp_tgt = inp_tgt_selector(self.inp_tgt_type, N_d, N_s, K, f_s, **kwargs)
s_sample, d_sample, x_sample, wav_len = self.sample(sample_size, sample_dir)
self.inp_tgt.stats(s_sample, d_sample, x_sample, wav_len)
with open(inp_tgt_obj_path, 'wb') as f:
pickle.dump(self.inp_tgt, f, pickle.HIGHEST_PROTOCOL)
self.inp = Input(name='inp', shape=[None, self.inp_tgt.n_feat], dtype='float32')
self.network = network_selector(self.network_type, self.inp,
self.inp_tgt.n_outp, **kwargs)
self.model = Model(inputs=self.inp, outputs=self.network.outp)
self.model.summary()
if not os.path.exists("log/summary"):
os.makedirs("log/summary")
with open("log/summary/" + self.ver + ".txt", "w") as f:
self.model.summary(print_fn=lambda x: f.write(x + '\n'))
def loadModels(self):
"""
load models and aux data
"""
if self.bAttentionLayer:
self.model = load_model(os.path.join(self.dirName,self.sModelName+'.hd5'),custom_objects={"AttentionDecoder": AttentionDecoder})
else:
self.model = load_model(os.path.join(self.dirName,self.sModelName+'.hd5'))
print('model loaded: %s/%s.hd5' % (self.dirName,self.sModelName))
try:
self.bMultiType,self.maxngram,self.max_features,self.max_sentence_len, self.lnbClasses,self.ltag_vector , self.node_transformer = pickle.load(gzip.open('%s/%s.%s'%(self.dirName,self.sModelName,self.sAux),'r'))
except:
self.maxngram,self.max_features,self.max_sentence_len, self.lnbClasses,self.ltag_vector , self.node_transformer = pickle.load(gzip.open('%s/%s.%s'%(self.dirName,self.sModelName,self.sAux),'r'))
self.bMultiType = False
print('aux data loaded: %s/%s.%s' % (self.dirName,self.sModelName,self.sAux))
print("ngram: %s\tmaxfea=%s\tpadding=%s\tnbclasses=%s" % (self.maxngram,self.max_features,self.max_sentence_len, self.lnbClasses))
print("multitype model:%s"%(self.bMultiType))
test_dataFrame = pd.read_csv('Output/finalTestingData.csv')
#Dropping values that weren't trained on
test_dataFrame = test_dataFrame.drop(['StartTime','SrcAddr','DstAddr'], axis=1)
#Splitting up the test dataframe into one with only features, other with classifications
test_dataFrame_Classification = test_dataFrame[['LabelDisc']].copy()
test_dataFrame = test_dataFrame.drop(['LabelDisc'], axis=1)
#Getting vals to use for testing
test_data = test_dataFrame.values
test_data_class = test_dataFrame_Classification.values
test_features = test_data[0::]
test_results = test_data_class[0::,0]
#Testing
modelsFolder = "Models"
modelFileNames = [modelsFolder + "/" + file for file in os.listdir(modelsFolder) if file.endswith(".pkl")]
model = None
for model_filename in modelFileNames:
print("\n\nTesting using model "+model_filename+"...")
with open(model_filename, 'rb') as file:
model = pickle.load(file)
predictionClassification = model.predict(test_features)
print("Scikit-learn classification report: ")
print(classification_report(test_dataFrame_Classification.LabelDisc, predictionClassification))
print ('Accuracy = ' + str(accuracy_score(test_results, predictionClassification)) )
def main():
plotonly = True
global resolved
resolved = False
pvalues = None
if plotonly:
print("Loading parameters...")
if resolved:
picklepath = "fitvalues_resolved.pickle"
else:
picklepath = "fitvalues_merged.pickle"
with open(picklepath, 'rb') as f:
pvalues = pickle.load(f)
massset = set()
for each in os.listdir("ntuples"):
if ".root" in each:
massset.add(int(each.split("w")[0]))
i = 0
processes = []
for each_mass in sorted(massset):
maxv = int(each_mass * 2.5)
if maxv > 3000:
maxv = 3000
i += 1
# plot_mass_modified(each_mass, linspace(0, int(maxv), 25))
if resolved:
bins_resolved = [
100, 180, 210.0, 230.0, 250.0, 270.0, 290.0, 310.0, 330.0,
350.0, 370.0, 390.0, 410.0, 430.0, 450.0, 470.0, 490.0, 510.0,
530.0, 550.0, 570.0, 590.0, 610.0, 640.0, 670.0, 700.0, 730.0,
760.0, 790.0, 820.0, 870.0, 920.0, 970.0, 1020.0, 1070.0,
1120.0, 1170.0, 1220.0, 1280.0, 1340.0, 1400.0, 1460.0, 1570.0,
1640.0, 1720.0, 1810, 1910, 2000.0, 2100, 2200, 2300, 2400,
2500, 2600, 2700
]
bins_merged = [
180, 260.0, 320.0, 390.0, 470.0, 550.0, 640.0, 740.0, 850.0,
970.0, 1100.0, 1250.0, 1410.0, 1590.0, 1790.0, 2010.0, 2250.0,
2510.0, 3000
]
else:
bins_resolved = linspace(0, int(maxv), 15)
if maxv > 1000:
bins_resolved = linspace(0, int(maxv), 30)
# if int(maxv) < 1000:
# bins_resolved = linspace(0, int(maxv), 25)
# else:
# bins_resolved = linspace(0, int(maxv), 40)
# t = multiprocessing.Process(target=plot_mass_modified, args=(each_mass, linspace(0, int(maxv), 25)))
t = multiprocessing.Process(target=plot_mass_modified,
args=(each_mass, np.array(bins_resolved),
pvalues))
processes.append(t)
t.start()
if (i + 1) % 10 == 0:
for each in processes:
each.join()
processes = []
for each in processes:
each.join()
if not plotonly:
paras = {}
allfiles = os.listdir("pickle")
for each in allfiles:
mass = int(each.replace("fitvalues", "").replace(".pickle", ""))
with open("pickle/" + each, 'rb') as f:
pvalues = pickle.load(f)
paras[mass] = pvalues
with open('fitvalues.pickle', 'wb') as f:
pickle.dump(paras, f)
#get only the boundary vertices boundaryPoints, correspondenceMap = getContourVertices(mesh, coordinates, coordinateMap) #plot the points to verify plt.scatter(boundaryPoints[:,0], boundaryPoints[:,1]) plt.show() return boundaryPoints, correspondenceMap else: print (args[0]+' plane not defined.') #get data from Vertices.data file dataPath = 'python_data/' f = open(dataPath+'Vertices_new.data', 'rb') vertices = pickle.load(f) f.close() print(len(vertices)) boundaryPoints, correspondenceMap = project (vertices, 'AP') #save boundary poins and correspondence map in python data f = open(dataPath+'AP_contourPoints.data','wb') pickle.dump(boundaryPoints, f) f.close() f = open(dataPath+'AP_correspondenceMap.data','wb') pickle.dump(correspondenceMap, f) f.close() boundaryPoints, correspondenceMap = project (vertices, 'ML')
# Creamos una clase de prueba
class Persona:
def __init__(self, nombre, apellido):
self.nombre = nombre
self.apellido = apellido
def __str__(self):
return "%s, %s" % (self.apellido, self.nombre)
# Creamos la lista con los objetos
nombres = ["Hector", "Mario", "Marta"]
apellidos = ["Fernandez", "Guzman", "Kicillof"]
personas = []
for i in range(len(nombres)):
p = Persona(nombres[i], apellidos[i])
personas.append(p)
# Escribimos la lista en el fichero con pickle
with open('personas.pck', 'wb') as f:
pickle.dump(personas, f)
# Leemos la lista del fichero con pickle
with open('personas.pck', 'rb') as f:
personas = pickle.load(f)
for p in personas:
print(p)
# take raw data and pre-process it for the model.
from flask import Flask, request
import pandas as pd
import numpy as np
import json
import pickle
import os
app = Flask(__name__)
# Load Model and Scaler Files
model_path = os.path.join(os.path.pardir,os.path.pardir,'models')
model_filepath = os.path.join(model_path, 'lr_model.pkl')
scaler_filepath = os.path.join(model_path, 'lr_scaler.pkl')
model = pickle.load(open(model_filepath, 'rb'))
scaler = pickle.load(open(scaler_filepath, 'rb'))
# columns
columns = [ u'Age', u'Fare', u'FamilySize', \
u'IsMother', u'IsMale', u'Deck_A', u'Deck_B', u'Deck_C', u'Deck_D', \
u'Deck_E', u'Deck_F', u'Deck_G', u'Deck_Z', u'Pclass_1', u'Pclass_2', \
u'Pclass_3', u'Title_Lady', u'Title_Master', u'Title_Miss', u'Title_Mr', \
u'Title_Mrs', u'Title_Officer', u'Title_Sir', u'Fare_Bin_very_low', \
u'Fare_Bin_low', u'Fare_Bin_high', u'Fare_Bin_very_high', u'Embarked_C', \
u'Embarked_Q', u'Embarked_S', u'AgeState_Adult', u'AgeState_Child']
@app.route('/api', methods=['POST'])
def make_prediction():
# read json object and conver to json string
write_sbml_model,
)
config = cobra.Configuration()
config.solver = "glpk"
if __name__ == "__main__":
# ecoli
ecoli_model = read_sbml_model("iJO1366.xml.gz")
with open("iJO1366.pickle", "wb") as outfile:
dump(ecoli_model, outfile, protocol=2)
# salmonella
salmonella = read_sbml_model("salmonella.xml")
with open("salmonella.genes", "rb") as infile:
gene_names = load(infile)
for gene in salmonella.genes:
gene.name = gene_names[gene.id]
with open("salmonella.media", "rb") as infile:
salmonella.media_compositions = load(infile)
with open("salmonella.pickle", "wb") as outfile:
dump(salmonella, outfile, protocol=2)
# create mini model from textbook
textbook = read_sbml_model("textbook.xml.gz")
mini = cobra.Model("mini_textbook")
mini.compartments = textbook.compartments
for r in textbook.reactions:
if r.id in (
"GLCpts",
def learn(self,
to_predict,
from_data,
test_from_data=None,
group_by=None,
window_size=None,
order_by=[],
sample_margin_of_error=CONFIG.DEFAULT_MARGIN_OF_ERROR,
ignore_columns=[],
stop_training_in_x_seconds=None,
stop_training_in_accuracy=None,
backend='lightwood',
rebuild_model=True,
use_gpu=False,
disable_optional_analysis=False,
equal_accuracy_for_all_output_categories=False,
output_categories_importance_dictionary=None,
unstable_parameters_dict={}):
"""
Learn to predict a column or columns from the data in 'from_data'
Mandatory arguments:
:param to_predict: what column or columns you want to predict
:param from_data: the data that you want to learn from, this can be either a file, a pandas data frame, or url or a mindsdb data source
Optional arguments:
:param test_from_data: If you would like to test this learning from a different data set
Optional Time series arguments:
:param order_by: this order by defines the time series, it can be a list. By default it sorts each sort by column in ascending manner, if you want to change this pass a touple ('column_name', 'boolean_for_ascending <default=true>')
:param group_by: This argument tells the time series that it should learn by grouping rows by a given id
:param window_size: The number of samples to learn from in the time series
Optional data transformation arguments:
:param ignore_columns: it simply removes the columns from the data sources
Optional sampling parameters:
:param sample_margin_of_error (DEFAULT 0): Maximum expected difference between the true population parameter, such as the mean, and the sample estimate.
Optional debug arguments:
:param stop_training_in_x_seconds: (default None), if set, you want training to finish in a given number of seconds
:return:
"""
from_ds = getDS(from_data)
test_from_ds = test_from_data if test_from_data is None else getDS(
test_from_data)
transaction_type = TRANSACTION_LEARN
sample_confidence_level = 1 - sample_margin_of_error
# lets turn into lists: predict, order_by and group by
predict_columns = [to_predict
] if type(to_predict) != type([]) else to_predict
group_by = group_by if type(group_by) == type(
[]) else [group_by] if group_by else []
order_by = order_by if type(order_by) == type(
[]) else [order_by] if order_by else []
if len(predict_columns) == 0:
error = 'You need to specify a column to predict'
self.log.error(error)
raise ValueError(error)
# lets turn order by into tuples if not already
# each element ('column_name', 'boolean_for_ascending <default=true>')
order_by = [(col_name, True) if type(col_name) != type(
()) else col_name for col_name in order_by]
is_time_series = True if len(order_by) > 0 else False
heavy_transaction_metadata = {}
heavy_transaction_metadata['name'] = self.name
heavy_transaction_metadata['from_data'] = from_ds
heavy_transaction_metadata['test_from_data'] = test_from_ds
heavy_transaction_metadata['bucketing_algorithms'] = {}
heavy_transaction_metadata['predictions'] = None
heavy_transaction_metadata['model_backend'] = backend
light_transaction_metadata = {}
light_transaction_metadata['version'] = str(__version__)
light_transaction_metadata['name'] = self.name
light_transaction_metadata['data_preparation'] = {}
light_transaction_metadata['predict_columns'] = predict_columns
light_transaction_metadata['model_columns_map'] = from_ds._col_map
light_transaction_metadata['model_group_by'] = group_by
light_transaction_metadata['model_order_by'] = order_by
light_transaction_metadata['model_is_time_series'] = is_time_series
light_transaction_metadata['data_source'] = from_data
light_transaction_metadata['type'] = transaction_type
light_transaction_metadata['ignore_columns'] = ignore_columns
light_transaction_metadata['window_size'] = window_size
light_transaction_metadata[
'sample_margin_of_error'] = sample_margin_of_error
light_transaction_metadata[
'sample_confidence_level'] = sample_confidence_level
light_transaction_metadata[
'stop_training_in_x_seconds'] = stop_training_in_x_seconds
light_transaction_metadata[
'stop_training_in_accuracy'] = stop_training_in_accuracy
light_transaction_metadata['rebuild_model'] = rebuild_model
light_transaction_metadata['model_accuracy'] = {
'train': {},
'test': {}
}
light_transaction_metadata['column_importances'] = None
light_transaction_metadata['columns_buckets_importances'] = None
light_transaction_metadata['columnless_prediction_distribution'] = None
light_transaction_metadata[
'all_columns_prediction_distribution'] = None
light_transaction_metadata['use_gpu'] = use_gpu
light_transaction_metadata['malformed_columns'] = {
'names': [],
'indices': []
}
light_transaction_metadata[
'disable_optional_analysis'] = disable_optional_analysis
light_transaction_metadata['validation_set_accuracy'] = None
light_transaction_metadata['lightwood_data'] = {}
light_transaction_metadata['ludwig_data'] = {}
light_transaction_metadata['weight_map'] = {}
light_transaction_metadata['confusion_matrices'] = {}
light_transaction_metadata[
'equal_accuracy_for_all_output_categories'] = equal_accuracy_for_all_output_categories
light_transaction_metadata['output_categories_importance_dictionary'] = output_categories_importance_dictionary if output_categories_importance_dictionary is not None else {}
if 'skip_model_training' in unstable_parameters_dict:
light_transaction_metadata[
'skip_model_training'] = unstable_parameters_dict[
'skip_model_training']
else:
light_transaction_metadata['skip_model_training'] = False
if 'skip_stats_generation' in unstable_parameters_dict:
light_transaction_metadata[
'skip_stats_generation'] = unstable_parameters_dict[
'skip_stats_generation']
else:
light_transaction_metadata['skip_stats_generation'] = False
if 'always_use_model_prediction' in unstable_parameters_dict:
light_transaction_metadata[
'always_use_model_prediction'] = unstable_parameters_dict[
'always_use_model_prediction']
else:
light_transaction_metadata['always_use_model_prediction'] = False
if 'optimize_model' in unstable_parameters_dict:
light_transaction_metadata[
'optimize_model'] = unstable_parameters_dict['optimize_model']
else:
light_transaction_metadata['optimize_model'] = False
if rebuild_model is False:
old_lmd = {}
for k in light_transaction_metadata:
old_lmd[k] = light_transaction_metadata[k]
old_hmd = {}
for k in heavy_transaction_metadata:
old_hmd[k] = heavy_transaction_metadata[k]
with open(
os.path.join(
CONFIG.MINDSDB_STORAGE_PATH,
light_transaction_metadata['name'] +
'_light_model_metadata.pickle'), 'rb') as fp:
light_transaction_metadata = pickle.load(fp)
with open(
os.path.join(
CONFIG.MINDSDB_STORAGE_PATH,
heavy_transaction_metadata['name'] +
'_heavy_model_metadata.pickle'), 'rb') as fp:
heavy_transaction_metadata = pickle.load(fp)
for k in [
'data_preparation', 'rebuild_model', 'data_source', 'type',
'ignore_columns', 'sample_margin_of_error',
'sample_confidence_level', 'stop_training_in_x_seconds',
'stop_training_in_accuracy'
]:
if old_lmd[k] is not None:
light_transaction_metadata[k] = old_lmd[k]
for k in ['from_data', 'test_from_data']:
if old_hmd[k] is not None:
heavy_transaction_metadata[k] = old_hmd[k]
Transaction(session=self,
light_transaction_metadata=light_transaction_metadata,
heavy_transaction_metadata=heavy_transaction_metadata,
logger=self.log)
from flask import Flask, render_template, request
import jsonify
import requests
import pickle
import numpy as np
import sklearn
from sklearn.preprocessing import StandardScaler
app = Flask(__name__)
model = pickle.load(open('US_prediction.pkl', 'rb'))
@app.route('/', methods=['GET'])
def Home():
return render_template('index.html')
standard_to = StandardScaler()
@app.route("/predict", methods=['POST'])
def predict():
try:
if request.method == 'POST':
GRE = float(request.form['GRE'])
TOEFL = float(request.form['TOEFL'])
UR = float(request.form['UR'])
SOP = float(request.form['SOP'])
LOR = float(request.form['LOR'])
GPA = float(request.form['GPA'])
RES = float(request.form['RES'])
from keras.utils.np_utils import to_categorical
# from keras.utils import print_summary
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization
# optimizers: SGD, RMSprop, Adagrad, Adadelta, Adam, Adamax, Nadam, TFOptimizer
from keras.optimizers import RMSprop, Adadelta
from keras.callbacks import ReduceLROnPlateau
from keras.preprocessing.image import ImageDataGenerator
import matplotlib.pyplot as plt
import seaborn as sns
# ---------------------------Data Preparation---------------------------------
# load the dataset
file = gzip.open('mnist.pkl.gz','rb')
train_set, valid_set, test_set = pk.load(file,encoding='latin1')
file.close()
x_train = train_set[0]
# print(x_train.shape) #(50000, 784)
y_train = train_set[1]
del train_set # free some space
x_test = test_set[0]
# print(x_test.shape) #(10000, 784)
y_test = test_set[1]
del test_set
x_val = valid_set[0]
y_val = valid_set[1]
del valid_set
def get_model_data(self, model_name):
with open(
os.path.join(CONFIG.MINDSDB_STORAGE_PATH,
f'{model_name}_light_model_metadata.pickle'),
'rb') as fp:
lmd = pickle.load(fp)
# ADAPTOR CODE
amd = {}
if lmd['current_phase'] == MODEL_STATUS_TRAINED:
amd['status'] = 'complete'
elif lmd['current_phase'] == MODEL_STATUS_ERROR:
amd['status'] = 'error'
else:
amd['status'] = 'training'
# Shared keys
for k in [
'name', 'version', 'is_active', 'data_source', 'predict',
'current_phase', 'train_end_at', 'updated_at', 'created_at',
'data_preparation', 'validation_set_accuracy'
]:
if k == 'predict':
amd[k] = lmd['predict_columns']
elif k in lmd:
amd[k] = lmd[k]
if k == 'validation_set_accuracy':
if lmd['validation_set_accuracy'] is not None:
amd['accuracy'] = round(lmd['validation_set_accuracy'],
3)
else:
amd['accuracy'] = None
else:
amd[k] = None
amd['data_analysis'] = {
'target_columns_metadata': [],
'input_columns_metadata': []
}
amd['model_analysis'] = []
for col in lmd['model_columns_map'].keys():
if col in lmd['malformed_columns']['names']:
continue
try:
icm = self._adapt_column(lmd['column_stats'][col], col)
except Exception as e:
icm = {'column_name': col}
#continue
amd['force_vectors'] = {}
if col in lmd['predict_columns']:
# Histograms for plotting the force vectors
if 'all_columns_prediction_distribution' in lmd and lmd[
'all_columns_prediction_distribution'] is not None:
amd['force_vectors'][col] = {}
amd['force_vectors'][col][
'normal_data_distribution'] = lmd[
'all_columns_prediction_distribution'][col]
amd['force_vectors'][col]['normal_data_distribution'][
'type'] = 'categorical'
amd['force_vectors'][col]['missing_data_distribution'] = {}
for missing_column in lmd[
'columnless_prediction_distribution'][col]:
amd['force_vectors'][col]['missing_data_distribution'][
missing_column] = lmd[
'columnless_prediction_distribution'][col][
missing_column]
amd['force_vectors'][col]['missing_data_distribution'][
missing_column]['type'] = 'categorical'
icm['importance_score'] = None
amd['data_analysis']['target_columns_metadata'].append(icm)
if 'confusion_matrices' in lmd and col in lmd[
'confusion_matrices']:
confusion_matrix = lmd['confusion_matrices'][col]
else:
confusion_matrix = None
# Model analysis building for each of the predict columns
mao = {
'column_name': col,
'overall_input_importance': {
"type": "categorical",
"x": [],
"y": []
},
"train_accuracy_over_time": {
"type": "categorical",
"x": [],
"y": []
},
"test_accuracy_over_time": {
"type": "categorical",
"x": [],
"y": []
},
"accuracy_histogram": {
"x": [],
"y": [],
'x_explained': []
},
"confusion_matrix": confusion_matrix
}
# This is a check to see if model analysis has run on this data
if 'model_accuracy' in lmd and lmd[
'model_accuracy'] is not None and 'train' in lmd[
'model_accuracy'] and 'combined' in lmd[
'model_accuracy']['train'] and lmd[
'model_accuracy']['train'][
'combined'] is not None:
train_acc = lmd['model_accuracy']['train']['combined']
test_acc = lmd['model_accuracy']['test']['combined']
for i in range(0, len(train_acc)):
mao['train_accuracy_over_time']['x'].append(i)
mao['train_accuracy_over_time']['y'].append(
train_acc[i])
for i in range(0, len(test_acc)):
mao['test_accuracy_over_time']['x'].append(i)
mao['test_accuracy_over_time']['y'].append([i])
if 'model_accuracy' in lmd and lmd[
'model_accuracy'] is not None and lmd[
'column_importances'] is not None:
mao['accuracy_histogram']['x'] = [
f'{x}'
for x in lmd['accuracy_histogram'][col]['buckets']
]
mao['accuracy_histogram']['y'] = lmd['accuracy_histogram'][
col]['accuracies']
for output_col_bucket in lmd[
'columns_buckets_importances'][col]:
x_explained_member = []
for input_col in lmd['columns_buckets_importances'][
col][output_col_bucket]:
stats = lmd['columns_buckets_importances'][col][
output_col_bucket][input_col]
adapted_sub_incol = self._adapt_column(
stats, input_col)
x_explained_member.append(adapted_sub_incol)
mao['accuracy_histogram']['x_explained'].append(
x_explained_member)
for icol in lmd['model_columns_map'].keys():
if icol in lmd['malformed_columns']['names']:
continue
if icol not in lmd['predict_columns']:
try:
mao['overall_input_importance']['x'].append(
icol)
mao['overall_input_importance']['y'].append(
round(lmd['column_importances'][icol], 1))
except:
print(
f'No column importances found for {icol} !'
)
amd['model_analysis'].append(mao)
else:
if 'column_importances' in lmd and lmd[
'column_importances'] is not None:
icm['importance_score'] = lmd['column_importances'][col]
amd['data_analysis']['input_columns_metadata'].append(icm)
return amd
for i in new_x.split():
for y in company_list:
if similar(i, y) > threshold:
first_list.append(y)
print(y, i)
else:
first_list.append('')
result_list.append(first_list)
with open('datafile_reuters.pkl', 'wb') as f:
print('pickle dumping...')
pickle.dump(result_list, f)
with open('datafile_reuters.pkl', 'rb') as f:
newlist = pickle.load(f)
df_new = pd.DataFrame()
df_i = []
df_j = []
df_k = []
print('begin appending...')
for new_name, new_file, new_path in zip(newlist, all_files, all_paths):
for i in set(new_name):
if i != '':
df_i.append(i)
else:
df_i.append('')
df_j.append(new_file)
df_k.append(new_path)
import pickle # filename should be .sav format filename = '' loaded_model = pickle.load(open(filename, 'rb')) result = loaded_model.most_similar(positive=[],negative=[], topn=5) print(result)
if i < len(cap)-1:
vhist[i, :] = np.logical_or(vhist[i, :], vhist[i-1, :])
vhist[i, ii_1] = 1
nxts.append(nxt)
imgs.append(img)
seqs.append(seq)
vhists.extend(vhist)
return imgs, curs, nxts, seqs, vhists
if __name__ == '__main__':
dec_map = pickle.load(open("dec_map.pkl", 'rb'))
enc_map = pickle.load(open("enc_map.pkl", 'rb'))
#"/general/home/manish.singhal/Image-Captioning-master/encoded_images_inceptionV3.p"
img_train = pickle.load(open("../encoded_images_inceptionV3.p", 'rb'))
img_test = pickle.load(open("../encoded_images_test_inceptionV3.p", 'rb'))
print (len(img_train))
df_cap = pd.read_csv("../processed_files/flickr8k_training_dataset.txt", delimiter = '\t')
vocab_size = len(dec_map)
print("vocab sixe is",vocab_size)
embedding_matrix = generate_embedding_matrix('pre_trained/glove.6B.100d.txt', dec_map)
model = image_caption_model(vocab_size=vocab_size, embedding_matrix=embedding_matrix)
if len(sys.argv) >= 2:
print('load weights from : {}'.format(sys.argv[1]))
def extract_features(data_folder, filename, ini, end):
df_data_x = pickle.load(open(data_folder + filename, "rb"))
_, sig_sz = df_data_x.shape
df_data_x = df_data_x[range(ini, min(end, sig_sz))]
sc_mean = pd.DataFrame(sc_mean_(df_data_x), columns=['sc_mean'])
sc_median = pd.DataFrame(sc_median_(df_data_x),columns=['sc_median'])
sc_std = pd.DataFrame(sc_std_(df_data_x),columns=['sc_std'])
sc_min = pd.DataFrame(sc_min_(df_data_x),columns=['sc_min'])
sc_max = pd.DataFrame(sc_max_(df_data_x),columns=['sc_max'])
sc_range = pd.DataFrame(sc_range_(sc_max,sc_min),columns=['sc_range'])
sc_minRatio = pd.DataFrame(sc_minRatio_(df_data_x,sc_min),columns=['sc_minRatio'])
sc_maxRatio = pd.DataFrame(sc_maxRatio_(df_data_x,sc_max),columns=['sc_maxRatio'])
sc1Diff_mean = pd.DataFrame( sc1Diff_mean_(df_data_x),columns=['sc1Diff_mean'])
sc1Diff_median = pd.DataFrame( sc1Diff_median_(df_data_x),columns=['sc1Diff_median'] )
sc1Diff_std = pd.DataFrame( sc1Diff_std_(df_data_x),columns=['sc1Diff_std'])
sc1Diff_min = pd.DataFrame( sc1Diff_min_(df_data_x),columns=['sc1Diff_min'])
sc1Diff_max = pd.DataFrame( sc1Diff_max_(df_data_x),columns=['sc1Diff_max'])
sc1Diff_range = pd.DataFrame( sc1Diff_range_(sc1Diff_max,sc1Diff_min),columns=['sc1Diff_range'])
sc1Diff_minRatio = sc1Diff_minRatio_(df_data_x,sc1Diff_min)
sc1Diff_minRatio.columns=['sc1Diff_minRatio']
sc1Diff_maxRatio = sc1Diff_maxRatio_(df_data_x,sc1Diff_max)
sc1Diff_maxRatio.columns=['sc1Diff_maxRatio']
sc2Diff_std = pd.DataFrame( sc2Diff_std_(df_data_x),columns=['sc2Diff_std'] )
sc2Diff_min = pd.DataFrame( sc2Diff_min_(df_data_x),columns=['sc2Diff_min'] )
sc2Diff_max = pd.DataFrame( sc2Diff_max_(df_data_x),columns=['sc2Diff_max'] )
sc2Diff_range = pd.DataFrame(sc2Diff_range_(sc2Diff_max,sc2Diff_min),columns=['sc2Diff_range'])
sc2Diff_minRatio = sc2Diff_minRatio_(df_data_x,sc2Diff_min)
sc2Diff_minRatio.columns = ['sc2Diff_minRatio']
sc2Diff_maxRatio = sc2Diff_maxRatio_(df_data_x,sc2Diff_max)
sc2Diff_maxRatio.columns = ['sc2Diff_maxRatio']
scfft_df = scfft_(df_data_x)
scfft_mean = pd.DataFrame( scfft_mean_(scfft_df),columns=['scfft_mean'])
scfft_median = pd.DataFrame( scfft_median_(scfft_df),columns=['scfft_median'])
scfft_std = pd.DataFrame( scfft_std_(scfft_df),columns=['scfft_std'])
scfft_min = pd.DataFrame( scfft_min_(scfft_df),columns=['scfft_min'])
scfft_max = pd.DataFrame( scfft_max_(scfft_df),columns=['scfft_max'])
scfft_range = pd.DataFrame( scfft_range_(scfft_max,scfft_min),columns=['scfft_range'])
feature_list = ['sc_mean','sc_median','sc_std','sc_min','sc_max','sc_range',
'sc_minRatio','sc_maxRatio','sc1Diff_mean','sc1Diff_median',
'sc1Diff_std','sc1Diff_min','sc1Diff_max','sc1Diff_range',
'sc1Diff_minRatio','sc1Diff_maxRatio','sc2Diff_std',
'sc2Diff_min','sc2Diff_max','sc2Diff_range','sc2Diff_minRatio',
'sc2Diff_maxRatio','scfft_mean','scfft_median','scfft_std',
'scfft_min','scfft_max','scfft_range']
temp_feature_df = pd.DataFrame()
for i in feature_list:
temp_feature_df = pd.concat([locals()[i], temp_feature_df], axis=1)
# rename columns with name of channel
tmp = filename.split('_')
ch_id = tmp[0] + "_" + tmp[1]
tmp_cols = temp_feature_df.columns
temp_feature_df.columns = [ch_id + "_" + name for name in tmp_cols]
pickle.dump(temp_feature_df, open(data_folder + "feat_" + filename, "wb"))
print('--- GSR features ---')
print(temp_feature_df.shape)
return temp_feature_df
params = {
'database': os.path.join(
'tests',
'data',
'BSA.fasta'
),
},
force = False
)
scan_rt_lookup = pickle.load(
open(
os.path.join(
'tests',
'data',
'_test_ursgal_lookup.pkl'
)
,
'rb'
)
)
unify_csv_main = R.unodes['unify_csv_1_0_0']['class'].import_engine_as_python_function()
input_csv = os.path.join(
'tests',
'data',
'omssa_2_1_9',
'test_BSA1_omssa_2_1_9.csv'
)
output_csv = os.path.join(
def loadPickle(fileName):
with open(fileName, mode="rb") as f:
return pickle.load(f)
def verify(source_uri, renditions, do_profiling, max_samples, model_dir, model_name):
"""
Function that returns the predicted compliance of a list of renditions
with respect to a given source file using a specified model.
"""
total_start = time.clock()
total_start_user = time.time()
source_video, source_audio, video_available, audio_available = retrieve_video_file(source_uri)
if video_available:
# Prepare source and renditions for verification
source = {'path': source_video,
'audio_path' : source_audio,
'video_available': video_available,
'audio_available': audio_available,
'uri': source_uri}
# Create a list of preverified renditions
pre_verified_renditions = []
for rendition in renditions:
pre_verification = pre_verify(source, rendition)
if rendition['video_available']:
pre_verified_renditions.append(pre_verification)
# Cleanup the audio file generated to avoid cluttering
if os.path.exists(source['audio_path']):
os.remove(source['audio_path'])
# Configure model for inference
model_name = 'OCSVM'
scaler_type = 'StandardScaler'
learning_type = 'UL'
loaded_model = pickle.load(open('{}/{}.pickle.dat'.format(model_dir,
model_name), 'rb'))
loaded_scaler = pickle.load(open('{}/{}_{}.pickle.dat'.format(model_dir,
learning_type,
scaler_type), 'rb'))
# Open model configuration file
with open('{}/param_{}.json'.format(model_dir, model_name)) as json_file:
params = json.load(json_file)
features = params['features']
# Remove non numeric features from feature list
non_temporal_features = ['attack_ID', 'title', 'attack', 'dimension', 'size', 'size_dimension_ratio']
metrics_list = []
for metric in features:
if metric not in non_temporal_features:
metrics_list.append(metric.split('-')[0])
# Initialize times for assets processing profiling
start = time.clock()
start_user = time.time()
# Instantiate VideoAssetProcessor class
asset_processor = VideoAssetProcessor(source,
pre_verified_renditions,
metrics_list,
do_profiling,
max_samples,
features)
# Record time for class initialization
initialize_time = time.clock() - start
initialize_time_user = time.time() - start_user
# Register times for asset processing
start = time.clock()
start_user = time.time()
# Assemble output dataframe with processed metrics
metrics_df, pixels_df, dimensions_df = asset_processor.process()
# Record time for processing of assets metrics
process_time = time.clock() - start
process_time_user = time.time() - start_user
# Normalize input data using the associated scaler
x_renditions = np.asarray(metrics_df)
x_renditions = loaded_scaler.transform(x_renditions)
# Make predictions for given data
start = time.clock()
y_pred = loaded_model.decision_function(x_renditions)
prediction_time = time.clock() - start
# Add predictions to rendition dictionary
i = 0
for _, rendition in enumerate(renditions):
if rendition['video_available']:
rendition.pop('path', None)
rendition['tamper'] = np.round(y_pred[i], 6)
# Append the post-verification of resolution and pixel count
if 'pixels' in rendition:
rendition['pixels_post_verification'] = float(rendition['pixels']) / pixels_df[i]
if 'resolution' in rendition:
rendition['resolution']['height_post_verification'] = float(rendition['resolution']['height']) / int(dimensions_df[i].split(':')[0])
rendition['resolution']['width_post_verification'] = float(rendition['resolution']['width']) / int(dimensions_df[i].split(':')[1])
i += 1
if do_profiling:
print('Features used:', features)
print('Total CPU time:', time.clock() - total_start)
print('Total user time:', time.time() - total_start_user)
print('Initialization CPU time:', initialize_time)
print('Initialization user time:', initialize_time_user)
print('Process CPU time:', process_time)
print('Process user time:', process_time_user)
print('Prediction CPU time:', prediction_time)
return renditions
def load_obj(name):
with open(name + '.pkl', 'rb') as f:
return pickle.load(f)
def main(args):
# Get data file
dt = h5py.File(args.prior, 'r')
# Extract data
face_basis = get_face_basis(dt, args.size_id, args.size_exp)
# SECTION 2
print("\nSection 2...")
# Sample alpha and delta
print("\tSampling latent variables")
if args.latent is None:
alpha = np.random.uniform(-1, 1, args.size_id).astype(np.float32)
delta = np.random.uniform(-1, 1, args.size_exp).astype(np.float32)
else:
with open(args.latent, "rb") as f:
(alpha, delta), _ = pickle.load(f)
alpha, delta = alpha.detach().numpy(), delta.detach().numpy()
# Generate face from respective alpha and delta
print("\tGenerating face 3D point-cloud")
face_3D = face_basis(alpha, delta)
# Save object for later visualization
print("\tSaving face data")
save_obj(
args.face_3D_file,
face_3D,
face_basis.color,
face_basis.mesh,
)
print("\tSaved to ", args.face_3D_file)
if args.up_to is not None and args.up_to == "3D":
return
# SECTION 3
print("\nSection 3...")
print("Rotating face")
# Transform face
print("\tTransforming face with omega: ", args.omega, " and t: ", args.t)
face_transform = FaceTransform()
face_wt = face_transform(face_3D, args.omega, args.t)
print("\tSaving rotated face data")
save_obj(args.face_wt_file, face_wt, face_basis.color, face_basis.mesh)
print("\tSaved to ", args.face_wt_file)
if args.up_to is not None and args.up_to == "rotate":
return
print("Applying camera projection")
# Init camera
print("\tInitializing camera with FOV: ", args.fov, " aspect ratio: ",
args.aratio, " near-far clips: ", args.near_far)
camera = Camera(args.fov, args.aratio, args.near_far)
# Generate image from face
print("\tGenerating uv image")
face_uv = camera(face_wt)
print("\tNormalizing uv image (z coordinate)")
uv_normalizer = UVNormalizer()
face_uv_n = uv_normalizer(face_uv)
# Extracting landmark points
print("\tExtracting landmark pointsi from", args.landmarks)
lmks = get_landmarks(args.landmarks)
face_lmks = face_uv_n[lmks, :2]
# Generate image
plt.scatter(face_lmks[:, 0], face_lmks[:, 1])
plt.axis('equal')
plt.savefig(args.face_uv_file + ".png", dpi=900)
print("\tSaved to ", args.face_uv_file + ".png")
save_obj(args.face_uv_file + ".obj", face_uv_n, face_basis.color,
face_basis.mesh)
print("\tSaved to ", args.face_uv_file + ".obj")
if args.up_to is not None and args.up_to == "project":
return
# Create model
def multilayer_perceptron(_X, _weights, _biases):
#Hidden layer with RELU activation
layer_1 = tf.nn.relu(tf.add(tf.matmul(_X, _weights['h1']), _biases['b1']))
#Hidden layer with sigmoid activation
layer_2 = tf.nn.relu(
tf.add(tf.matmul(layer_1, _weights['h2']), _biases['b2']))
# layer_3 = tf.nn.dropout(tf.nn.relu(tf.add(tf.matmul(layer_2, _weights['h3']), _biases['b3'])), 0.5)
return tf.matmul(layer_2, _weights['out']) + _biases['out']
print("Loading saved Weights for layer 1 ...")
file_ID = "./data/parameters_mfcc_1.pkl"
f = open(file_ID, "rb")
W = pickle.load(f)
b = pickle.load(f)
f.close()
weights = {
'h1': tf.Variable(W['h1']),
'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])),
# 'h3': tf.Variable(tf.random_normal([n_hidden_2, n_hidden_3])),
'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes]))
}
biases = {
'b1': tf.Variable(b['b1']),
'b2': tf.Variable(tf.random_normal([n_hidden_2])),
# 'b3': tf.Variable(tf.random_normal([n_hidden_3])),
'out': tf.Variable(tf.random_normal([n_classes]))
}
print(self.played_memory)
if __name__ == "__main__":
train_game = 20000
lr = 0.75
y = 0.90
q1cumul_reward_list = []
q1actions_list = []
q1states_list = []
q2cumul_reward_list = []
q2actions_list = []
q2states_list = []
winner1 = []
with open('player.pkl', 'rb') as model:
q_player = pickle.load(model)
q_player1 = q_player
q_player2 = q_player(9)
for i in range(train_game):
game = Game(3, 3)
grille = game.get_grille()
q1actions = []
q1states = [grille]
q1cumul_reward = 0
q2actions = []
q2states = [grille]
q2cumul_reward = 0
while game.end == False:
grille = game.get_grille()
choice_1 = q_player1.choose_move(i, grille)
game.game_turn_x(choice_1)
def load_photo_features(filename, dataset):
# load all features
all_features = load(open(filename, 'rb'))
# filter features
features = {k: all_features[k] for k in dataset}
return features
def initialize_cast(self):
# This fixes the `No handlers could be found for logger
# "pychromecast.socket_client` warning"`.
# See commit 18005ebd4c96faccd69757bf3d126eb145687e0d.
if chromecast:
from pychromecast import socket_client
self.cclist = self._get_chromecasts()
self.cclist = [[i, _, 'Gcast'] for i, _ in enumerate(self.cclist)]
else:
self.cclist = []
if sonos:
try:
self.sonos_list = list(soco.discover())
for self.index, device in enumerate(self.sonos_list):
add_sonos = [self.index, device, 'Sonos']
self.cclist.append(add_sonos)
except TypeError:
pass
if self.debug is True:
print('self.cclist', self.cclist)
if (len(self.cclist) != 0 and self.select_device is False and
self.device_name is None):
if self.debug is True:
print('if len(self.cclist) != 0 and self.select_device == False:')
print(' ')
print_available_devices(self.available_devices())
print(' ')
if self.discover is False:
print(colors.important('Casting to first device shown above!'))
print(colors.important('Select devices by using the -s flag.'))
print(' ')
self.cast_to = self.cclist[0][1]
if self.cclist[0][2] == 'Sonos':
print(colors.success(self.cast_to.player_name))
else:
print(colors.success(self.cast_to))
print(' ')
elif (len(self.cclist) != 0 and self.select_device is True and
self.tray is False and self.device_name is None):
if self.debug is True:
print('elif len(self.cclist) != 0 and self.select_device == True'
' and self.tray == False:')
if os.path.exists('/tmp/mkchromecast.tmp') is False:
self.tf = open('/tmp/mkchromecast.tmp', 'wb')
print(' ')
print_available_devices(self.available_devices())
else:
if self.debug is True:
print('else:')
self.tf = open('/tmp/mkchromecast.tmp', 'rb')
self.index = pickle.load(self.tf)
self.cast_to = self.cclist[int(self.index)]
print(' ')
print(colors.options('Casting to:') + ' ' +
colors.success(self.cast_to))
print(' ')
elif (len(self.cclist) != 0 and self.select_device is True and
self.tray is True):
if self.debug is True:
print('elif len(self.cclist) != 0 and self.select_device == True'
' and self.tray == True:')
if os.path.exists('/tmp/mkchromecast.tmp') is False:
self.tf = open('/tmp/mkchromecast.tmp', 'wb')
print(' ')
print_available_devices(self.available_devices())
else:
if self.debug is True:
print('else:')
self.tf = open('/tmp/mkchromecast.tmp', 'rb')
self.cast_to = pickle.load(self.tf)
print_available_devices(self.available_devices())
print(' ')
print(colors.options('Casting to:') + ' ' +
colors.success(self.cast_to))
print(' ')
elif len(self.cclist) == 0 and self.tray is False:
if self.debug is True:
print('elif len(self.cclist) == 0 and self.tray == False:')
print(colors.error('No devices found!'))
if self.platform == 'Linux' and self.adevice is None:
remove_sink()
elif self.platform == 'Darwin':
inputint()
outputint()
terminate()
exit()
elif len(self.cclist) == 0 and self.tray is True:
print(colors.error(':::Tray::: No devices found!'))
self.available_devices = []
staticFeatures, dynamicFeatures, char2indices, indices2char = prepareData(
args.dataFile, args.nSample, doPlot=False)
saveFeatures = True
if (saveFeatures):
with open('staticFeatures.pckl', 'wb') as f:
pickle.dump(staticFeatures, f)
with open('dynamicFeatures.pckl', 'wb') as f:
pickle.dump(dynamicFeatures, f)
loadFeatures = False
if (loadFeatures):
with open('staticFeatures.pckl', 'rb') as f:
staticFeatures = pickle.load(f)
with open('dynamicFeatures.pckl', 'rb') as f:
dynamicFeatures = pickle.load(f)
chosenFeatures = ['full_mwt', 'heavy_atoms', 'smiles_length']
staticFeaturesSlice = staticFeatures[chosenFeatures]
staticFeaturesSliceScaled, scaler = scaleFeatures(staticFeaturesSlice)
aeDimensions = [64, 64, 32]
#model, history = trainModel(dynamicFeatures, staticFeaturesSliceScaled, aeDimensions, args.modelWeightsFile, args.nEpoch, args.nBatch)
model, history = trainModelDynamic(dynamicFeatures, aeDimensions,
args.modelWeightsFile, args.nEpoch,
args.nBatch)
nCharInSmiles = dynamicFeatures.shape[1]
nCharSet = dynamicFeatures.shape[2]
nStatic = staticFeaturesSlice.shape[1]
