def _pack_message(self,device = None):
addresses = (device or [self.device_token])
deliver_before_timestamp = (datetime.now() + (5 * 60))
push_id = int(round(int(time.time()) * random()) + int(time.time())) # simplify
t = get_template("bbpush.post")
c = Context({'addresses':addresses,'deliver_before_timestamp':deliver_before_timestamp ,'push_id':push_id,'app_id':settings.BB_APP_ID})
content = t.render(c)
def buffer_newevents(evtype=None, timeout=3000, verbose=False):
"""
Wait for and return any new events recieved from the buffer between
calls to this function
timeout = maximum time to wait in milliseconds before returning
"""
global ftc, nEvents # use to store number events processed accross function calls
if not "nEvents" in globals(): # first time initialize to events up to now
start, nEvents = ftc.poll()
if verbose:
print "Waiting for event(s) " + str(evtypes) + " with timeout_ms " + str(timeout_ms)
start = time.time()
elapsed_ms = 0
events = []
while len(events) == 0 and elapsed_ms < timeout:
nSamples, curEvents = ftc.wait(-1, nEvents, timeout_ms - elapsed_ms)
if curEvents > nEvents:
events = ftc.getEvents([nEvents, curEvents - 1])
if not evttype is None:
events = filter(lambda x: x.type in evtype, events)
nEvents = curEvents # update starting number events (allow for buffer restarts)
elapsed_ms = (time.time() - start) * 1000
return events
def why_opinion(self):
feature_names = self.vectorizer.get_feature_names()
opinion_idx = list(np.argsort(self.clf.feature_log_prob_[1])) #most important are last
opinion_idx.reverse() #most important are first
self.opinion_idx = opinion_idx
i_filtered=0
i_opinion_idx = 0
filtered_keywords = []
stime=time.time()
while i_filtered<100:
if not self.vectorizer.get_feature_names()[opinion_idx[i_opinion_idx]] in nltk.corpus.stopwords.words('english'):
filtered_keywords.append(self.vectorizer.get_feature_names()[opinion_idx[i_opinion_idx]])
i_filtered+=1
i_opinion_idx+=1
self.fk=filtered_keywords
print 'why while: '+str(time.time()-stime)
news_words=[]
opinion_words = []
ratio = self.clf.feature_log_prob_[1]-self.clf.feature_log_prob_[0]
for i,count in enumerate(self.vect.toarray()[0]):
if count>0:
opinion_words.append([count*np.exp(ratio[i]), count, ratio[i], self.vectorizer.get_feature_names()[i]])
opinion_words_sorted = sorted(opinion_words, key=lambda x:x[2])
return opinion_words_sorted #, news_words_sorted
def testLargeDirReinstall(self):
"""Benchmark and test reinstalling FSD with a directory holding a large number of people"""
from time import time
from random import choice
qi = getToolByName(self.portal, 'portal_quickinstaller')
acl = getToolByName(self.portal, 'acl_users')
# pick a user and make sure they exist in acl_users before we start
user_id = choice(self.person_ids)
person = self.directory[user_id]
self.failUnless(acl.getUserById(id=user_id),"Problem: person is not listed in acl_users")
self.failUnless(person.UID(),"Problem: expected person object %s to have a UID. It does not" % person)
# how long does it take to reinstall FSD?
import time
start_time = time.time()
qi.reinstallProducts(products='FacultyStaffDirectory')
end_time = time.time()
elapsed_time = end_time-start_time
reinstall_report = "\nreinstalling FSD with a directory containing %s people took %s seconds\n" % (self.numPeople, elapsed_time)
print "\n" + ("*" * 20) + reinstall_report + ("*" * 20)
# test that a person in the FSD is still a user
self.failUnless(acl.getUserById(id=user_id),"Problem: after reinstall person is not listed in acl_users")
self.failUnless(person.UID(),"Problem: after reinstall expected person object %s to have a UID. It does not" % person)
def InsertKeyWordToDB(fromSubDir,toSubDir):
db = DB()
parser = Parser()
for index in range(fromSubDir,toSubDir):
for root,dirs,files in os.walk('test/keyword/'+str(index)+"/"):
#each subdir: 1000record
start = time.time()
for afile in files:
if afile == '.DS_Store':
continue
words = afile.split('_')
aExpert = Expert(words[0].strip(),words[1].strip(),words[2].replace(".html","").strip())
aExpert.setKeyword(parser.parseKeyword(root,afile))
aExpert.ChangeKeywordsToString()
#print aExpert.keywordsList
if not db.isExpertExist(aExpert):
db.insertExpert(aExpert)
end = time.time()
db.conn.commit()
print ("KeywordSubDir %d is Done!"%index),
print time.strftime('%m-%d %H:%M:%S',time.localtime(time.time())),"total:",end-start
f = open("KeywordsToDB.log","a")
f.write(time.strftime('%m-%d %H:%M:%S',time.localtime(time.time()))+" keywordSubDir"+str(index)+" is Done! "+"total"+str(end-start) )
f.close()
db.close()
def InsertPaperToDB(fromSubDir,toSubDir):
db = DB()
parser = Parser()
for index in range(fromSubDir,toSubDir):
for root,dirs,files in os.walk('test/paper/'+str(index)+"/"):
n = 1000*index
start = time.time()
for afile in files:
if afile == '.DS_Store':
continue
words = afile.split('_')
papers = (parser.parsePaper(root,afile))
for eachPapaer in papers:
if not db.isPaperExist(eachPapaer):
db.insertPaper(eachPapaer)
print "n:",n,
print "Expert_ID %s is done"%words[0]
n = n + 1
db.conn.commit()
end = time.time()
print ("PaperSubDir %d is Done!"%index),
print time.strftime('%m-%d %H:%M:%S',time.localtime(time.time())),"time:",end-start,
f = open("PaperToDB.log","a")
f.write(time.strftime('%m-%d %H:%M:%S',time.localtime(time.time()))+" paperSubDir"+str(index)+" is Done! "+"total"+str(end-start) )
f.close()
db.close()
def wait_for_server_status(self, server_id, desired_status, interval_time=None, timeout=None):
interval_time = int(interval_time or self.servers_api_config.server_status_interval)
timeout = int(timeout or self.servers_api_config.server_build_timeout)
end_time = time.time() + timeout
time.sleep(interval_time)
while time.time() < end_time:
resp = self.nova_cli_client.show_server(server_id)
server = resp.entity
if server.status.lower() == ServerStates.ERROR.lower():
raise BuildErrorException('Build failed. Server with uuid "{0} entered ERROR status.'.format(server.id))
if server.status == desired_status:
break
time.sleep(interval_time)
else:
raise TimeoutException(
"wait_for_server_status ran for {0} seconds and did not "
"observe server {1} reach the {2} status.".format(timeout, server_id, desired_status)
)
return resp
def env454run_info_upload(runobj):
my_read_csv = dbUpload(runobj)
start = time.time()
my_read_csv.put_run_info()
elapsed = (time.time() - start)
print "put_run_info time = %s" % str(elapsed)
def loadIntents(node_id, urllist, intPerGroup, addrate, duration):
urlindex = 0
group = 0
start_id = 0
sleeptimer = (1.000/addrate)
tstart = time.time()
while ( (time.time() - tstart) <= duration ):
if urlindex < len(urllist):
realurlind = urlindex
else:
realurlind = 0
urlindex = 0
u = str(urllist[realurlind])
gstart = time.time()
intents,start_id = setIntentJSN(node_id, intPerGroup, group, start_id)
#print (str(intents))
#print ("Starting intent id: " + str(start_id))
result = post_json(u, intents)
#print json.dumps(intents[group])
#print ("post result: " + str(result))
gelapse = time.time() - gstart
print ("Group: " + str(group) + " with " + str(intPerGroup) + " intents were added in " + str('%.3f' %gelapse) + " seconds.")
sleep(sleeptimer)
urlindex += 1
group += 1
telapse = time.time() - tstart
#print ( "Number of groups: " + str(group) + "; Totoal " + str(args.groups * args.intPerGroup) + " intents were added in " + str(telapse) + " seconds.")
return telapse, group
def waitnewevents(evtypes, timeout_ms=1000, verbose=True):
"""Function that blocks until a certain type of event is recieved.
evttypes is a list of event type strings, recieving any of these event types termintes the block.
All such matching events are returned
"""
global ftc, nEvents, nSamples, procnEvents
start = time.time()
update()
elapsed_ms = 0
if verbose:
print "Waiting for event(s) " + str(evtypes) + " with timeout_ms " + str(timeout_ms)
evt = None
while elapsed_ms < timeout_ms and evt is None:
nSamples, nEvents2 = ftc.wait(-1, procnEvents, timeout_ms - elapsed_ms)
if nEvents2 > nEvents: # new events to process
procnEvents = nEvents2
evts = ftc.getEvents((nEvents, nEvents2 - 1))
evts = filter(lambda x: x.type in evtype, evts)
if len(evts) > 0:
evt = evts
elapsed_ms = (time.time() - start) * 1000
nEvents = nEvents2
return evt
def ana_file_sync(self,pflag):
g.tprinter('Running ana_file_sync',pflag)
self.missing_data_list = np.array(['missing'])
tbaf = 0
for i in self.data_list:
tstart = time.time()
if i in self.ana_file: ##Overvej om det virkelig er det kvikkeste!
#g.printer('is in file',pflag)
pass
else:
#g.printer('is not in file',pflag)
self.missing_data_list = np.append(self.missing_data_list,i)
print time.time()-tstart
if tbaf == 0:
g.printer('ana_file is up to date',pflag)
else:
g.printer(str(tbaf)+' files have to be added to ana_file',pflag)
for i in self.missing_data_list[1:]:
k = os.path.split(i)
info = k[-1].split('.')[-2].split('_')
num_of_zeros = len(self.ana_file[0])-len(info)-1
info = np.append(info,np.zeros(num_of_zeros))
info = np.append(info,np.array(i))
self.ana_file = np.vstack((self.ana_file,info))
def DetectSound():
# Get data from landmark detection (assuming face detection has been activated).
data = memoryProxy.getData("SoundDetected")
##The SoundDetected key is organized as follows:
##
##[[index_1, type_1, confidence_1, time_1],
## ...,
##[index_n, type_n, confidence_n, time_n]]
##
##n is the number of sounds detected in the last audio buffer,
##index is the index (in samples) of either the sound start (if type is equal to 1) or the sound end (if type is equal to 0),
##time is the detection time in micro seconds
##confidence gives an estimate of the probability [0;1] that the sound detected by the module corresponds to a real sound.
if len(data)==0:
detected=False
timestamp=time.time()
soundInfo=[]
else:
detected=True
timestamp=time.time()
soundInfo=[]
for snd in data:
soundInfo.append([ snd[0], #index of sound start/end
snd[1], #type: 1=start, 0=end
snd[2] #confidence: probability that there was a sound
])
return detected, timestamp, soundInfo
def wrapper(*args, **kargs):
t1 = time.time()
res = func(*args, **kargs)
tel = time.time()-t1
timeformated = time.strftime( "%H:%M:%S",time.gmtime(tel))
print '-'*5 + '%s took %0.3f ms' % (func.func_name + str(kargs) + str(args), (tel)*1000.0) + '|' + timeformated + '|'+ '-'*10
return res
def timer(self, object, cdata=''):
if object:
t = time.time()
r = self.render(object, cdata)
self.tagTimes[object.__class__.__name__] = time.time() - t
return r
return self.render(object, cdata)
def crawl(self):
_starttime = time.time()
if self.restrict == None:
self.restrict = "http://%s.*" % self.init_domain
print "Deadlink-crawler version 1.1"
print "Starting crawl from URL %s at %s with restriction %s\n" % (self.init_url, strftime("%Y-%m-%d %H:%M:%S", gmtime()), "http://%s.*" % self.init_domain)
while len(self.frontier) > 0:
time.sleep(self.wait_time)
next_time, next_url = self.frontier.next()
while time.time() < next_time:
time.sleep(0.5)
try:
self.visit_url(next_url[0], next_url[1])
except urllib2.URLError:
continue
self.print_deadlinks(self.deadlinks)
_elapsed = time.time() - _starttime
print "\nSummary:\n--------"
print "Crawled %d pages and checked %d links in %s time." % (self._pages, self._links, strftime("%H:%M:%S", gmtime(_elapsed)))
print "Found a total of %d deadlinks in %d different pages" % (self._dead, self._via)
if len(self.deadlinks) == 0:
exit(0)
else:
exit(2)
def illumina_files(runobj):
utils = PipelneUtils()
start = time.time()
# illumina_files_demultiplex_only(runobj)
illumina_files = IlluminaFiles(runobj)
if runobj.do_perfect:
# illumina_files.perfect_reads()
script_file_name = illumina_files.merge_perfect()
utils.run_until_done_on_cluster(script_file_name)
script_file_name = illumina_files.trim_primers_perfect()
utils.run_until_done_on_cluster(script_file_name)
else:
# illumina_files.partial_overlap_reads()
# pass
# TODO: test utils.run_until_done_on_cluster(illumina_files.partial_overlap_reads_cluster())
#TODO: add cutting to 251
script_file_name = illumina_files.partial_overlap_reads_cluster()
utils.run_until_done_on_cluster(script_file_name)
script_file_name = illumina_files.filter_mismatches_cluster()
utils.run_until_done_on_cluster(script_file_name)
# illumina_files.filter_mismatches()
# illumina_files.uniq_fa()
script_file_name = illumina_files.uniq_fa_cluster()
utils.run_until_done_on_cluster(script_file_name)
# illumina_chimera(runobj)
elapsed = (time.time() - start)
print "illumina_files time = %s" % str(elapsed)
def env454upload_main(runobj, full_upload):
"""
Run: pipeline dbUpload testing -c test/data/JJH_KCK_EQP_Bv6v4.ini -s env454upload -l debug
For now upload only Illumina data to env454 from files, assuming that all run info is already on env454 (run, run_key, dataset, project, run_info_ill tables)
Tables:
sequence_ill
sequence_pdr_info_ill
taxonomy
sequence_uniq_info_ill
"""
whole_start = time.time()
my_env454upload = dbUpload(runobj)
filenames = my_env454upload.get_fasta_file_names()
if not filenames:
logger.debug("\nThere is something wrong with fasta files or their names, please check pathes, contents and suffixes in %s." % my_env454upload.fasta_dir)
# sequences = get_sequences(my_env454upload, filenames)
for filename in filenames:
sequences = my_env454upload.make_seq_upper(filename)
if full_upload:
env454upload_seq(my_env454upload, filename, sequences)
wrapped = wrapper(my_env454upload.get_seq_id_dict, sequences)
get_seq_id_dict_time = timeit.timeit(wrapped, number=1)
logger.debug("get_seq_id_dict() took %s sec to finish" % get_seq_id_dict_time)
total_seq = env454upload_all_but_seq(my_env454upload, filenames, full_upload)
my_env454upload.check_seq_upload()
logger.debug("total_seq = %s" % total_seq)
whole_elapsed = (time.time() - whole_start)
print "The whole upload took %s s" % whole_elapsed
def docs():
s = time.time()
try:
return render_template(
'docs.html', version=skyline_version, duration=(time.time() - s)), 200
except:
return 'Uh oh ... a Skyline 500 :(', 500
def illumina_files_demultiplex_only(runobj):
start = time.time()
illumina_files = IlluminaFiles(runobj)
illumina_files.open_dataset_files()
illumina_files.split_files(compressed = runobj.compressed)
elapsed = (time.time() - start)
print "illumina_files demultiplex only time = %s" % str(elapsed)
def do_real_import(self, vsfile, filepath,mdXML,import_tags):
"""
Make the import call to vidispine, and wait for self._importer_timeout seconds for the job to complete.
Raises a VSException representing the job error if the import job fails, or ImportStalled if the timeout occurs
:param vsfile: VSFile object to import
:param filepath: filepath of the VSFile
:param mdXML: compiled metadata XML to import alongside the media
:param import_tags: shape tags describing required transcodes
:return: None
"""
import_job = vsfile.importToItem(mdXML, tags=import_tags, priority="LOW", jobMetadata={"gnm_app": "vsingester"})
job_start_time = time.time()
close_sent = False
while import_job.finished() is False:
self.logger.info("\tJob status is %s" % import_job.status())
if time.time() - job_start_time > self._importer_timeout:
self.logger.error("\tJob has taken more than {0} seconds to complete, concluding that it must be stalled.".format(self._importer_timeout))
import_job.abort()
self.logger.error("\tSent abort signal to job")
raise ImportStalled(filepath)
if time.time() - job_start_time > self._close_file_timeout and not close_sent:
vsfile.setState("CLOSED")
sleep(5)
import_job.update(noraise=False)
def treat_movement(self, rp, movement):
if movement["status"] in ["error", "done"]:
return
now = int(time.time())
movement["begin"] = now
movement["status"] = "treating"
self.memory.update_movement(movement)
# checking sources
movement["src_exist"] = True
if not os.path.exists(rp["src"]):
now = int(time.time())
movement["src_exist"] = False
movement["status"] = "error"
movement["end"] = now
movement["error"] = "source file : %s does not exist" % (rp["src"])
self.memory.update_movement(movement)
return
# adapt in function of protocol
if movement["protocol"] == "cp":
self.do_cp(movement, rp)
return
if movement["protocol"] == "ftp":
self.do_ftp(movement, rp)
return
now = int(time.time())
movement["status"] = "error"
movement["end"] = now
movement["error"] = "protocol %s not known" % (movement["protocol"])
self.memory.update_movement(movement)
rp["status"] = "done"
self.memory.update_relative_path(rp)
def wait_ms(prev_time_ms, wait_time_ms):
t = time.time()*1000. - prev_time_ms
while t < wait_time_ms:
if t < 0:
break
sleep((wait_time_ms - t)/1000.)
t = time.time()*1000. - prev_time_ms
def test_vrr():
import time
xa,ya,za = 0.,0.,0.
xb,yb,zb = 0.,0.,0.
xc,yc,zc = 0.,0.,0.
xd,yd,zd = 0.,0.,0.
norma = normb = normc = normd = 1.
alphaa = alphab = alphac = alphad = 1.
la,ma,na = 0,0,0
lc,mc,nc = 0,0,0
M = 0
t0 = time.time()
val1 = vrr((xa,ya,za),norma,(la,ma,na),alphaa,
(xb,yb,zb),normb,alphab,
(xc,yc,zc),normc,(lc,mc,nc),alphac,
(xd,yd,zd),normd,alphad,M)
t1 = time.time()
val2 = vrr((xc,yc,zc),normc,(lc,mc,nc),alphac,
(xd,yd,zd),normd,alphad,
(xa,ya,za),norma,(la,ma,na),alphaa,
(xb,yb,zb),normb,alphab,M)
t2 = time.time()
print "Values: ",val1,val2
print "Timings: ",t1-t0,t2-t1
return
def alligator():
import time
start_time = time.time()
product_sum = 0
check_pandigital = False
set_of_products = set()
def pandigital(l):
seen = set() # Start with an empty set
for i in l:
if i in seen:
return False
seen.add(i) # If it is not in the seen set, then add it.
return True
for x in range(1, 2000):
for y in range(x+1, 2000):
z = x * y
digits = str(x) + str(y) + str(z) # This takes all three numbers and joins them into one long number
if (len(digits) == 9 and "0" not in (digits) and pandigital(digits)):
if z not in set_of_products:
set_of_products.add(z)
product_sum = product_sum + z
print ("The sum of the pandigital products is: ", product_sum)
print("--- %s seconds ---" % (time.time() - start_time))
def func(*args):
import time
start = time.time()
ret = f(*args)
took = time.time() - start
print("%s took %f" % (f.__name__,took))
return ret
def DetectSoundLocation():
# Get data from landmark detection (assuming face detection has been activated).
data = memoryProxy.getData("SoundLocated")
##The SoundDetected key is organized as follows:
##
##[ [time(sec), time(usec)],
##
## [azimuth(rad), elevation(rad), confidence],
##
## [Head Position[6D]]
##]
if len(data)==0:
detected=False
timestamp=time.time()
soundInfo=[]
else:
detected=True
#timestamp=data[0][0]+1E-6*data[0][1] #this works but only if a sound is located
timestamp=time.time()
soundInfo=[]
for snd in data:
soundInfo.append([ snd[1][0], #azimuth angle
snd[1][1], #elvation angle
snd[1][2], #confidence: probability that there was a sound
snd[2]]) #Headposition 6D
return detected, timestamp, soundInfo
def _log_rate(output_f,d, message=None):
"""Log a message for the Nth time the method is called.
d is the object returned from init_log_rate
"""
import time
if d[2] <= 0:
if message is None:
message = d[4]
# Average the rate over the length of the deque.
d[6].append(int( d[3]/(time.time()-d[1])))
rate = sum(d[6])/len(d[6])
# Prints the processing rate in 1,000 records per sec.
output_f(message+': '+str(rate)+'/s '+str(d[0]/1000)+"K ")
d[1] = time.time()
# If the print_rate was specified, adjuect the number of records to
# aaproximate that rate.
if d[5]:
target_rate = rate * d[5]
d[3] = int((target_rate + d[3]) / 2)
d[2] = d[3]
d[0] += 1
d[2] -= 1
def exe():
init_time = time_time()
pder = PornDetector()
print(time.time() - init_time)
print("ARE YOU OK?")
while True
pder.porn_detector(True, False)
print(time.time() - init_time)
def timed(*args, **kw):
ts = time.time()
result = method(*args, **kw)
te = time.time()
print '%r (%r, %r) %2.2f sec' % \
(method.__name__, args, kw, te-ts)
return result
def update_output(self):
if (time.time() - self.last_output > self.OUTPUT_PERIOD):
print("O%s A%s G%s" % (
pf(self.data["orientation"]["data"]),
pf(self.data["acceleration"]["data"]),
pf(self.data["gyroscope"]["data"]),
))
self.last_output = time.time()
send_notification(
2, products_list[productlist[productcounter]])
#TODO make a document to log all this information for debugging
print(products_list[productlist[productcounter]][12:], "online at ",
date, " code: ", 200, " using header #",
headerlist[headercounter], "proxy ",
proxy_dict[proxylist[proxycounter]]['http'], " #",
proxylist[proxycounter])
else:
print(products_list[productlist[productcounter]][12:], "offline at",
date, " code", 400, " using header #", headerlist[headercounter],
"proxy ", proxy_dict[proxylist[proxycounter]]['http'], " #",
proxylist[proxycounter])
# put proxie in timeout dictoinary
time_key = time.time()
timeoutdict[time_key] = proxy_dict[proxylist[proxycounter]]
# take proxie out of active proxies
proxy_dict.pop(proxylist[proxycounter])
# reorder dictionary
proxy_dict = proxyorganize(proxy_dict)
# increase time per request
delaylimit += 2
timeoutonweb = False
# used to make new list of random numbers
proxycounter = proxy_dict.__len__()
#add proxie back into active
for x in timeoutdict.keys():
if (x + 400 < time.time()):
proxy_dict[proxy_dict.__len__()] = timeoutdict[x]
def train_model_and_evaluate(lods, out_data, seed=10):
np.random.seed(seed)
tf.compat.v1.set_random_seed(seed)
n_knob_cols = len(lods.config['COLS_KNOBS'])
nn_params = HYPER_PARAMS['nn_params']
kpca_params = HYPER_PARAMS['kpca_params']
tmp_trainval = lods.trainval
tmp_shared_trainval = lods.shared_trainval
if N_TRAIN_PER_JOB != -1:
tmp_trainval = lods.trainval.get_x(N_TRAIN_PER_JOB)
if N_SHARED_TRAIN_PER_JOB != -1:
tmp_shared_trainval = lods.shared_trainval.get_x(
N_SHARED_TRAIN_PER_JOB)
if tmp_trainval is not None:
logging.info("shape of remaining trainval (X): {}".format(
tmp_trainval.X.shape))
else:
logging.info("tmp_trainval is None (perhaps because of get_x(0))")
if tmp_shared_trainval is not None:
logging.info("shape of remaining shared trainval (X): {}".format(
tmp_shared_trainval.X.shape))
else:
logging.info(
"tmp_shared_trainval is None (perhaps because of get_x(0))")
if tmp_trainval is None:
# in case we're invoking dataset.get_x(0)
ds_train = tmp_shared_trainval
else:
ds_train = tmp_trainval + tmp_shared_trainval
X_train = np.hstack([ds_train.a, ds_train.X, ds_train.Y])
y_train = ds_train.targets.ravel()
logging.info("Fitting KPCA on data of shape: {}".format(X_train.shape))
# Make PCA and fit on loaded data
fit_t = time.time()
pca = KernelPCA(**kpca_params)
pca.altered_centroids = None
logging.info("Fitting KPCA on data of shape: {}".format(ds_train.Y.shape))
if ENCODING_STRATEGY == 'shared':
shared_train = lods.shared_trainval.get_x(N_OBS)
pca.fit(ds_train.Y)
encods_shared = pca.transform(shared_train.Y)
centroids = compute_centroids(encods_shared, shared_train.a)
else:
encods = pca.fit_transform(ds_train.Y)
centroids = compute_centroids(encods, ds_train.a)
pca.centroids = centroids # thisis why I love Python! :-)
fit_t = time.time() - fit_t
logging.info("KPCA fitting time is: {} minutes and {} seconds".format(
fit_t // 60, int(fit_t / 60)))
# Adjust the X vector by transforming Y into job's centroid
X, y = translate_X_y(X_train, y_train, pca.centroids, n_knob_cols)
# Make and fit a NN Regressor
logging.info("Fitting regressor on data of shapes: {}, {}".format(
X.shape, y.shape))
reg = NNregressor(with_calibration=True,
**nn_params,
random_state=seed,
v1_compat_mode=True)
reg.fit(X, y, log_time=True)
training_mape = reg.MAPE(X, y)
logging.info("Training Error: {:.2f}%".format(training_mape))
out_data['training_errs'].append(training_mape)
if ENCODING_STRATEGY == 'shared':
observed_traces = lods.shared_traincomplement.get_x(N_OBS)
else:
observed_traces = lods.traincomplement.get_x(N_OBS)
logging.info("observed_traces description: ")
observed_traces.describe()
observed_traces_slices = observed_traces.slice_by_job_id(
alias_to_id=lods.alias_to_id)
test_aliases = sorted(list(set(lods.test.a.ravel())))
for test_job in observed_traces_slices:
evaluate(test_job, pca, reg, lods, observed_traces,
observed_traces_slices, out_data, test_aliases)
# out_data['kpcas'].append(pca)
# Append trained regressor information to output_data
out_data['regressors'].append(reg.get_persist_info())
persist_data(copyDict(out_data), DATA_FNAME)
def main():
colormap='viridis'
job_id = os.environ['PBS_JOBID']
log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout)
args = build_argparser().parse_args()
model_xml = args.model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin))
device=args.device
fp16=True
if device=="CPU":
fp16=False
# Plugin initialization for specified device and load extensions library if specified
plugin = IEPlugin(device=device)
# Read IR
net = IENetwork(model=model_xml, weights=model_bin)
assert len(net.inputs.keys()) == 1, "Sample supports only single input topologies"
bn = "relu_1/Relu"
print(bn)
# add the last convolutional layer as output
net.add_outputs(bn)
fc="predictions_1/MatMul"
# name of the inputs and outputs
input_blob = next(iter(net.inputs))
out_blob = "predictions_1/Sigmoid"
net.batch_size = 1
exec_net = plugin.load(network=net)
n,c,h,w=net.inputs[input_blob].shape
files=glob.glob(os.getcwd()+args.input[0])
if not os.path.isdir(args.output_dir):
os.makedirs(args.output_dir, exist_ok=True)
f=open(os.path.join(args.output_dir, 'result'+job_id+'.txt'), 'w')
f1=open(os.path.join(args.output_dir, 'stats'+job_id+'.txt'), 'w')
progress_file_path = os.path.join(args.output_dir, "progress"+job_id+".txt")
print(progress_file_path)
time_images=[]
tstart=time.time()
for index_f, file in enumerate(files):
[image1,image]= read_image(file)
t0 = time.time()
for i in range(args.number_iter):
res = exec_net.infer(inputs={input_blob: image1})
#infer_time.append((time()-t0)*1000)
infer_time = (time.time() - t0)*1000
log.info("Average running time of one iteration: {} ms".format(np.average(np.asarray(infer_time))))
if args.perf_counts:
perf_counts = exec_net.requests[0].get_perf_counts()
log.info("Performance counters:")
print("{:<70} {:<15} {:<15} {:<15} {:<10}".format('name', 'layer_type', 'exet_type', 'status', 'real_time, us'))
for layer, stats in perf_counts.items():
print("{:<70} {:<15} {:<15} {:<15} {:<10}".format(layer, stats['layer_type'], stats['exec_type'],
stats['status'], stats['real_time']))
res_pb = res[out_blob]
probs=res_pb[0][0]
print("Probability of having disease= "+str(probs)+", performed in " + str(np.average(np.asarray(infer_time))) +" ms")
# Class Activation Map
t0 = time.time()
cam=class_activation_map_openvino(res, bn, fc , net, fp16)
cam_time=(time.time() - t0) * 1000
print("Time for CAM: {} ms".format(cam_time))
fig,ax = plt.subplots(1,2)
# Visualize the CAM heatmap
cam = (cam - np.min(cam))/(np.max(cam)-np.min(cam))
im=ax[0].imshow(cam, cmap=colormap)
ax[0].axis('off')
plt.colorbar(im,ax=ax[0],fraction=0.046, pad=0.04)
# Visualize the CAM overlaid over the X-ray image
colormap_val=cm.get_cmap(colormap)
imss=np.uint8(colormap_val(cam)*255)
im = Image.fromarray(imss)
width, height = image.size
cam1=resize_image(im, (height,width))
heatmap = np.asarray(cam1)
img1 = heatmap [:,:,:3] * 0.3 + image
ax[1].imshow(np.uint16(img1))
plt.xticks([]), plt.yticks([]) # to hide tick values on X and Y axis
plt.savefig(os.path.join(args.output_dir, 'result'+job_id+'_'+str(index_f)+'.png'), bbox_inches='tight', pad_inches=0,dpi=300)
avg_time = round((infer_time/args.number_iter), 1)
#f.write(res + "\n Inference performed in " + str(np.average(np.asarray(infer_time))) + "ms")
f.write("Pneumonia probability: "+ str(probs) + ", Inference performed in " + str(avg_time) + "ms \n")
time_images.append(avg_time)
simpleProgressUpdate(progress_file_path,index_f* avg_time , (len(files)-1)* avg_time)
f1.write(str(np.average(np.asarray(time_images)))+'\n')
f1.write(str(1))
from datetime import datetime
import argparse
from urllib import parse
import os, sys
import subprocess
import pandas as pd
from rdkit import Chem
from rdkit.Chem import Descriptors
from rdkit.Chem.rdMolDescriptors import CalcMolFormula
from pubchempy import Compound, get_compounds, get_synonyms
import tqdm
from tqdm import tqdm
from tqdm import tqdm_notebook
import time
s = time.time()
# Loading SMILES
print('Loading SMILES')
# Read appropriate csv file below
SMILES = pd.read_csv('drug_central_drugs.csv', index_col=0)['SMILES'].tolist()
def multi_preprocess_smi(smi):
new_dict = {}
try:
# Filter 1- Convert to Canonical Smiles
mol = Chem.MolFromSmiles(smi)
from pandas import DataFrame
#import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
import random
import numpy as np
from time import time
import timeit
import sys
import time
number_of_data_points = int(sys.argv[1])
number_of_clusters = int(sys.argv[2])
def get_random_cluster_points(number_points, number_dim):
mu = np.random.randn()
sigma = np.random.randn()
p = sigma * np.random.randn(number_points, number_dim) + mu
return p
p = get_random_cluster_points(number_of_data_points, 2)
print(time.time())
kmeans = KMeans(n_clusters=number_of_clusters, n_jobs=-1).fit(p)
print(time.time())
centroids = kmeans.cluster_centers_
def serial_reader(self, dev, channel, writer_packet_queue,
socket_packet_queue):
"""
Thread responsible for reading from serial port, parsing the output and
storing parsed packets into queue.
:param dev: Device sniffing.
:param channel: Channel to sniff passed as argument.
:param writer_packet_queue: Queue string packet to write.
:param socket_packet_queue: Queue string packet to send by socket.
"""
sleep(2)
timeout = time.time() + self.connection_open_timeout if self.connection_open_timeout else None
while self.running.is_set():
try:
self.serial = Serial(dev, timeout=1, exclusive=True)
break
except Exception as e:
if timeout and time.time() > timeout:
self.running.clear()
raise Exception(
"Could not open serial connection to sniffer before timeout of {} seconds".format(
self.connection_open_timeout))
self.logger.debug(
"Can't open serial device: {} reason: {}".format(dev, e))
sleep(0.5)
try:
self.serial.reset_input_buffer()
self.serial.reset_output_buffer()
init_cmd = []
init_cmd.append(b'')
init_cmd.append(b'sleep')
init_cmd.append(b'channel ' + bytes(str(channel).encode()))
for cmd in init_cmd:
self.serial_queue.put(cmd)
# Function serial_write appends twice '\r\n' to each command, so we have to calculate that for the echo.
init_res = self.serial.read(
len(b"".join(c + b"\r\n\r\n" for c in init_cmd)))
if not all(cmd.decode() in init_res.decode() for cmd in init_cmd):
msg = "{} did not reply properly to setup commands. Please re-plug the device and make sure firmware is correct. " \
"Recieved: {}\n".format(self, init_res)
self.logger.error(msg)
self.serial_queue.put(b'receive')
self.setup_done.set()
buf = b''
while self.running.is_set():
ch = self.serial.read()
if ch == b'':
continue
elif ch != b'\n' and ch != '\n':
buf += ch
else:
m = search(self.RCV_REGEX, str(buf))
if m:
packet = a2b_hex(m.group(1)[:-4])
rssi = int(m.group(2))
lqi = int(m.group(3))
timestamp = int(m.group(4)) & 0xffffffff
channel = int(channel)
writer_packet_queue.put(
self.pcap_packet(packet, self.dlt, channel, rssi,
lqi,
self.correct_time(timestamp)))
socket_packet_queue.put(
self.pcap_packet(packet, self.dlt, channel, rssi,
lqi,
self.correct_time(timestamp)))
buf = b''
except (
serialutil.SerialException, serialutil.SerialTimeoutException) as e:
self.logger.error(
"Cannot communicate with serial device: {} reason: {}".format(
dev, e))
finally:
self.setup_done.set() # In case it wasn't set before.
if self.running.is_set(): # Another precaution.
self.stop_sig_handler()
def restart(self):
self.start = time.time()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--batch_size', type=int)
parser.add_argument('--lr', type=float)
parser.add_argument('--epochs', type=int, default=20)
parser.add_argument('--eval_every', type=int, default=10)
parser.add_argument('--actf',
type=str,
choices=['sigmoid', 'relu', 'linear'],
default='linear')
args = parser.parse_args()
######
# 4.5 YOUR CODE HERE
batch_size = args.batch_size
lr = args.lr
epochs = args.epochs
eval_every = args.eval_every
train_loader, val_loader = load_data(batch_size)
model, loss_fnc, optimizer = load_model(lr)
t = 0
train_accs = []
tvals = []
valid_accs = []
timearray = []
st = time.time()
for epoch in range(0, epochs):
accum_loss = 0
total_corr = 0
for i, batch in enumerate(train_loader):
feats, label = batch
optimizer.zero_grad()
predictions = model(feats)
batch_loss = loss_fnc(input=predictions.squeeze(),
target=label.float())
accum_loss = accum_loss + batch_loss
batch_loss.backward()
optimizer.step()
corr = (predictions > 0.5).squeeze().long() == label.long()
total_corr = total_corr + int(corr.sum())
end = time.time()
if (t + 1) % (args.eval_every) == 0:
valid_acc = evaluate(model, val_loader)
train_acc = evaluate(model, train_loader)
print("Epoch: {}, Step: {} | Loss: {} | Valid acc: {}".format(
epoch + 1, t + 1, accum_loss / eval_every, valid_acc))
accum_loss = 0
valid_accs = valid_accs + [valid_acc]
train_accs = train_accs + [train_acc]
tvals = tvals + [t + 1]
timearray = timearray + [end - st]
t = t + 1
print("Train acc:{}".format(
float(total_corr) / len(train_loader.dataset)))
print("Time", timearray[len(timearray) - 1])
#lines = plt.plot(tvals, train_accs,'r', valid_accs,'b')
plt.plot(tvals, sg.savgol_filter(train_accs, 9, 3, mode="nearest"), 'r')
plt.plot(tvals, sg.savgol_filter(valid_accs, 9, 3, mode="nearest"), 'b')
plt.title("Accuracy " + " vs." + " Steps")
plt.xlabel("Steps")
plt.ylabel("Accuracy")
plt.legend(['Training', 'Validation'])
plt.show()
plt.close()
plt.plot(timearray, sg.savgol_filter(train_accs, 9, 3, mode="nearest"),
'r')
plt.plot(timearray, sg.savgol_filter(valid_accs, 9, 3, mode="nearest"),
'b')
plt.title("Accuracy " + " vs." + " Time")
plt.xlabel("Time")
plt.ylabel("Accuracy")
plt.legend(['Training', 'Validation'])
plt.show()
plt.close()
def RunH2Oaiglm_ptr(arg):
import h2o4gpu as h2o4gpu
import time
trainX, trainY, validX, validY, trainW, fortran, mTrain, n, mvalid, intercept, lambda_min_ratio, n_folds, n_alphas, n_lambdas, n_gpus = arg
# assume ok with 32-bit float for speed on GPU if using this wrapper
if trainX is not None:
trainX.astype(np.float32)
if trainY is not None:
trainY.astype(np.float32)
if validX is not None:
validX.astype(np.float32)
if validY is not None:
validY.astype(np.float32)
if trainW is not None:
trainW.astype(np.float32)
print("Begin Setting up Solver")
os.system(
"rm -f error.txt ; touch error.txt ; rm -f varimp.txt ; touch varimp.txt"
) ## for visualization
enet = h2o4gpu.ElasticNetH2O(n_gpus=n_gpus,
fit_intercept=intercept,
lambda_min_ratio=lambda_min_ratio,
n_lambdas=n_lambdas,
n_folds=n_folds,
n_alphas=n_alphas)
print("End Setting up Solver")
## First, get backend pointers
sourceDev = 0
t0 = time.time()
a, b, c, d, e = enet.prepare_and_upload_data(trainX,
trainY,
validX,
validY,
trainW,
source_dev=sourceDev)
t1 = time.time()
print("Time to ingest data: %r" % (t1 - t0))
## Solve
if 1 == 1:
print("Solving")
t0 = time.time()
order = 'c' if fortran else 'r'
double_precision = 0 # Not used
store_full_path = 0
enet.fit_ptr(mTrain,
n,
mvalid,
double_precision,
order,
a,
b,
c,
d,
e,
source_dev=sourceDev)
t1 = time.time()
print("Done Solving")
print("Time to train H2O AI ElasticNetH2O: %r" % (t1 - t0))
return r, W, arrVW1, arrVW2 ,arrVW3 ,arrVW4
fhandle.close()
except IOError as ex:
print('There was an error reading the instance')
raise Exception
sys.exit(1)
if __name__ == "__main__":
fname = sys.argv[1]
pool = ThreadPool(processes=1)
n, W, arrVW1,arrVW2 ,arrVW3 ,arrVW4 = readData(fname)
start_time = time.time()
X = 0
t = pool.apply_async(heristic, (n, W, X, arrVW1))
X1, W1 = t.get()
time.sleep(0.0001)
t1 = pool.apply_async(heristic, (n, W1, X1, arrVW2))
X2, W2 = t1.get()
time.sleep(0.0001)
t2 = pool.apply_async(heristic, (n, W2, X2, arrVW3))
X3, W3 = t2.get()
time.sleep(0.0001)
t3 = pool.apply_async(heristic, (n, W3, X3, arrVW4))
Xfinal, W4 = t3.get()
labels = sshmm.labels
print('\tModel lables are: ', labels)
print()
print('Testing %s algorithm load disagg...' % algo_name)
acc = Accuracy(len(labels), folds)
print()
print('Connecting to EMU2 on %s...' % (device))
emu2 = serial.Serial(dev, 115200, timeout=1)
y0 = y1 = -1
while True:
msg = emu2.readlines()
ts = int(time.time())
dt = datetime.datetime.fromtimestamp(ts)
if msg == [] or msg[0].decode()[0] != '<':
continue
msg = ''.join([line.decode() for line in msg])
try:
tree = et.fromstring(msg)
except:
continue
if tree.tag == 'InstantaneousDemand':
power = int(tree.find('Demand').text, 16)
power = int(power * precision)
def GetCurrentUnixTime():
return int(time.time() * 1000.0)
def expired(self):
return int(time.time()) > self.expire_at
def train2(self, epochs, DQN, memory, game):
batch_size = 32
replay_memory_size = 1000
play_steps = 100
learning_steps = 200
frame_repeat = 12
display_step = 200
loss_list = []
total_steps = 0
epoch_time = 0.0
for epoch in range(epochs):
start_time = time.time()
# print("Playing...")
game.new_episode()
# game_step_count = 0
# game_steps = []
for play_step in range(play_steps):
s1 = game.get_state().screen_buffer #[3,480,640] uint8
# a = get_eps_action(epoch, actions, preprocess(s1))
s1 = s1.reshape([1, 3, resolution[0], resolution[1]])
state = torch.from_numpy(preprocess(s1)).cuda()
# best_action_index = get_best_action(state)
# print (state.size())
# fasdf
q = DQN[0](Variable(state))
val, index = torch.max(q, 1)
index = index.data.cpu().numpy()[0]
action = actions[index]
reward = game.make_action(action,
frame_repeat) / float(frame_repeat)
isterminal = game.is_episode_finished()
# game_step_count +=1
s2 = game.get_state().screen_buffer if not isterminal else None
memory.add_transition(s1, index, s2, isterminal, reward)
if game.is_episode_finished():
# score = game.get_total_reward()
# train_scores.append(score)
# train_episodes_finished += 1
# game_steps.append(game_step_count)
game.new_episode()
# game_step_count = 0
for learning_step in range(learning_steps):
# if learning_step%10==0:
# print (learning_step)
idxs = sample(range(0, memory.size), batch_size)
s1 = memory.s1[idxs]
batch = Variable(preprocess_pytorch(s1))
self.optimizer.zero_grad()
loss, dif, prec_sum = self.forward(batch, DQN)
loss.backward()
self.optimizer.step()
if total_steps % display_step == 0: # and batch_idx == 0:
print(
'Train Epoch: {}/{}'.format(epoch, epochs),
'epoch_time:{:.2f}'.format(epoch_time),
'Loss:{:.4f}'.format(loss.data.item()),
'dif:{:.4f}'.format(dif.data.item()),
'prec_sum:{:.4f}'.format(prec_sum.data.item()),
)
if total_steps != 0:
loss_list.append(loss.data.item())
total_steps += 1
epoch_time = time.time() - start_time
if epoch % 10 == 0 and epoch > 2:
#Save params
save_params_v3(save_dir=exp_path_2,
model=self,
epochs=epoch + start_epoch)
if len(loss_list) > 7:
#plot the training curve
plt.plot(loss_list[2:])
# save_dir = home+'/Documents/tmp/Doom/'
plt_path = exp_path_2 + 'training_plot.png'
plt.savefig(plt_path)
print('saved training plot', plt_path)
plt.close()
def __init__(self):
self.start = time.time()
import cobra
import matplotlib.pyplot as plt
import multiprocessing as mp
import numpy as np
import pandas as pd
import scipy.stats
from time import time
start_time = time.time()
# modifiable variables: cell # (max 8444), gene # (max 1800), threshold is % unique cells
cells = 2000
genes = 1800
threshold = .7
threads = mp.cpu_count() - 1
plot = False
modelOriginal = cobra.io.load_matlab_model('Recon3D.mat')
met_name = "etoh_"
modelOriginal.reactions.get_by_id("ETOHt").lower_bound = 1000
modelOriginal.reactions.get_by_id("ETOHt").upper_bound = 1000
modelOriginal.objective = "ALCD2x"
# read in scRNAseq data set
data = pd.read_csv('GSE115469_Data.csv', index_col=0)
# gene names should be the 0th column, which is the index column
genes_in_sc = data.index
# read in the map from gene name -> model name
f = open('map.txt', 'r')
dict_temp = f.readlines()
def get_time(self):
end = time.time()
m, s = divmod(end - self.start, 60)
h, m = divmod(m, 60)
time_str = "%02d:%02d:%02d" % (h, m, s)
return time_str
frame.append(byte)
str = ''
for i in frame:
str += chr(i)
print len(str)
while True:
start = bufLen*iter
end = bufLen+bufLen*iter
indata = str[start:end]
readSoFar = len(indata)
if end > len(str) and len(decoded):
if time.time() - lastByteTime > 2:
#save wav file
save_wav()
print mFramesCount
print iter
exit(0)
elif indata:
inbuffer += indata
if len(inbuffer) == bufLen:
seqNum, SI_received, PV_received = struct.unpack('BBh', inbuffer[0:4])
seqNum = (seqNum >> 3)
print "Frame sequence number: %d" % seqNum
print "HDR_1 local: %d, HDR_1 received: %d" % (SI_Dec, SI_received)
print "HDR_2 local: %d, HDR_2 received: %d" % (PV_Dec, PV_received)
def keyGo(self, result=None):
if not self.timerentry_showendtime.value:
self.timerentry_endtime.value = self.timerentry_starttime.value
self.timer.resetRepeated()
self.timer.timerType = {
"wakeup": TIMERTYPE.WAKEUP,
"wakeuptostandby": TIMERTYPE.WAKEUPTOSTANDBY,
"autostandby": TIMERTYPE.AUTOSTANDBY,
"autodeepstandby": TIMERTYPE.AUTODEEPSTANDBY,
"standby": TIMERTYPE.STANDBY,
"deepstandby": TIMERTYPE.DEEPSTANDBY,
"reboot": TIMERTYPE.REBOOT,
"restart": TIMERTYPE.RESTART
}[self.timerentry_timertype.value]
self.timer.afterEvent = {
"nothing": AFTEREVENT.NONE,
"wakeuptostandby": AFTEREVENT.WAKEUPTOSTANDBY,
"standby": AFTEREVENT.STANDBY,
"deepstandby": AFTEREVENT.DEEPSTANDBY
}[self.timerentry_afterevent.value]
if self.timerentry_type.value == "once":
self.timer.begin, self.timer.end = self.getBeginEnd()
if self.timerentry_timertype.value == "autostandby" or self.timerentry_timertype.value == "autodeepstandby":
self.timer.begin = int(time()) + 10
self.timer.end = self.timer.begin
self.timer.autosleepinstandbyonly = self.timerentry_autosleepinstandbyonly.value
self.timer.autosleepdelay = self.timerentry_autosleepdelay.value
self.timer.autosleeprepeat = self.timerentry_autosleeprepeat.value
# Ensure that the timer repeated is cleared if we have an autosleeprepeat
if self.timerentry_type.value == "repeated":
self.timer.resetRepeated()
self.timerentry_type.value = "once" # Stop it being set again
if self.timerentry_type.value == "repeated":
if self.timerentry_repeated.value == "daily":
for x in (0, 1, 2, 3, 4, 5, 6):
self.timer.setRepeated(x)
if self.timerentry_repeated.value == "weekly":
self.timer.setRepeated(self.timerentry_weekday.index)
if self.timerentry_repeated.value == "weekdays":
for x in (0, 1, 2, 3, 4):
self.timer.setRepeated(x)
if self.timerentry_repeated.value == "user":
for x in (0, 1, 2, 3, 4, 5, 6):
if self.timerentry_day[x].value:
self.timer.setRepeated(x)
self.timer.repeatedbegindate = self.getTimestamp(self.timerentry_repeatedbegindate.value, self.timerentry_starttime.value)
if self.timer.repeated:
self.timer.begin = self.getTimestamp(self.timerentry_repeatedbegindate.value, self.timerentry_starttime.value)
self.timer.end = self.getTimestamp(self.timerentry_repeatedbegindate.value, self.timerentry_endtime.value)
else:
self.timer.begin = self.getTimestamp(time.time(), self.timerentry_starttime.value)
self.timer.end = self.getTimestamp(time.time(), self.timerentry_endtime.value)
# when a timer end is set before the start, add 1 day
if self.timer.end < self.timer.begin:
self.timer.end += 86400
self.saveTimer()
self.close((True, self.timer))
x = Activation('relu')(x)
# x = Dense(256)(x)
# x = Activation('relu')(x)
# x = Dense(256)(x)
# x = Activation('relu')(x)
x = Dense(num_classes)(x)
x = Activation('sigmoid')(x)
model = Model(inputs=image, outputs=x)
model.summary()
adam = optimizers.Adam(lr=.001)
# model.load_weights("vgg19_att.h5")
model.compile(loss='binary_crossentropy',
optimizer=adam,
metrics=['accuracy'])
start = time.time()
model.fit(X_train,
y_train,
epochs=20,
verbose=1,
validation_data=(X_test, y_test),
batch_size=32)
end = time.time()
time = end - start
print("TIME: ", time)
model.save_weights("vgg16_att.h5")
# pred = model.predict(X_test)
# prediction = []
# for p in pred:
def run(model_name,
seed,
target,
number_of_patients,
test_size,
models,
params,
X,
y,
AUC_test_dict,
Importances,
Probabilities,
save_csv=False,
save_probabilities=True,
save_models=False,
imputation_method='mean',
is_creatinine=False,
is_continuous=False):
start = time.time()
time_now = datetime.now(central)
print("Time when start: ", time_now.strftime("%H:%M:%S"))
continuous = True
if target == 'target_bin':
continuous = False
path_to_add = 'Hemoglobin/'
if is_creatinine:
path_to_add = 'Creatinine/'
path_to_save = '../../Data/Generated_csv/' + path_to_add
path_to_save_models = '../../Data/Results/' + path_to_add + 'Models/'
#run only on a sample of the patients
X_sample = X.iloc[:number_of_patients]
y_sample = y.iloc[:number_of_patients]
print(seed)
#sample
if not is_continuous:
train_X, test_X, train_y, test_y = train_test_split(
X_sample,
y_sample,
stratify=y_sample[target],
test_size=test_size,
random_state=seed)
if is_continuous:
train_X, test_X, train_y, test_y = train_test_split(
X_sample, y_sample, test_size=test_size, random_state=seed)
# function to test a model trained on all the data but tested on a filtered testing set
filter_after = False
if filter_after:
test_X = test_X.loc[test_X.Hb_value_initial <= 10]
test_y = test_y.loc[test_y.index.isin(test_X.index)]
cols = train_X.columns
if model_name == 'Log' or model_name == 'Lin':
scaler = StandardScaler()
train_X = scaler.fit_transform(train_X)
test_X = scaler.transform(test_X)
if save_csv:
#save the csv for reproductibility and OCT
train_X.to_csv(path_to_save + 'train_and_test/' + imputation_method +
'_train_X_seed_' + str(seed) + '.csv')
test_X.to_csv(path_to_save + 'train_and_test/' + imputation_method +
'_test_X_seed_' + str(seed) + '.csv')
train_y.to_csv(path_to_save + 'train_and_test/' + imputation_method +
'_train_Y_seed_' + str(seed) + '.csv')
test_y.to_csv(path_to_save + 'train_and_test/' + imputation_method +
'_test_Y_seed_' + str(seed) + '.csv')
# Uncomment this code to do a grid search within the boostrapping
#if model_name=='OCT':
# crit='gini'
# print('starting grid search')
#
# grid = iai.GridSearch(
# iai.OptimalTreeClassifier(
# random_seed=1,
# ),
# max_depth=range(5, 7), #do range
# criterion=crit,
# ls_num_tree_restarts=300
# )
# grid.fit(train_X, train_y[target])
# model = grid.get_learner()
# print(grid.score(test_X, test_y[target], criterion='auc'))
# print(grid.get_best_params())
#elif model_name=='ORT':
# crit='mse'
# print('starting grid search ORT')
#
# grid = iai.GridSearch(
# iai.OptimalTreeRegressor(
# random_seed=1,
# ),
# max_depth=range(3, 10), #do range
# criterion=crit,
# ls_num_tree_restarts=300)
# grid.fit(train_X, train_y[target])
# model = grid.get_learner()
# print(grid.score(test_X, test_y[target], criterion='mse'))
# print(grid.get_best_params())
elif model_name != 'Baseline':
model = models[model_name](**params[model_name])
model.fit(train_X, train_y[target])
# uncomment this code if you want to load and already trained tree
#if model_name=='OCT':
#file_path=path_to_save_models+model_name+'_'+target+'_'+str(number_of_patients) +'_patients_test_size_'+str(test_size)+'mean'+'_seed'+str(1)
#model=iai.read_json(file_path)
if not is_continuous:
if model_name == 'OCT':
preds = model.predict_proba(test_X)['1']
elif model_name == 'Baseline':
preds = test_y[target + '_baseline'].values
else:
preds = model.predict_proba(test_X)[:, 1]
AUC_test = roc_auc_score(test_y[target], preds)
if not is_creatinine:
test_X.reset_index(inplace=True)
test_y.reset_index(inplace=True)
# test the performances on a filtered dataset
#subset_list = test_X.loc[test_X.Hb_value_initial<10].index
#suberror = roc_auc_score(test_y.loc[test_y.index.isin(subset_list)][target], preds[subset_list])
#print("Normal :",AUC_test)
#print("Sub :",suberror)
# if want to ouptut error on subgroup of patients
#AUC_test=suberror
if is_continuous:
if model_name == 'ORT':
preds = model.predict(test_X)
elif model_name == 'Baseline':
preds = test_y[target + '_baseline'].values
else:
preds = model.predict(test_X)
AUC_test = mean_absolute_error(test_y[target], preds)
if not is_creatinine:
test_X.reset_index(inplace=True)
test_y.reset_index(inplace=True)
#subset_list = test_X.loc[test_X.Hb_value_initial<8].index
#suberror = mean_absolute_error(test_y.loc[test_y.index.isin(subset_list)][target], preds[subset_list])
#print("Normal :",AUC_test)
#print("Sub :",suberror)
# if want to ouptut error on subgroup of patients
#AUC_test=suberror
AUC_test_dict[target][model_name][seed] = AUC_test
if save_probabilities:
prob = pd.DataFrame()
prob[target] = test_y[target]
#prob['proba_'+target]=preds.reset_index()
if model_name == 'OCT':
prob['proba_' + target] = preds.values
else:
prob['proba_' + target] = preds
Probabilities[target][model_name][seed] = prob
if model_name == 'Baseline':
return AUC_test
if save_models:
file_path = path_to_save_models + model_name + '_' + target + '_' + str(
number_of_patients) + '_patients_test_size_' + str(
test_size) + imputation_method + '_seed' + str(seed)
if model_name == 'OCT' or model_name == 'ORT':
model.write_json(file_path)
model.write_html(file_path + '.html')
else:
f = open(file_path, "wb")
pickle.dump(model, f)
f.close()
#add feature importance
if model_name == 'OCT' or model_name == 'ORT':
Importance = model.variable_importance()
else:
if model_name == 'Log':
features = model.coef_[0]
elif model_name == 'Lin':
features = model.coef_
else:
features = model.feature_importances_
Importance = pd.DataFrame(features, columns=['Importance'])
Importance['Feature'] = list(cols)
Importance['Model'] = model_name
Importance['seed'] = seed
Importance['target'] = target
Importances[target][model_name][seed] = Importance
print("End of one thread!")
stop = time.time()
print("took %.2f seconds" % ((stop - start)))
if save_probabilities:
return AUC_test, Importance, prob
else:
return AUC_test
for dilution in np.unique(toc[toc.isRef == 'N'].Dilution):
logging.debug("Processing dilution: %0.1f" % dilution)
h5FileName = "Case%s.hdf5" % str(dilution).replace(".", "_", 1)
try:
with h5py.File(h5FileName, 'r') as f:
pass
except IOError as e:
time_all = []
caseFileList = ["../2015-10-15_Plot_time_vs_region_length_rvd3_synthetic_data/depth_chart/100/top_400_positions/%s"\
% filename for filename in toc.Filename[toc.Dilution==dilution] ]
logging.info('Estimate %s' % caseFileList)
t0 = time.time()
(r, n, loc, refb) = rvd3.load_depth(caseFileList)
t1 = time.time()
time_load_depth = t1 - t0
time_all.extend([time_load_depth])
casephi, caseq, time_ini_model_para, time_ini_var_para, time_ini_ELBO, time_opt_gam, time_opt_delta, time_conv, \
time_opt_mu0, time_opt_M0, time_opt_M, time_update_ELBO = rvd3.ELBO_opt(r, n, seed = 19860522, pool=None)
time_all.extend([time_ini_model_para, time_ini_var_para, time_ini_ELBO, time_opt_gam, time_opt_delta, time_conv, \
time_opt_mu0, time_opt_M0, time_opt_M, time_update_ELBO])
logging.debug("Saving model in %s" % h5FileName)
t2 = time.time()
rvd3.save_model(h5FileName, r, n, casephi, caseq, loc, refb)
t3 = time.time()
def create(self,**kwargs):
'''PARAMS:
-------
overwrite = [True | False]
backup = [True | False]
If database exists, with overwrite FALSE and backup FALSE, method exists with False
If database does not exist it is created
If database exists and overwrite without backup, it is deleted
If database exists and backup is set, database will be backed up; regardless of overwrite.
RETURN
-----
FLAG = True if a database is created, False if a database is not created.
MESSAGE = database name + actions taken
'''
method = sys._getframe().f_code.co_name
message = self.database + " "
flag = True
db_exists = False
now = time.time()
bak_name = ""
modifier = time.strftime("%Y-%m-%d-%s", time.gmtime(now))
overwrite = kwargs.get('overwrite',False)
backup = kwargs.get('backup',True)
if DEBUG: print('Class {classname} Method {method} Arguments {args}'.format(classname=self.class_name, method=method, args=kwargs))
db_exists = self.name()[0]
if DEBUG and db_exists: print('Database {database} exists'.format(database=self.database))
if backup:
if DEBUG: print("Database backed up to filename is {}".format(bak_name))
dir_name = os.path.dirname(self.database)
base_name = os.path.basename(self.database)
file_name, file_ext = base_name.rsplit('.', 1)
bak_name = dir_name + "/" + file_name + "_" + modifier + "." + file_ext
try:
copyfile(self.database, bak_name)
message += "- File was backed up"
except IOError as e:
message += "- Error file was not backed up {error}".format(error=e)
return False, message
if overwrite and db_exists:
try:
os.remove(self.database)
message += "- database file deleted"
except Error as e:
message += "- Error file was not deleted {error}".format(error=e)
return False, message
self.connect()
wal = "PRAGMA journal_mode=WAL;"
cur = self.db_connect.cursor()
cur.execute(wal)
results = cur.fetchall()
message += "- " + str(sqlite3.version) + ":" + str(results)
if db_exists and not overwrite:
message += "- Wal mode is enabled"
else:
message += "- New Database created with WAL mode enabled"
self.db_connect.close()
return flag, message
PORT = 33843 # webam
TIME_PORT = 37133
CAPTURE_FILE_NAME = "capture.jpg"
URL_PREFIX = "data:image/jpg;base64,"
q = queue.Queue()
cap = cv2.VideoCapture(0)
if cap.isOpened() is False:
raise ("Camera IO Error")
#time synchronization
soc = socket(AF_INET)
soc.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
print("[*] connecting to %s:%s" % (HOST, TIME_PORT))
soc.connect((HOST, TIME_PORT))
t1 = time.time()
#print t1
soc.send(str(t1))
t2 = soc.recv(1024)
t2 = float(t2)
#print t2
t3 = time.time()
#print t3
dt = t2 - t1 / 2 - t3 / 2
#print dt
soc.close()
def compare_capture(cap1, cap2):
cap1_hist = cv2.calcHist([cap1], [0], None, [256], [0, 256])
cap2_hist = cv2.calcHist([cap2], [0], None, [256], [0, 256])
def timestamp():
return int(time.time())
def keyGo(self, result=None):
if not self.timerentry_service_ref.isRecordable():
self.session.openWithCallback(
self.selectChannelSelector, MessageBox,
_("You didn't select a channel to record from."),
MessageBox.TYPE_ERROR)
return
self.timer.name = self.timerentry_name.value
self.timer.description = self.timerentry_description.value
self.timer.justplay = self.timerentry_justplay.value == "zap"
self.timer.always_zap = self.timerentry_justplay.value == "zap+record"
self.timer.rename_repeat = self.timerentry_renamerepeat.value
if self.timerentry_justplay.value == "zap":
if not self.timerentry_showendtime.value:
self.timerentry_endtime.value = self.timerentry_starttime.value
self.timer.resetRepeated()
self.timer.afterEvent = {
"nothing": AFTEREVENT.NONE,
"deepstandby": AFTEREVENT.DEEPSTANDBY,
"standby": AFTEREVENT.STANDBY,
"auto": AFTEREVENT.AUTO
}[self.timerentry_afterevent.value]
self.timer.descramble = {
"normal": True,
"descrambled+ecm": True,
"scrambled+ecm": False,
}[self.timerentry_recordingtype.value]
self.timer.record_ecm = {
"normal": False,
"descrambled+ecm": True,
"scrambled+ecm": True,
}[self.timerentry_recordingtype.value]
self.timer.service_ref = self.timerentry_service_ref
self.timer.tags = self.timerentry_tags
if self.timer.dirname or self.timerentry_dirname.value != defaultMoviePath(
):
self.timer.dirname = self.timerentry_dirname.value
config.movielist.last_timer_videodir.value = self.timer.dirname
config.movielist.last_timer_videodir.save()
if self.timerentry_type.value == "once":
self.timer.begin, self.timer.end = self.getBeginEnd()
if self.timerentry_type.value == "repeated":
if self.timerentry_repeated.value == "daily":
for x in (0, 1, 2, 3, 4, 5, 6):
self.timer.setRepeated(x)
if self.timerentry_repeated.value == "weekly":
self.timer.setRepeated(self.timerentry_weekday.index)
if self.timerentry_repeated.value == "weekdays":
for x in (0, 1, 2, 3, 4):
self.timer.setRepeated(x)
if self.timerentry_repeated.value == "user":
for x in (0, 1, 2, 3, 4, 5, 6):
if self.timerentry_day[x].value:
self.timer.setRepeated(x)
self.timer.repeatedbegindate = self.getTimestamp(
self.timerentry_repeatedbegindate.value,
self.timerentry_starttime.value)
if self.timer.repeated:
self.timer.begin = self.getTimestamp(
self.timerentry_repeatedbegindate.value,
self.timerentry_starttime.value)
self.timer.end = self.getTimestamp(
self.timerentry_repeatedbegindate.value,
self.timerentry_endtime.value)
else:
self.timer.begin = self.getTimestamp(
time.time(), self.timerentry_starttime.value)
self.timer.end = self.getTimestamp(
time.time(), self.timerentry_endtime.value)
# when a timer end is set before the start, add 1 day
if self.timer.end < self.timer.begin:
self.timer.end += 86400
if self.timer.eit is not None:
event = eEPGCache.getInstance().lookupEventId(
self.timer.service_ref.ref, self.timer.eit)
if event:
n = event.getNumOfLinkageServices()
if n > 1:
tlist = []
ref = self.session.nav.getCurrentlyPlayingServiceOrGroup()
parent = self.timer.service_ref.ref
selection = 0
for x in range(n):
i = event.getLinkageService(parent, x)
if i.toString() == ref.toString():
selection = x
tlist.append((i.getName(), i))
self.session.openWithCallback(
self.subserviceSelected,
ChoiceBox,
title=_("Please select a subservice to record..."),
list=tlist,
selection=selection)
return
elif n > 0:
parent = self.timer.service_ref.ref
self.timer.service_ref = ServiceReference(
event.getLinkageService(parent, 0))
self.saveTimer()
self.close((True, self.timer))
def __init__(self, config=None):
self.current_time_epoch = int(time.time())
def oauth_grant():
'''
This endpoint takes the following parameters:
* code - The code parameter provided in the redirect
* client_id - Your client ID
* client_secret - your client secret
'''
application = g.db.query(OauthApp).filter_by(
client_id=request.values.get("client_id"),
client_secret=request.values.get("client_secret")).first()
if not application:
return jsonify(
{"oauth_error": "Invalid `client_id` or `client_secret`"}), 401
if application.is_banned:
return jsonify({"oauth_error": f"Application `{application.app_name}` is suspended."}), 403
if request.values.get("grant_type") == "code":
code = request.values.get("code")
if not code:
return jsonify({"oauth_error": "code required"}), 400
auth = g.db.query(ClientAuth).filter_by(
oauth_code=code,
access_token=None,
oauth_client=application.id
).first()
if not auth:
return jsonify({"oauth_error": "Invalid code"}), 401
auth.oauth_code = None
auth.access_token = secrets.token_urlsafe(128)[0:128]
auth.access_token_expire_utc = int(time.time()) + 60 * 60
g.db.add(auth)
g.db.commit()
data = {
"access_token": auth.access_token,
"scopes": auth.scopelist,
"expires_at": auth.access_token_expire_utc,
"token_type": "Bearer"
}
if auth.refresh_token:
data["refresh_token"] = auth.refresh_token
return jsonify(data)
elif request.values.get("grant_type") == "refresh":
refresh_token = request.values.get('refresh_token')
if not refresh_token:
return jsonify({"oauth_error": "refresh_token required"}), 401
auth = g.db.query(ClientAuth).filter_by(
refresh_token=refresh_token,
oauth_code=None,
oauth_client=application.id
).first()
if not auth:
return jsonify({"oauth_error": "Invalid refresh_token"}), 401
auth.access_token = secrets.token_urlsafe(128)[0:128]
auth.access_token_expire_utc = int(time.time()) + 60 * 60
g.db.add(auth)
data = {
"access_token": auth.access_token,
"scopes": auth.scopelist,
"expires_at": auth.access_token_expire_utc
}
return jsonify(data)
else:
return jsonify({"oauth_error": f"Invalid grant_type `{request.values.get('grant_type','')}`. Expected `code` or `refresh`."}), 400
def __init__(self, user_id: int):
self.user_id = user_id
self.id = uuid4().hex
self.expire_at = int(time.time()) + CONFIRMATION_EXPIRATION_DELTA
self.confirmed = False
def getConfigListValues(self):
if not self.timerentry_service_ref.isRecordable():
self.session.openWithCallback(
self.selectChannelSelector, MessageBox,
_("You didn't select a channel to record from."),
MessageBox.TYPE_ERROR)
return
if self.timerentry_justplay.value == 'record':
if not harddiskmanager.inside_mountpoint(
self.timerentry_dirname.value):
if harddiskmanager.HDDCount(
) and not harddiskmanager.HDDEnabledCount():
self.session.open(MessageBox, _("Unconfigured storage devices found!") + "\n" \
+ _("Please make sure to set up your storage devices with the improved storage management in menu -> setup -> system -> storage devices."), MessageBox.TYPE_ERROR)
return
elif harddiskmanager.HDDEnabledCount(
) and defaultStorageDevice() == "<undefined>":
self.session.open(MessageBox, _("No default storage device found!") + "\n" \
+ _("Please make sure to set up your default storage device in menu -> setup -> system -> recording paths."), MessageBox.TYPE_ERROR)
return
elif harddiskmanager.HDDEnabledCount(
) and defaultStorageDevice() != "<undefined>":
part = harddiskmanager.getDefaultStorageDevicebyUUID(
defaultStorageDevice())
if part is None:
self.session.open(MessageBox, _("Default storage device is not available!") + "\n" \
+ _("Please verify if your default storage device is attached or set up your default storage device in menu -> setup -> system -> recording paths."), MessageBox.TYPE_ERROR)
return
else:
self.session.open(
MessageBox,
_("Recording destination for this timer does not exists."
), MessageBox.TYPE_ERROR)
return
self.timer.name = self.timerentry_name.value
self.timer.description = self.timerentry_description.value
self.timer.justplay = self.timerentry_justplay.value == "zap"
if self.timerentry_justplay.value == "zap":
if not self.timerentry_showendtime.value:
self.timerentry_endtime.value = self.timerentry_starttime.value
self.timer.resetRepeated()
self.timer.afterEvent = {
"nothing": AFTEREVENT.NONE,
"deepstandby": AFTEREVENT.DEEPSTANDBY,
"standby": AFTEREVENT.STANDBY,
"auto": AFTEREVENT.AUTO
}[self.timerentry_afterevent.value]
self.timer.service_ref = self.timerentry_service_ref
self.timer.tags = self.timerentry_tags
if self.timer.dirname or self.timerentry_dirname.value != defaultMoviePath(
):
self.timer.dirname = self.timerentry_dirname.value
config.movielist.last_timer_videodir.value = self.timer.dirname
config.movielist.last_timer_videodir.save()
if self.timerentry_type.value == "once":
self.timer.begin, self.timer.end = self.getBeginEnd()
if self.timerentry_type.value == "repeated":
if self.timerentry_repeated.value == "daily":
for x in (0, 1, 2, 3, 4, 5, 6):
self.timer.setRepeated(x)
if self.timerentry_repeated.value == "weekly":
self.timer.setRepeated(self.timerentry_weekday.index)
if self.timerentry_repeated.value == "weekdays":
for x in (0, 1, 2, 3, 4):
self.timer.setRepeated(x)
if self.timerentry_repeated.value == "user":
for x in (0, 1, 2, 3, 4, 5, 6):
if self.timerentry_day[x].value:
self.timer.setRepeated(x)
self.timer.repeatedbegindate = self.getTimestamp(
self.timerentry_repeatedbegindate.value,
self.timerentry_starttime.value)
if self.timer.repeated:
self.timer.begin = self.getTimestamp(
self.timerentry_repeatedbegindate.value,
self.timerentry_starttime.value)
self.timer.end = self.getTimestamp(
self.timerentry_repeatedbegindate.value,
self.timerentry_endtime.value)
else:
self.timer.begin = self.getTimestamp(
time.time(), self.timerentry_starttime.value)
self.timer.end = self.getTimestamp(
time.time(), self.timerentry_endtime.value)
# when a timer end is set before the start, add 1 day
if self.timer.end < self.timer.begin:
self.timer.end += 86400
