def run(self, console, frames=None, times=4):
if times > 1:
print_result('Running %s' % self.__class__.__name__)
while times > 0:
self.run(console, frames, times=1)
times -= 1
print_result('')
return
if frames is None:
frames = self.default_frames
self.total_frames = 0
self.tiles = 0
console.clear()
self.start_time = time.clock()
while self.total_frames < frames:
self.total_frames += 1
self.test(console)
for event in tdl.event.get():
if event.type == 'QUIT':
raise SystemExit('Benchmark Canceled')
self.total_time = time.clock() - self.start_time
self.tiles_per_second = self.tiles / self.total_time
print_result(
'%i tiles drawn in %.2f seconds, %.2f characters/ms, %.2f FPS' %
(self.tiles, self.total_time,self.tiles_per_second / 1000,
self.total_frames / self.total_time))
def recolorize(self):
self.after_id = None
if not self.delegate:
if DEBUG: print("no delegate")
return
if not self.allow_colorizing:
if DEBUG: print("auto colorizing is off")
return
if self.colorizing:
if DEBUG: print("already colorizing")
return
try:
self.stop_colorizing = False
self.colorizing = True
if DEBUG: print("colorizing...")
t0 = time.clock()
self.recolorize_main()
t1 = time.clock()
if DEBUG: print("%.3f seconds" % (t1-t0))
finally:
self.colorizing = False
if self.allow_colorizing and self.tag_nextrange("TODO", "1.0"):
if DEBUG: print("reschedule colorizing")
self.after_id = self.after(1, self.recolorize)
if self.close_when_done:
top = self.close_when_done
self.close_when_done = None
top.destroy()
def __init__(self,fname):
#dump to binary
fndata=fname;
if (not os.path.isfile(fndata)):
fndatain=fndata.replace('/bin/','/');
datain=Epix100a(fndatain);
#write header
binheader=np.zeros(16).astype(np.uint32);
binheader[0:6]=[datain.nframes, datain.my*datain.mx, datain.my, datain.mx, datain.nblocks, datain.nbcols];
binheader.tofile(fndata);
#write data
dataout=np.memmap(fndata,dtype=np.int16,mode='r+', shape=(datain.nframes,datain.my,datain.mx),offset=64);
t0=time.clock();
for iframe in range(datain.nframes):
dataout[iframe]=datain.frames(iframe);
if (iframe%100==0):
#progress(iframe,nframes,iframe);
print str(iframe)+' - '+str(1000*(time.clock()-t0)/(iframe+1))+' ms. average frame: '+str(np.mean(datain.frames(iframe)));
dataout.flush();
del dataout;
del datain;
#get nr of frames
data=np.memmap(fndata,dtype=np.uint32,mode='r',shape=((64)),offset=0);
self.nframes=data[0]; self.nframesize=data[1]; self.my=data[2]; self.mx=data[3]; self.nblocks=data[4]; self.nbcols=data[5];
self.data=np.memmap(fndata,dtype=np.int16,mode='c',shape=(self.nframes,self.my,self.mx),offset=64);
def RunTest(testNum, p0, p1, p2, hasAnswer, p3):
obj = JanuszInTheCasino()
startTime = time.clock()
answer = obj.findProbability(p0, p1, p2)
endTime = time.clock()
testTime.append(endTime - startTime)
res = True
if hasAnswer:
res = answer == p3
if res:
print(str("Test #") + str(testNum) + ": Passed")
return res
print(str("Test #") + str(testNum) + str(":"))
print(("[") + str(p0) + str(",") + str(p1) + str(",") + str(p2) + str("]"))
if (hasAnswer):
print(str("Expected:"))
print(str(p3))
print(str("Received:"))
print(str(answer))
print(str("Verdict:"))
if (not res):
print(("Wrong answer!!"))
elif ((endTime - startTime) >= 20):
print(str("FAIL the timeout"))
res = False
elif (hasAnswer):
print(str("OK!!"))
else:
print(str("OK, but is it right?"))
print("Time: %.11f seconds" % (endTime - startTime))
print(str("-----------------------------------------------------------"))
return res
def __init__(self, fndark, nblocksize):
if (os.path.isfile(fndark+'-dark.npz')):
npzfile=np.load(fndark+'-dark.npz');
self.dmean=npzfile['dmean'];
self.dstd=npzfile['dstd'];
self.dbpm=npzfile['dbpm'];
else:
dark=Binary(fndark);
nframes=dark.nframes; my=dark.my; mx=dark.mx;
nblocks=nframes//nblocksize;
bmed=np.zeros((nblocks,my,mx));
bstd=np.zeros((nblocks,my,mx));
for iblock in range(nblocks):
t0=time.clock();
a=dark.data[iblock*nblocksize:(iblock+1)*nblocksize];
a,idx=dropbadframes(a);
print '- read block, dropped bad, subtracted dark in '+str(time.clock()-t0)+'s';
nfb=a.shape[0];
bmed[iblock,:,:]=np.median(a,axis=0);
bstd[iblock,:,:]=np.std(a,axis=0);
self.dmean=np.mean(bmed,axis=0);
self.dstd=np.sqrt(np.sum((bstd)**2,axis=0));
self.dbpm=self.dstd<(np.median(self.dstd)+5*np.std(self.dstd));
self.dbpm=self.dstd<(np.median(self.dstd*self.dbpm)+5*np.std(self.dstd*self.dbpm));
np.savez(fndark+'-dark',dmean=self.dmean,dstd=self.dstd,dbpm=self.dbpm);
del dark;
def main():
# n = raw_input("What's the N?")
n = sys.argv[1]
start = time.clock()
tryAll(n)
print "It took", (time.clock()-start),"seconds to complete"
print "now:", time.clock(), "start:", start
def kontrola(self, datadir=None):
"""
Jednoduché vyhodnocení výsledků
"""
obrazky, reseni = self.readImageDir(datadir)
vysledky = []
for i in range(0, len(obrazky)):
cas1 = time.clock()
im = skimage.io.imread(obrazky[i])
result = self.rozpoznejZnacku(im)
cas2 = time.clock()
if((cas2 - cas1) >= 1.0):
print "cas vyprsel"
result = 0
vysledky.append(result)
hodnoceni = np.array(reseni) == np.array(vysledky)
skore = np.sum(hodnoceni.astype(np.int)) / np.float(len(reseni))
print skore
def wait_for_srq(self, timeout=25):
"""Wait for a serial request (SRQ) coming from the instrument.
Note that this method is not ended when *another* instrument signals an
SRQ, only *this* instrument.
:param timeout: the maximum waiting time in seconds.
Defaul: 25 (seconds).
None means waiting forever if necessary.
"""
vpp43.enable_event(self.vi, VI_EVENT_SERVICE_REQ, VI_QUEUE)
if timeout and not(0 <= timeout <= 4294967):
raise ValueError("timeout value is invalid")
starting_time = time.clock()
while True:
if timeout is None:
adjusted_timeout = VI_TMO_INFINITE
else:
adjusted_timeout = int((starting_time + timeout - time.clock())
* 1000)
if adjusted_timeout < 0:
adjusted_timeout = 0
event_type, context = \
vpp43.wait_on_event(self.vi, VI_EVENT_SERVICE_REQ,
adjusted_timeout)
vpp43.close(context)
if self.stb & 0x40:
break
vpp43.discard_events(self.vi, VI_EVENT_SERVICE_REQ, VI_QUEUE)
def timing(self):
t0 = time.clock()
try:
yield
finally:
te = time.clock()
self.laps.append(te - t0)
def BENCH():
print('\n\n\n--->BEGIN BENCHMARK')
bt0 = time.clock()
###---MAKE A BIG LIST
tsize = 25
tlist = []
for x in range(tsize):
for y in range(tsize):
for z in range(tsize):
tlist.append((x,y,z))
tlist.append((x,y,z))
###---FUNCTION TO TEST
bt1 = time.clock()
#ll = deDupe(tlist)
#ll = f5(tlist)
print('LENS - ', len(tlist), len(ll) )
bt2 = time.clock()
btRUNb = bt2 - bt1
btRUNa = bt1 - bt0
print('--->SETUP TIME : ', btRUNa)
print('--->BENCHMARK TIME: ', btRUNb)
print('--->GRIDSIZE: ', tsize, ' - ', tsize*tsize*tsize)
def draw(self): self.screen.clear_pixel() start = time.clock() * 2.77 + math.sin(time.clock()) end = time.clock() * 7.35 + 0.5 * math.pi distance = end - start start = start % (2 * math.pi) end = end % (2 * math.pi) invert = distance % (4 * math.pi) > 2 * math.pi if invert: buf = end end = start start = buf hue = (time.clock() * 0.01) % 1 color = hsv_to_color(hue, 1, 1) for x in range(16): for y in range(16): r = ((x - 8)**2 + (y - 8)**2)**0.5 if r == 0: r = 0.001 angle = math.acos((x - 8) / r) if y - 8 < 0: angle = 2 * math.pi - angle if (angle > start and angle < end) or (end < start and (angle > start or angle < end)): self.screen.pixel[x][y] = color self.screen.update()
def saveBrain(self, filename):
"""Dump the contents of the bot's brain to a file on disk."""
if self._verboseMode: print("Saving brain to %s..." % filename, end=' ')
start = time.clock()
self._brain.save(filename)
if self._verboseMode:
print("done (%.2f seconds)" % (time.clock() - start))
def learn(self, filename):
"""Load and learn the contents of the specified AIML file.
If filename includes wildcard characters, all matching files
will be loaded and learned.
"""
for f in glob.glob(filename):
if self._verboseMode: print("Loading %s..." % f, end=' ')
start = time.clock()
# Load and parse the AIML file.
parser = AimlParser.create_parser()
handler = parser.getContentHandler()
handler.setEncoding(self._textEncoding)
try:
parser.parse(f)
except xml.sax.SAXParseException as msg:
err = "\nFATAL PARSE ERROR in file %s:\n%s\n" % (f, msg)
sys.stderr.write(err)
continue
# store the pattern/template pairs in the PatternMgr.
for key, tem in list(handler.categories.items()):
self._brain.add(key, tem)
# Parsing was successful.
if self._verboseMode:
print("done (%.2f seconds)" % (time.clock() - start))
def get_content(self, path, method='POST', body=None, params=''):
uri= self.get_base_api_url() % (path, params)
startTime = time.clock()
response, content = http.request(
uri,
method=method, body=json.dumps(body) if body else None,
headers={'Content-Type': 'application/json; charset=UTF-8'})
contentLen = len(content)
try:
content = json.loads(content)
except ValueError:
logging.error('while requesting {}'.format(uri))
logging.error('non-json api content %s' % content[:1000])
raise ApiException('The API returned invalid JSON')
if response.status >= 300:
logging.error('error api response %s' % response)
logging.error('error api content %s' % content)
if 'error' in content:
raise ApiException(content['error']['message'])
else:
raise ApiException('Something went wrong with the API call!')
logging.info('get_content {}: {}kb {}s'.format(uri, contentLen/1024, time.clock() - startTime))
return content
def req():
# Get URLs from a text file, remove white space.
db = MySQLDatabase(DATABASE_HOST, DATABASE_USER, DATABASE_PASSWORD, DATABASE_NAME)
db_worker_view = db.get_work_view()
articles = db_worker_view.retrieve_all_articles()
#articles = db_worker_view.retrieve_all_articles_questionmark()
# measure time
start = time.clock()
start_time_iteration = start
iteration_number = 483
for i, article in enumerate(articles):
# print some progress
if i % 10000 == 0:
#print time for the iteration
seconds = time.clock() - start_time_iteration
m, s = divmod(seconds, 60)
h, m = divmod(m, 60)
print "Number of crawled articles: %d. Total time for last iteration of 10000 articles: %d:%02d:%02d" % (i, h, m, s)
start_time_iteration = time.clock()
iteration_number += 1
# Thread pool.
# Blocks other threads (more than the set limit).
pool.acquire(blocking=True)
# Create a new thread.
# Pass each URL (i.e. u parameter) to the worker function.
t = threading.Thread(target=worker, args=(MEDIAWIKI_API_ENDPOINT+urllib.quote(article['title'])+'/'+str(article['rev_id']), article, iteration_number))
# Start the newly create thread.
t.start()
seconds = time.clock() - start
m, s = divmod(seconds, 60)
h, m = divmod(m, 60)
print "Total time: %d:%02d:%02d" % (h, m, s)
def apply(self, expr, evaluation):
'Timing[expr_]'
start = time.clock()
result = expr.evaluate(evaluation)
stop = time.clock()
return Expression('List', Real(stop - start), result)
def run(test):
global test_name
test_name = test
t0 = time.clock()
msg("setting up supervisor")
exe = test + '.exe'
proc = Popen(exe, bufsize=1<<20 , stdin=PIPE, stdout=PIPE, stderr=PIPE)
done = multiprocessing.Value(ctypes.c_bool)
queue = multiprocessing.Queue(maxsize=5)#(maxsize=1024)
workers = []
for n in range(NUM_WORKERS):
worker = multiprocessing.Process(name='Worker-' + str(n + 1),
target=init_worker,
args=[test, MAILBOX, queue, done])
workers.append(worker)
child_processes.append(worker)
for worker in workers:
worker.start()
msg("running test")
interact(proc, queue)
with done.get_lock():
done.value = True
for worker in workers:
worker.join()
msg("python is done")
assert queue.empty(), "did not validate everything"
dt = time.clock() - t0
msg("took", round(dt, 3), "seconds")
def main():
u"メインメソッド"
starttime = time.clock()
run()
time1 = time.clock()
print "time: %f" % (time1 - starttime)
def fit(self, X, y, valid_X=None, valid_y=None):
input_size = X.shape[1]
output_size = len(np.unique(y))
X_sym = T.matrix('x')
y_sym = T.ivector('y')
self.layers_ = []
self.layer_sizes_ = [input_size]
self.layer_sizes_.extend(self.hidden_layer_sizes)
self.layer_sizes_.append(output_size)
self.dropout_layers_ = []
self.training_scores_ = []
self.validation_scores_ = []
self.training_loss_ = []
self.validation_loss_ = []
if not hasattr(self, 'fit_function'):
self._setup_functions(X_sym, y_sym,
self.layer_sizes_)
batch_indices = list(range(0, X.shape[0], self.batch_size))
if X.shape[0] != batch_indices[-1]:
batch_indices.append(X.shape[0])
start_time = time.clock()
itr = 0
best_validation_score = np.inf
while (itr < self.max_iter):
print("Starting pass %d through the dataset" % itr)
itr += 1
batch_bounds = list(zip(batch_indices[:-1], batch_indices[1:]))
# Random minibatches
self.random_state.shuffle(batch_bounds)
for start, end in batch_bounds:
self.partial_fit(X[start:end], y[start:end])
current_training_score = (self.predict(X) != y).mean()
self.training_scores_.append(current_training_score)
current_training_loss = self.loss_function(X, y)
self.training_loss_.append(current_training_loss)
# Serialize each save_frequency iteration
if (itr % self.save_frequency) == 0 or (itr == self.max_iter):
f = open(self.model_save_name + "_snapshot.pkl", 'wb')
cPickle.dump(self, f, protocol=2)
f.close()
if valid_X is not None:
current_validation_score = (
self.predict(valid_X) != valid_y).mean()
self.validation_scores_.append(current_validation_score)
current_training_loss = self.loss_function(valid_X, valid_y)
self.validation_loss_.append(current_training_loss)
print("Validation score %f" % current_validation_score)
# if we got the best validation score until now, save
if current_validation_score < best_validation_score:
best_validation_score = current_validation_score
f = open(self.model_save_name + "_best.pkl", 'wb')
cPickle.dump(self, f, protocol=2)
f.close()
end_time = time.clock()
print("Total training time ran for %.2fm" %
((end_time - start_time) / 60.))
return self
def Check(): start =I end=perm_apply(F, start) shortest_path(start, end) #path length 1 start=I middle1 =perm_apply(F, start) middle2 =perm_apply(L, middle1) middle3 =perm_apply(F, middle2) end =perm_apply(L, middle3) shortest_path(start, end) #path length 4 start =I middle1=perm_apply(F, start) middle2=perm_apply(F, middle1) end =perm_apply(Li, middle2) shortest_path(start,end)#path length 3 from time import clock m=clock() start = (6, 7, 8, 20, 18, 19, 3, 4, 5, 16, 17, 15, 0, 1, 2, 14, 12, 13, 10, 11, 9, 21, 22, 23) end =I shortest_path(start,end) #path length 14 print 'time taken:',clock()-m start=I middle1 =perm_apply(F, start) middle2 =perm_apply(Li, middle1) middle4=perm_apply(Fi, middle2) middle3=perm_apply(U, middle4) end =perm_apply(U, middle3) shortest_path(start, end) #path length 5 start=I middle1=perm_apply(F, start) middle2=perm_apply(F, middle1) shortest_path(start,middle2) #path length 2 #Check() #input_configuration()
def testTime(self):
nUIDs = 100000
startTime = time.clock()
for i in range(0,nUIDs):
makeUUID()
print("We can make %i UUIDs in %f seconds" %(nUIDs, time.clock() - startTime))
def testRun():
# For testing
# count = [500, 1000, 1500, 2000, 2500, 3000, 3500, 4000, 4500, 5000, 5500, 6000, 6500, 7000, 7500, 8000, 8500, 9000, 9500, 10000, 15000, 20000, 25000, 30000, 35000, 40000, 45000, 50000, 55000, 60000, 65000, 70000, 75000, 80000, 85000, 90000, 95000, 100000]
count = 5000
while count < 5000000000:
# for num in testList:
randList = [None] * count
for n in range(0, count):
randNum = random.randrange(0, 101)
if (random.randrange(0, 2) == 0):
randNum = randNum * -1
randList[n] = randNum
startTime = time.clock()
result = linearSearch(randList)
stopTime = time.clock()
resultTime = stopTime - startTime
print("n: " + str(len(randList)))
print("Largest Result: " + str(result[2]))
print("Running Time: " + str(resultTime))
count = count * 10
def run(self):
global q
while(time.clock() < 10):
if(self.nextTime < time.clock() and not q.empty()):
f = q.get()
print("Removing " + f)
self.nextTime += random.random()*2
def SaveData (self, fname, verbose = True):
if (verbose):
print (" Saving measurement to %s ... " % fname)
start = time.clock()
f = h5py.File(fname, 'w')
f['data_r'] = np.squeeze(np.real(self.data).transpose())
f['data_i'] = np.squeeze(np.imag(self.data).transpose())
if (self.noise != 0):
f['noise_r'] = np.squeeze(np.real(self.noise).transpose())
f['noise_i'] = np.squeeze(np.imag(self.noise).transpose())
if (self.acs != 0):
f['acs_r'] = np.squeeze(np.real(self.acs).transpose())
f['acs_i'] = np.squeeze(np.imag(self.acs).transpose())
if (self.sync.any() != 0):
f['sync'] = self.sync.transpose()
f.close()
if (verbose):
print ' ... saved in %(time).1f s.\n' % {"time": time.clock()-start}
return
def demo(print_times=True, print_grammar=True,
print_trees=True, print_sentence=True,
trace=1,
parser=FeatureChartParser,
sent='I saw John with a dog with my cookie'):
import sys, time
print()
grammar = demo_grammar()
if print_grammar:
print(grammar)
print()
print("*", parser.__name__)
if print_sentence:
print("Sentence:", sent)
tokens = sent.split()
t = time.clock()
cp = parser(grammar, trace=trace)
chart = cp.chart_parse(tokens)
trees = list(chart.parses(grammar.start()))
if print_times:
print("Time: %s" % (time.clock() - t))
if print_trees:
for tree in trees: print(tree)
else:
print("Nr trees:", len(trees))
def do_effect(self, can_msg, args): # read full packet from serial port
if args.get('action') == 'read':
can_msg = self.do_read(can_msg)
elif args.get('action') == 'write':
# KOSTYL: workaround for BMW e90 bus
if self._restart and self._run and (time.clock() - self.last) >= self.act_time:
self.dev_write(0, "O")
self.last = time.clock()
self.do_write(can_msg)
else:
self.dprint(1, 'Command ' + args['action'] + ' not implemented 8(')
"""
if self._restart:
if self.wait_for and can_msg.debugData and can_msg.debugText['text'][0] == "F" and len(can_msg.debugText['text']) > 2:
error = int(can_msg.debugText['text'][1:3],16)
self.dprint(1,"BUS ERROR:" + hex(error))
if error & 8: # Fix for BMW CAN where it could be overloaded
self.dev_write(0, "C")
time.sleep(0.4)
self.dev_write(0, "O")
can_msg.debugText['do_not_send'] = True
else:
can_msg.debugText['please_send'] = True
self.wait_for = False
elif time.clock() - self.last >= self.act_time and not self.wait_for:
self.dev_write(0, "F")
self.wait_for = True
self.last = time.clock()
"""
return can_msg
def printOutput(config, outputDirName, varDF):
'''Output run statistics and variant details to the specified output directory.'''
startTime = time.clock()
print("\n=== Writing output files to {0}/ ===".format(outputDirName))
ow = output.Writer()
# identify formats ending in 'xlsx' as the "excel formats," requiring XlsxWriter
excel_formats = [ plugin_name for plugin_name,file_name in ow.file_names.items() if os.path.splitext(file_name)[1]==".xlsx" ]
if "all" in config.outputFormats:
# replace output type 'all' with a lsit of all supported output types
# and remove 'all' and 'default' to prevent recursive execution of the modules
config.outputFormats = ow.supported_formats.keys()
config.outputFormats.remove('all')
config.outputFormats.remove('default')
if 'xlsxwriter' not in sys.modules and excel_formats:
# if xlsxwriter is not present and the user selected excel output formats, remove excel formats from output formats
config.outputFormats = [ x for x in config.outputFormats if x not in excel_formats ]
throwWarning("xlsxwriter module not found; Excel outputs disabled")
for format in config.outputFormats:
ow.write(varDF,format,outputDirName,config)
totalTime = time.clock() - startTime
print("\tTime to write: {0:02d}:{1:02d}".format(int(totalTime/60), int(totalTime % 60)))
return 0
def executeOneSetting(tensor, density, roundId, para):
logger.info('density=%.2f, %2d-round starts.'%(density, roundId + 1))
(numUser, numService, numTime) = tensor.shape
# remove the entries of data to generate trainTensor and testTensor
(trainTensor, testTensor) = evallib.removeTensor(tensor, density, roundId, para)
# invocation to the prediction function
startTime = time.clock() # to record the running time for one round
predictedTensor = Average.predict(trainTensor, para)
runningTime = float(time.clock() - startTime) / numTime
# evaluate the prediction error
for sliceId in xrange(numTime):
testMatrix = testTensor[:, :, sliceId]
predictedMatrix = predictedTensor[:, :, sliceId]
(testVecX, testVecY) = np.where(testMatrix)
testVec = testMatrix[testVecX, testVecY]
predVec = predictedMatrix[testVecX, testVecY]
evalResult = evallib.errMetric(testVec, predVec, para['metrics'])
result = (evalResult, runningTime)
# dump the result at each density
outFile = '%s%s_%s_result_%02d_%.2f_round%02d.tmp'%(para['outPath'],
para['dataName'], para['dataType'], sliceId + 1, density, roundId + 1)
evallib.dumpresult(outFile, result)
logger.info('density=%.2f, %2d-round done.'%(density, roundId + 1))
logger.info('----------------------------------------------')
def update(params):
# Descarga el ZIP
xbmc.output("[updater.py] update")
xbmc.output("[updater.py] cwd="+os.getcwd())
remotefilename = REMOTE_FILE+params.get("version")+".zip"
localfilename = LOCAL_FILE+params.get("version")+".zip"
xbmc.output("[updater.py] remotefilename=%s" % remotefilename)
xbmc.output("[updater.py] localfilename=%s" % localfilename)
xbmc.output("[updater.py] descarga fichero...")
inicio = time.clock()
urllib.urlretrieve(remotefilename,localfilename)
fin = time.clock()
xbmc.output("[updater.py] Descargado en %d segundos " % (fin-inicio+1))
# Lo descomprime
xbmc.output("[updater.py] descomprime fichero...")
import ziptools
unzipper = ziptools.ziptools()
destpathname = DESTINATION_FOLDER
xbmc.output("[updater.py] destpathname=%s" % destpathname)
unzipper.extract(localfilename,destpathname)
# Borra el zip descargado
xbmc.output("[updater.py] borra fichero...")
os.remove(localfilename)
def solveUnconstrainedOptimizationAndOutputStatus(optimizationSolver):
opt = optimizationSolver(p1)
start = time.clock()
output = opt.solve()
elapsed = time.clock() - start
outputStatus(output, elapsed)
plotRosenbrockFunctionAndOptimizationPath(output)
def ray_ds(ds, ray, name):
ds[name + ".origin"] = ray.origin.to_ds()
ds[name + ".dir"] = ray.dir.to_ds()
def random_ray(ds):
origin = Vector3(0.0, 0.0, 0.0)
direction = Vector3(random(), random(), random())
direction.normalize()
ray = Ray(origin, direction)
ray_ds(ds, ray, 'ray1')
return ray
for i in range(5):
ray = random_ray(ds)
start = time.clock()
hit = linear.isect(ray)
end = time.clock()
dur1 = end - start
start = time.clock()
runtime.run('test')
end = time.clock()
dur2 = end - start
if hit:
print(hit.t, ds['hp1.t'], ds['ret'], dur1, dur2)
else:
print("False ", ds['ret'], dur1, dur2)
if arrayLen == 2:
if array[1] < array[0]:
array[0], array[1] = array[1], array[0]
return array
lo, hi, pivot = partition(array)
lo = quickSort(lo)
hi = quickSort(hi)
sortedArray = lo
sortedArray.extend(pivot)
sortedArray.extend(hi)
return sortedArray
# 可以将输入直接转列表
originArray = eval(input('input an array: '))
start1 = time.clock()
sortedArray1 = mergeSort(originArray)
end1 = time.clock()
start2 = time.clock()
sortedArray2 = selectionSort(originArray)
end2 = time.clock()
start3 = time.clock()
sortedArray3 = selectionSort(originArray)
end3 = time.clock()
start4 = time.clock()
sortedArray4 = quickSort(originArray)
end4 = time.clock()
# print 'Doing period',period # x, y = amplitude*np.sin(2*np.pi*t/period), amplitude*np.cos(2*np.pi*t/period) # smooth # folded = t % period x, y = amplitude*np.random.randn(t.shape[0]),amplitude*np.random.randn(t.shape[0]) f = 20*np.ones(ncad) + np.sin(t/6.) # make this whatever function you like! f[400:500] *= 0.990 # toy transit '''------------------------ Define a PSF and aperture ------------------------''' width = 3. start = clock() nx, ny = 10, 10 npix = nx*ny pixels = np.zeros((nx,ny)) '''------------------------ Simulate data ------------------------''' tpf = np.zeros((nx,ny,ncad)) sensitivity = 1-0.1*np.random.rand(nx,ny) white = 0 for j in range(ncad): tpf[:,:,j] = f[j]*gaussian_psf(pixels,x[j],y[j],width)*sensitivity + np.random.randn(nx,ny)*white
# This script copies injury and fatality data from Transbase database.
#Last modified: 11/30/2017 by Jonathan Engelbert
#
### No Known Issues
################################################################################################
import arcpy
from arcpy import env
import sys, string, os, time, datetime
# SET TO OVERWRITE
arcpy.env.overwriteOutput = True
# Logging script
myStartDate = str(datetime.date.today())
myStartTime = time.clock()
theStartTime = time.ctime()
print theStartTime
try:
myStartDate = str(datetime.date.today())
myStartTime = time.clock()
theStartTime = time.ctime()
# thisfile = os.path.realpath(__file__)
file = open("C:/ETLs/TIM/TIMUpdates/Logs/" + myStartDate + "Transbase2" + ".txt", "w")
file.write(theStartTime + "\n")
when =datetime.date.today()
theDate = when.strftime("%d")
theDay=when.strftime("%A")
print theDay
if self.searcher:
return self.searcher.document(word=word)["syns"]
else:
return synonyms(self.w2n, self.n2w, word)
if __name__ == "__main__":
from time import clock
from whoosh.filedb.filestore import FileStorage
st = FileStorage("c:/testindex")
# t = clock()
# th = Thesaurus.from_filename("c:/wordnet/wn_s.pl")
# print clock() - t
#
# t = clock()
# th.to_storage(st)
# print clock() - t
#
# t = clock()
# print th.synonyms("light")
# print clock() - t
t = clock()
th = Thesaurus.from_storage(st)
print clock() - t
t = clock()
print th.synonyms("hail")
print clock() - t
def draw(self, seconds, beats, uniforms, additionalTextureUniforms=None):
if not self.shaders:
# compiler errors
return
isProfiling = Profiler.instance and Profiler.instance.isVisible(
) and Profiler.instance.isProfiling() and self._debugPassId is None
if isProfiling:
self.profileLog = []
glFinish()
startT = time.clock()
else:
startT = time.clock()
maxActiveInputs = 0
for i, passData in enumerate(self.passes):
if not self.__passDirtyState[i]:
continue
if self.passes[i].is3d:
bail = False
for buffer in self.colorBuffers[passData.targetBufferId]:
if isinstance(buffer, Texture3D):
# can't rebake
bail = True
break
if bail:
continue
self.__passDirtyState[i] = self.passes[
i].realtime # dirty again only if realtime
uniforms['uSeconds'] = seconds
uniforms['uBeats'] = beats
uniforms['uResolution'] = self.frameBuffers[passData.targetBufferId].width(), \
self.frameBuffers[passData.targetBufferId].height()
if i >= len(self.shaders) or self.shaders[i] == 0:
self._rebuild(None, index=i)
# make sure we don't take into account previous GL calls when measuring time
if isProfiling:
glFinish()
beforeT = time.clock()
self.frameBuffers[passData.targetBufferId].use()
glUseProgram(self.shaders[i])
activeInputs = self._bindInputs(i, additionalTextureUniforms)
fn = (glUniform1f, glUniform2f, glUniform3f, glUniform4f)
for name in uniforms:
if isinstance(uniforms[name], (int, long)):
glActiveTexture(GL_TEXTURE0 + activeInputs)
glBindTexture(GL_TEXTURE_2D, uniforms[name])
glUniform1i(glGetUniformLocation(self.shaders[i], name),
activeInputs)
activeInputs += 1
elif isinstance(uniforms[name], float):
fn[0](glGetUniformLocation(self.shaders[i], name),
uniforms[name])
elif len(uniforms[name]) == 9:
glUniformMatrix3fv(
glGetUniformLocation(self.shaders[i], name), 1, False,
(ctypes.c_float * 9)(*uniforms[name]))
elif len(uniforms[name]) == 16:
glUniformMatrix4fv(
glGetUniformLocation(self.shaders[i], name), 1, False,
(ctypes.c_float * 16)(*uniforms[name]))
else:
fn[len(uniforms[name]) - 1](glGetUniformLocation(
self.shaders[i], name), *uniforms[name])
for name in passData.uniforms:
if isinstance(passData.uniforms[name], float):
fn[0](glGetUniformLocation(self.shaders[i], name),
passData.uniforms[name])
else:
fn[len(passData.uniforms[name]) - 1](glGetUniformLocation(
self.shaders[i], name), *passData.uniforms[name])
maxActiveInputs = max(maxActiveInputs, activeInputs)
if self.passes[i].drawCommand is not None:
exec(self.passes[i].drawCommand)
else:
glRecti(-1, -1, 1, 1)
# duct tape the 2D color buffer(s) into 3D color buffer(s)
if self.passes[i].is3d:
buffers = self.colorBuffers[passData.targetBufferId]
for j, buffer in enumerate(buffers):
buffer.use()
data = glGetTexImage(GL_TEXTURE_2D, 0, GL_RGBA, GL_FLOAT)
FrameBuffer.clear()
buffers[j] = Texture3D(Texture.RGBA32F, buffer.height(),
True, data)
buffers[j].original = buffer
# enable mip mapping on static textures
if not self.passes[i].realtime:
# after rendering grab all render targets & enable mip maps, then generate them
for buffer in self.colorBuffers[passData.targetBufferId]:
buffer.use()
mode = GL_TEXTURE_3D if isinstance(
buffer, Texture3D) else GL_TEXTURE_2D
glTexParameteri(mode, GL_TEXTURE_MIN_FILTER,
GL_LINEAR_MIPMAP_LINEAR)
glTexParameteri(mode, GL_TEXTURE_MAG_FILTER, GL_LINEAR)
# requires openGL 4.6?
glTexParameterf(
mode, texture_filter_anisotropic.
GL_TEXTURE_MAX_ANISOTROPY_EXT, 16.0)
glGenerateMipmap(mode)
# make sure all graphics calls are finished processing in GL land before we measure time
if isProfiling:
glFinish()
afterT = time.clock()
self.profileLog.append((passData.name
or str(i), afterT - beforeT))
if self._debugPassId is not None and i == self._debugPassId[0]:
# debug mode, we want to view this pass on the screen, avoid overwriting it's buffers with future passes
break
if isProfiling:
glFinish()
# inform the profiler a new result is ready
endT = time.clock()
self.profileInfoChanged.emit(endT - startT)
return maxActiveInputs
from pprint import pprint
pprint(ToLatLon.from_cc2(pd.Series(['RU', 'SI'])))
coords = [
(46.0555, 14.5083),
(40.7127, -74.0059),
(49.761667, -77.802778),
(31.814700, 79.886838),
(52.818925, 92.567674),
(61.760153, -121.236525),
(64.295556,-15.227222),
(0, 0),
]
start = time.clock()
pprint(latlon2region(coords, 0))
elpassed = time.clock() - start
print(elpassed)
print()
start = time.clock()
pprint(latlon2region(coords, 1))
elpassed = time.clock() - start
print(elpassed)
print()
start = time.clock()
pprint(latlon2region(coords, 2))
elpassed = time.clock() - start
print(elpassed)
def stop(self):
elapsed = time.clock() - self._timings[self._name][0]
self._timings[self._name][1] += elapsed
self._timings[self._name][2] += 1
def pause(self):
self._paused = time.clock()
def timedsum1(n, zero):
start=time.clock()
tot=sum(range(n), zero)
stend=time.clock()
return type(zero), stend-start, tot
def solve(self):
"""
Runs the simulation.
"""
result_handler = self.result_handler
h = (self.final_time - self.start_time) / self.ncp
grid = N.linspace(self.start_time, self.final_time, self.ncp + 1)[:-1]
status = 0
final_time = 0.0
#For result writing
result_handler.integration_point()
#Start of simulation, start the clock
time_start = time.clock()
for t in grid:
status = self.model.do_step(t, h)
self.status = status
if status != 0:
if status == fmi.FMI_ERROR:
result_handler.simulation_end()
raise Exception(
"The simulation failed. See the log for more information. Return flag %d."
% status)
elif status == fmi.FMI_DISCARD and isinstance(
self.model, fmi.FMUModelCS1):
try:
last_time = self.model.get_real_status(
fmi.FMI1_LAST_SUCCESSFUL_TIME)
if last_time > t: #Solver succeeded in taken a step a little further than the last time
self.model.time = last_time
final_time = last_time
result_handler.integration_point()
except fmi.FMUException:
pass
break
#result_handler.simulation_end()
#raise Exception("The simulation failed. See the log for more information. Return flag %d"%status)
final_time = t + h
result_handler.integration_point()
if self.input_traj != None:
self.model.set(self.input_traj[0],
self.input_traj[1].eval(t + h)[0, :])
#End of simulation, stop the clock
time_stop = time.clock()
result_handler.simulation_end()
if self.status != 0:
print(
'Simulation terminated prematurely. See the log for possibly more information. Return flag %d.'
% status)
#Log elapsed time
print('Simulation interval : ' + str(self.start_time) + ' - ' +
str(final_time) + ' seconds.')
print('Elapsed simulation time: ' + str(time_stop - time_start) +
' seconds.')
def resume(self):
elapsed = time.clock() - self._paused
self._timings[self._name][0] += elapsed
def wrapper(*args, **kwargs):
t = time.clock()
res = func(*args, **kwargs)
print("\t%s" % func.__name__, time.clock()-t)
return res
def start(self, name):
self._name = name
if name not in self._timings:
self._timings[name] = [0.] * 3
self._timings[name][0] = time.clock()
B = mx.sym.Variable('B')
C = mx.symbol.dot(A, B)
executor = C.simple_bind(mx.gpu(1),
'write',
A=(4096, 4096),
B=(4096, 4096))
a = mx.random.uniform(-1.0, 1.0, shape=(4096, 4096))
b = mx.random.uniform(-1.0, 1.0, shape=(4096, 4096))
a.copyto(executor.arg_dict['A'])
b.copyto(executor.arg_dict['B'])
flag = False
print "execution begin"
for i in range(args.iter_num):
if i == args.begin_profiling_iter:
t0 = time.clock()
mx.profiler.profiler_set_state('run')
if i == args.end_profiling_iter:
t1 = time.clock()
mx.profiler.profiler_set_state('stop')
executor.forward()
c = executor.outputs[0]
c.wait_to_read()
print "execution end"
duration = t1 - t0
print('duration: {0}s'.format(duration))
print(' {0}ms/operator'.format(duration * 1000 / args.iter_num))
l1_b = []
l2_b = []
train_loss = []
test_loss = []
test_mean_loss = []
prev_loss = -1
loss_std = 0
loss_rate = []
# Learning loop
for epoch in xrange(1, n_epoch + 1):
print('epoch', epoch)
start_time = time.clock()
# training
perm = np.random.permutation(N)
sum_loss = 0
for i in xrange(0, N, batchsize):
x_batch = x_train[perm[i:i + batchsize]]
y_batch = y_train[perm[i:i + batchsize]]
optimizer.zero_grads()
loss = forward(x_batch, y_batch)
loss.backward()
optimizer.update()
train_loss.append(loss.data)
sum_loss += float(cuda.to_cpu(loss.data)) * batchsize
def read(humfile, sonpath, cs2cs_args, c, draft, doplot, t, bedpick, flip_lr,
model, calc_bearing, filt_bearing, chunk): #cog = 1,
'''
Read a .DAT and associated set of .SON files recorded by a Humminbird(R)
instrument.
Parse the data into a set of memory mapped files that will
subsequently be used by the other functions of the PyHum module.
Export time-series data and metadata in other formats.
Create a kml file for visualising boat track
Syntax
----------
[] = PyHum.read(humfile, sonpath, cs2cs_args, c, draft, doplot, t, bedpick, flip_lr, chunksize, model, calc_bearing, filt_bearing, chunk)
Parameters
------------
humfile : str
path to the .DAT file
sonpath : str
path where the *.SON files are
cs2cs_args : int, *optional* [Default="epsg:26949"]
arguments to create coordinates in a projected coordinate system
this argument gets given to pyproj to turn wgs84 (lat/lon) coordinates
into any projection supported by the proj.4 libraries
c : float, *optional* [Default=1450.0]
speed of sound in water (m/s). Defaults to a value of freshwater
draft : float, *optional* [Default=0.3]
draft from water surface to transducer face (m)
doplot : float, *optional* [Default=1]
if 1, plots will be made
t : float, *optional* [Default=0.108]
length of transducer array (m).
Default value is that of the 998 series Humminbird(R)
bedpick : int, *optional* [Default=1]
if 1, bedpicking with be carried out automatically
if 0, user will be prompted to pick the bed location on screen
flip_lr : int, *optional* [Default=0]
if 1, port and starboard scans will be flipped
(for situations where the transducer is flipped 180 degrees)
model: int, *optional* [Default=998]
A 3 or 4 number code indicating the model number
Examples: 998, 997, 1198, 1199
calc_bearing : float, *optional* [Default=0]
if 1, bearing will be calculated from coordinates
filt_bearing : float, *optional* [Default=0]
if 1, bearing will be filtered
chunk : str, *optional* [Default='d100' (distance, 100 m)]
letter, followed by a number.
There are the following letter options:
'd' - parse chunks based on distance, then number which is distance in m
'p' - parse chunks based on number of pings, then number which is number of pings
'h' - parse chunks based on change in heading, then number which is the change in heading in degrees
'1' - process just 1 chunk
Returns
---------
sonpath+base+'_data_port.dat': memory-mapped file
contains the raw echogram from the port side
sidescan sonar (where present)
sonpath+base+'_data_port.dat': memory-mapped file
contains the raw echogram from the starboard side
sidescan sonar (where present)
sonpath+base+'_data_dwnhi.dat': memory-mapped file
contains the raw echogram from the high-frequency
echosounder (where present)
sonpath+base+'_data_dwnlow.dat': memory-mapped file
contains the raw echogram from the low-frequency
echosounder (where present)
sonpath+base+"trackline.kml": google-earth kml file
contains the trackline of the vessel during data
acquisition
sonpath+base+'rawdat.csv': comma separated value file
contains time-series data. columns corresponding to
longitude
latitude
easting (m)
northing (m)
depth to bed (m)
alongtrack cumulative distance (m)
vessel heading (deg.)
sonpath+base+'meta.mat': .mat file
matlab format file containing a dictionary object
holding metadata information. Fields are:
e : ndarray, easting (m)
n : ndarray, northing (m)
es : ndarray, low-pass filtered easting (m)
ns : ndarray, low-pass filtered northing (m)
lat : ndarray, latitude
lon : ndarray, longitude
shape_port : tuple, shape of port scans in memory mapped file
shape_star : tuple, shape of starboard scans in memory mapped file
shape_hi : tuple, shape of high-freq. scans in memory mapped file
shape_low : tuple, shape of low-freq. scans in memory mapped file
dep_m : ndarray, depth to bed (m)
dist_m : ndarray, distance along track (m)
heading : ndarray, heading of vessel (deg. N)
pix_m: float, size of 1 pixel in across-track dimension (m)
bed : ndarray, depth to bed (m)
c : float, speed of sound in water (m/s)
t : length of sidescan transducer array (m)
spd : ndarray, vessel speed (m/s)
time_s : ndarray, time elapsed (s)
caltime : ndarray, unix epoch time (s)
'''
# prompt user to supply file if no input file given
if not humfile:
print('An input file is required!!!!!!')
Tk().withdraw(
) # we don't want a full GUI, so keep the root window from appearing
humfile = askopenfilename(filetypes=[("DAT files", "*.DAT")])
# prompt user to supply directory if no input sonpath is given
if not sonpath:
print('A *.SON directory is required!!!!!!')
Tk().withdraw(
) # we don't want a full GUI, so keep the root window from appearing
sonpath = askdirectory()
# print given arguments to screen and convert data type where necessary
if humfile:
print('Input file is %s' % (humfile))
if sonpath:
print('Son files are in %s' % (sonpath))
if cs2cs_args:
print('cs2cs arguments are %s' % (cs2cs_args))
if draft:
draft = float(draft)
print('Draft: %s' % (str(draft)))
if c:
c = float(c)
print('Celerity of sound: %s m/s' % (str(c)))
if doplot:
doplot = int(doplot)
if doplot == 0:
print("Plots will not be made")
if flip_lr:
flip_lr = int(flip_lr)
if flip_lr == 1:
print("Port and starboard will be flipped")
if t:
t = np.asarray(t, float)
print('Transducer length is %s m' % (str(t)))
if bedpick:
bedpick = np.asarray(bedpick, int)
if bedpick == 1:
print('Bed picking is auto')
elif bedpick == 0:
print('Bed picking is manual')
else:
print('User will be prompted per chunk about bed picking method')
if chunk:
chunk = str(chunk)
if chunk[0] == 'd':
chunkmode = 1
chunkval = int(chunk[1:])
print('Chunks based on distance of %s m' % (str(chunkval)))
elif chunk[0] == 'p':
chunkmode = 2
chunkval = int(chunk[1:])
print('Chunks based on %s pings' % (str(chunkval)))
elif chunk[0] == 'h':
chunkmode = 3
chunkval = int(chunk[1:])
print('Chunks based on heading devation of %s degrees' %
(str(chunkval)))
elif chunk[0] == '1':
chunkmode = 4
chunkval = 1
print('Only 1 chunk will be produced')
else:
print(
"Chunk mode not understood - should be 'd', 'p', or 'h' - using defaults"
)
chunkmode = 1
chunkval = 100
print('Chunks based on distance of %s m' % (str(chunkval)))
if model:
try:
model = int(model)
print("Data is from the %s series" % (str(model)))
except:
if model == 'onix':
model = 0
print("Data is from the ONIX series")
elif model == 'helix':
model = 1
print("Data is from the HELIX series")
elif model == 'mega':
model = 2
print("Data is from the MEGA series")
# if cog:
# cog = int(cog)
# if cog==1:
# print "Heading based on course-over-ground"
if calc_bearing:
calc_bearing = int(calc_bearing)
if calc_bearing == 1:
print("Bearing will be calculated from coordinates")
if filt_bearing:
filt_bearing = int(filt_bearing)
if filt_bearing == 1:
print("Bearing will be filtered")
## for debugging
#humfile = r"test.DAT"; sonpath = "test_data"
#cs2cs_args = "epsg:26949"; doplot = 1; draft = 0
#c=1450; bedpick=1; fliplr=1; chunk = 'd100'
#model=998; cog=1; calc_bearing=0; filt_bearing=0
#if model==2:
# f = 1000
#else:
f = 455
try:
print(
"Checking the epsg code you have chosen for compatibility with Basemap ... "
)
from mpl_toolkits.basemap import Basemap
m = Basemap(projection='merc',
epsg=cs2cs_args.split(':')[1],
resolution='i',
llcrnrlon=10,
llcrnrlat=10,
urcrnrlon=30,
urcrnrlat=30)
del m
print("... epsg code compatible")
except (ValueError):
print(
"Error: the epsg code you have chosen is not compatible with Basemap"
)
print(
"please choose a different epsg code (http://spatialreference.org/)"
)
print("program will now close")
sys.exit()
# start timer
if os.name == 'posix': # true if linux/mac or cygwin on windows
start = time.time()
else: # windows
start = time.clock()
# if son path name supplied has no separator at end, put one on
if sonpath[-1] != os.sep:
sonpath = sonpath + os.sep
# get the SON files from this directory
sonfiles = glob.glob(sonpath + '*.SON')
if not sonfiles:
sonfiles = glob.glob(os.getcwd() + os.sep + sonpath + '*.SON')
base = humfile.split('.DAT') # get base of file name for output
base = base[0].split(os.sep)[-1]
# remove underscores, negatives and spaces from basename
base = humutils.strip_base(base)
print("WARNING: Because files have to be read in byte by byte,")
print("this could take a very long time ...")
#reading each sonfile in parallel should be faster ...
try:
o = Parallel(n_jobs=np.min([len(sonfiles), cpu_count()]), verbose=0)(
delayed(getscans)(sonfiles[k], humfile, c, model, cs2cs_args)
for k in range(len(sonfiles)))
X, Y, A, B = zip(*o)
for k in range(len(Y)):
if Y[k] == 'sidescan_port':
dat = A[k] #data.gethumdat()
metadat = B[k] #data.getmetadata()
if flip_lr == 0:
data_port = X[k].astype('int16')
else:
data_star = X[k].astype('int16')
elif Y[k] == 'sidescan_starboard':
if flip_lr == 0:
data_star = X[k].astype('int16')
else:
data_port = X[k].astype('int16')
elif Y[k] == 'down_lowfreq':
data_dwnlow = X[k].astype('int16')
elif Y[k] == 'down_highfreq':
data_dwnhi = X[k].astype('int16')
elif Y[k] == 'down_vhighfreq': #hopefully this only applies to mega systems
data_dwnhi = X[k].astype('int16')
del X, Y, A, B, o
old_pyread = 0
if 'data_port' not in locals():
data_port = ''
print("portside scan not available")
if 'data_star' not in locals():
data_star = ''
print("starboardside scan not available")
if 'data_dwnhi' not in locals():
data_dwnlow = ''
print("high-frq. downward scan not available")
if 'data_dwnlow' not in locals():
data_dwnlow = ''
print("low-frq. downward scan not available")
except: # revert back to older version if paralleleised version fails
print(
"something went wrong with the parallelised version of pyread ...")
try:
import pyread
except:
from . import pyread
data = pyread.pyread(sonfiles, humfile, c, model, cs2cs_args)
dat = data.gethumdat()
metadat = data.getmetadata()
old_pyread = 1
nrec = len(metadat['n'])
metadat['instr_heading'] = metadat['heading'][:nrec]
#metadat['heading'] = humutils.get_bearing(calc_bearing, filt_bearing, cog, metadat['lat'], metadat['lon'], metadat['instr_heading'])
try:
es = humutils.runningMeanFast(metadat['e'][:nrec],
len(metadat['e'][:nrec]) / 100)
ns = humutils.runningMeanFast(metadat['n'][:nrec],
len(metadat['n'][:nrec]) / 100)
except:
es = metadat['e'][:nrec]
ns = metadat['n'][:nrec]
metadat['es'] = es
metadat['ns'] = ns
try:
trans = pyproj.Proj(init=cs2cs_args)
except:
trans = pyproj.Proj(cs2cs_args.lstrip(), inverse=True)
lon, lat = trans(es, ns, inverse=True)
metadat['lon'] = lon
metadat['lat'] = lat
metadat['heading'] = humutils.get_bearing(calc_bearing, filt_bearing,
metadat['lat'], metadat['lon'],
metadat['instr_heading']) #cog
dist_m = humutils.get_dist(lat, lon)
metadat['dist_m'] = dist_m
if calc_bearing == 1: # recalculate speed, m/s
ds = np.gradient(np.squeeze(metadat['time_s']))
dx = np.gradient(np.squeeze(metadat['dist_m']))
metadat['spd'] = dx[:nrec] / ds[:nrec]
# theta at 3dB in the horizontal
theta3dB = np.arcsin(c / (t * (f * 1000)))
#resolution of 1 sidescan pixel to nadir
ft = (np.pi / 2) * (1 / theta3dB) #/ (f/455)
dep_m = humutils.get_depth(metadat['dep_m'][:nrec])
if old_pyread == 1: #older pyread version
# port scan
try:
if flip_lr == 0:
data_port = data.getportscans().astype('int16')
else:
data_port = data.getstarscans().astype('int16')
except:
data_port = ''
print("portside scan not available")
if data_port != '':
Zt, ind_port = makechunks_scan(chunkmode, chunkval, metadat, data_port,
0)
del data_port
## create memory mapped file for Z
shape_port = io.set_mmap_data(sonpath, base, '_data_port.dat', 'int16',
Zt)
##we are only going to access the portion of memory required
port_fp = io.get_mmap_data(sonpath, base, '_data_port.dat', 'int16',
shape_port)
if old_pyread == 1: #older pyread version
# starboard scan
try:
if flip_lr == 0:
data_star = data.getstarscans().astype('int16')
else:
data_star = data.getportscans().astype('int16')
except:
data_star = ''
print("starboardside scan not available")
if data_star != '':
Zt, ind_star = makechunks_scan(chunkmode, chunkval, metadat, data_star,
1)
del data_star
# create memory mapped file for Z
shape_star = io.set_mmap_data(sonpath, base, '_data_star.dat', 'int16',
Zt)
star_fp = io.get_mmap_data(sonpath, base, '_data_star.dat', 'int16',
shape_star)
if 'star_fp' in locals() and 'port_fp' in locals():
# check that port and starboard are same size
# and trim if not
if np.shape(star_fp) != np.shape(port_fp):
print(
"port and starboard scans are different sizes ... rectifying")
if np.shape(port_fp[0])[1] > np.shape(star_fp[0])[1]:
tmp = port_fp.copy()
tmp2 = np.empty_like(star_fp)
for k in range(len(tmp)):
tmp2[k] = tmp[k][:, :np.shape(star_fp[k])[1]]
del tmp
# create memory mapped file for Z
shape_port = io.set_mmap_data(sonpath, base, '_data_port2.dat',
'int16', tmp2)
#shape_star = shape_port.copy()
shape_star = tuple(np.asarray(shape_port).copy())
##we are only going to access the portion of memory required
port_fp = io.get_mmap_data(sonpath, base, '_data_port2.dat',
'int16', shape_port)
ind_port = list(ind_port)
ind_port[-1] = np.shape(star_fp[0])[1]
ind_port = tuple(ind_port)
elif np.shape(port_fp[0])[1] < np.shape(star_fp[0])[1]:
tmp = star_fp.copy()
tmp2 = np.empty_like(port_fp)
for k in range(len(tmp)):
tmp2[k] = tmp[k][:, :np.shape(port_fp[k])[1]]
del tmp
# create memory mapped file for Z
shape_port = io.set_mmap_data(sonpath, base, '_data_star2.dat',
'int16', tmp2)
#shape_star = shape_port.copy()
shape_star = tuple(np.asarray(shape_port).copy())
#we are only going to access the portion of memory required
star_fp = io.get_mmap_data(sonpath, base, '_data_star2.dat',
'int16', shape_star)
ind_star = list(ind_star)
ind_star[-1] = np.shape(port_fp[0])[1]
ind_star = tuple(ind_star)
if old_pyread == 1: #older pyread version
# low-freq. sonar
try:
data_dwnlow = data.getlowscans().astype('int16')
except:
data_dwnlow = ''
print("low-freq. scan not available")
if data_dwnlow != '':
Zt, ind_low = makechunks_scan(chunkmode, chunkval, metadat,
data_dwnlow, 2)
del data_dwnlow
# create memory mapped file for Z
shape_low = io.set_mmap_data(sonpath, base, '_data_dwnlow.dat',
'int16', Zt)
##we are only going to access the portion of memory required
dwnlow_fp = io.get_mmap_data(sonpath, base, '_data_dwnlow.dat',
'int16', shape_low)
if old_pyread == 1: #older pyread version
# hi-freq. sonar
try:
data_dwnhi = data.gethiscans().astype('int16')
except:
data_dwnhi = ''
print("high-freq. scan not available")
if data_dwnhi != '':
Zt, ind_hi = makechunks_scan(chunkmode, chunkval, metadat, data_dwnhi,
3)
del data_dwnhi
# create memory mapped file for Z
shape_hi = io.set_mmap_data(sonpath, base, '_data_dwnhi.dat', 'int16',
Zt)
dwnhi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi.dat', 'int16',
shape_hi)
if 'dwnhi_fp' in locals() and 'dwnlow_fp' in locals():
# check that low and high are same size
# and trim if not
if (np.shape(dwnhi_fp) != np.shape(dwnlow_fp)) and (chunkmode != 4):
print("dwnhi and dwnlow are different sizes ... rectifying")
if np.shape(dwnhi_fp[0])[1] > np.shape(dwnlow_fp[0])[1]:
tmp = dwnhi_fp.copy()
tmp2 = np.empty_like(dwnlow_fp)
for k in range(len(tmp)):
tmp2[k] = tmp[k][:, :np.shape(dwnlow_fp[k])[1]]
del tmp
# create memory mapped file for Z
shape_low = io.set_mmap_data(sonpath, base, '_data_dwnhi2.dat',
'int16', tmp2)
#shape_hi = shape_low.copy()
shape_hi = tuple(np.asarray(shape_low).copy())
##we are only going to access the portion of memory required
dwnhi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi2.dat',
'int16', shape_hi)
ind_hi = list(ind_hi)
ind_hi[-1] = np.shape(dwnlow_fp[0])[1]
ind_hi = tuple(ind_hi)
elif np.shape(dwnhi_fp[0])[1] < np.shape(dwnlow_fp[0])[1]:
tmp = dwnlow_fp.copy()
tmp2 = np.empty_like(dwnhi_fp)
for k in range(len(tmp)):
tmp2[k] = tmp[k][:, :np.shape(dwnhi_fp[k])[1]]
del tmp
# create memory mapped file for Z
shape_low = io.set_mmap_data(sonpath, base,
'_data_dwnlow2.dat', 'int16',
tmp2)
#shape_hi = shape_low.copy()
shape_hi = tuple(np.asarray(shape_low).copy())
##we are only going to access the portion of memory required
dwnlow_fp = io.get_mmap_data(sonpath, base,
'_data_dwnlow2.dat', 'int16',
shape_low)
ind_low = list(ind_low)
ind_low[-1] = np.shape(dwnhi_fp[0])[1]
ind_low = tuple(ind_low)
if old_pyread == 1: #older pyread version
del data
if ('shape_port' in locals()) and (chunkmode != 4):
metadat['shape_port'] = shape_port
nrec = metadat['shape_port'][0] * metadat['shape_port'][2]
elif ('shape_port' in locals()) and (chunkmode == 4):
metadat['shape_port'] = shape_port
nrec = metadat['shape_port'][1]
else:
metadat['shape_port'] = ''
if ('shape_star' in locals()) and (chunkmode != 4):
metadat['shape_star'] = shape_star
nrec = metadat['shape_star'][0] * metadat['shape_star'][2]
elif ('shape_star' in locals()) and (chunkmode == 4):
metadat['shape_star'] = shape_star
nrec = metadat['shape_star'][1]
else:
metadat['shape_star'] = ''
if ('shape_hi' in locals()) and (chunkmode != 4):
metadat['shape_hi'] = shape_hi
#nrec = metadat['shape_hi'][0] * metadat['shape_hi'][2] * 2
elif ('shape_hi' in locals()) and (chunkmode == 4):
metadat['shape_hi'] = shape_hi
else:
metadat['shape_hi'] = ''
if ('shape_low' in locals()) and (chunkmode != 4):
metadat['shape_low'] = shape_low
#nrec = metadat['shape_low'][0] * metadat['shape_low'][2] * 2
elif ('shape_low' in locals()) and (chunkmode == 4):
metadat['shape_low'] = shape_low
else:
metadat['shape_low'] = ''
#make kml boat trackline
humutils.make_trackline(lon, lat, sonpath, base)
if 'port_fp' in locals() and 'star_fp' in locals():
#if not os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'meta.mat'))):
if 2 > 1:
if bedpick == 1: # auto
x, bed = humutils.auto_bedpick(ft, dep_m, chunkmode, port_fp,
c)
if len(dist_m) < len(bed):
dist_m = np.append(
dist_m, dist_m[-1] * np.ones(len(bed) - len(dist_m)))
if doplot == 1:
if chunkmode != 4:
for k in range(len(star_fp)):
plot_2bedpicks(
port_fp[k], star_fp[k],
bed[ind_port[-1] * k:ind_port[-1] * (k + 1)],
dist_m[ind_port[-1] * k:ind_port[-1] *
(k + 1)],
x[ind_port[-1] * k:ind_port[-1] * (k + 1)], ft,
shape_port, sonpath, k, chunkmode)
else:
plot_2bedpicks(port_fp, star_fp, bed, dist_m, x, ft,
shape_port, sonpath, 0, chunkmode)
# 'real' bed is estimated to be the minimum of the two
bed = np.min(np.vstack((bed[:nrec], np.squeeze(x[:nrec]))),
axis=0)
bed = humutils.runningMeanFast(bed, 3)
elif bedpick > 1: # user prompt
x, bed = humutils.auto_bedpick(ft, dep_m, chunkmode, port_fp,
c)
if len(dist_m) < len(bed):
dist_m = np.append(
dist_m, dist_m[-1] * np.ones(len(bed) - len(dist_m)))
# 'real' bed is estimated to be the minimum of the two
bed = np.min(np.vstack((bed[:nrec], np.squeeze(x[:nrec]))),
axis=0)
bed = humutils.runningMeanFast(bed, 3)
# manually intervene
fig = plt.figure()
ax = plt.gca()
if chunkmode != 4:
im = ax.imshow(np.hstack(port_fp),
cmap='gray',
origin='upper')
else:
im = ax.imshow(port_fp, cmap='gray', origin='upper')
plt.plot(bed, 'r')
plt.axis('normal')
plt.axis('tight')
pts1 = plt.ginput(
n=300,
timeout=30) # it will wait for 200 clicks or 60 seconds
x1 = map(lambda x: x[0],
pts1) # map applies the function passed as
y1 = map(lambda x: x[1],
pts1) # first parameter to each element of pts
plt.close()
del fig
if x1 != []: # if x1 is not empty
tree = KDTree(zip(np.arange(1, len(bed)), bed))
try:
dist, inds = tree.query(zip(x1, y1),
k=100,
eps=5,
n_jobs=-1)
except:
dist, inds = tree.query(zip(x1, y1), k=100, eps=5)
b = np.interp(inds, x1, y1)
bed2 = bed.copy()
bed2[inds] = b
bed = bed2
if doplot == 1:
if chunkmode != 4:
for k in range(len(star_fp)):
plot_2bedpicks(
port_fp[k], star_fp[k],
bed[ind_port[-1] * k:ind_port[-1] * (k + 1)],
dist_m[ind_port[-1] * k:ind_port[-1] *
(k + 1)],
x[ind_port[-1] * k:ind_port[-1] * (k + 1)], ft,
shape_port, sonpath, k, chunkmode)
else:
plot_2bedpicks(port_fp, star_fp, bed, dist_m, x, ft,
shape_port, sonpath, 0, chunkmode)
else: #manual
beds = []
if chunkmode != 4:
for k in range(len(port_fp)):
raw_input(
"Bed picking " + str(k + 1) + " of " +
str(len(port_fp)) +
", are you ready? 30 seconds. Press Enter to continue..."
)
bed = {}
fig = plt.figure()
ax = plt.gca()
im = ax.imshow(port_fp[k], cmap='gray', origin='upper')
pts1 = plt.ginput(
n=300, timeout=30
) # it will wait for 200 clicks or 60 seconds
x1 = map(lambda x: x[0],
pts1) # map applies the function passed as
y1 = map(
lambda x: x[1],
pts1) # first parameter to each element of pts
bed = np.interp(np.r_[:ind_port[-1]], x1, y1)
plt.close()
del fig
beds.append(bed)
extent = np.shape(port_fp[k])[0]
bed = np.asarray(np.hstack(beds), 'float')
else:
raw_input(
"Bed picking - are you ready? 30 seconds. Press Enter to continue..."
)
bed = {}
fig = plt.figure()
ax = plt.gca()
im = ax.imshow(port_fp, cmap='gray', origin='upper')
pts1 = plt.ginput(
n=300, timeout=30
) # it will wait for 200 clicks or 60 seconds
x1 = map(lambda x: x[0],
pts1) # map applies the function passed as
y1 = map(lambda x: x[1],
pts1) # first parameter to each element of pts
bed = np.interp(np.r_[:ind_port[-1]], x1, y1)
plt.close()
del fig
beds.append(bed)
extent = np.shape(port_fp)[1]
bed = np.asarray(np.hstack(beds), 'float')
# now revise the depth in metres
dep_m = (1 / ft) * bed
if doplot == 1:
if chunkmode != 4:
for k in range(len(star_fp)):
plot_bedpick(
port_fp[k], star_fp[k], (1 / ft) *
bed[ind_port[-1] * k:ind_port[-1] * (k + 1)],
dist_m[ind_port[-1] * k:ind_port[-1] * (k + 1)],
ft, shape_port, sonpath, k, chunkmode)
else:
plot_bedpick(port_fp, star_fp, (1 / ft) * bed, dist_m, ft,
shape_port, sonpath, 0, chunkmode)
metadat['bed'] = bed[:nrec]
else:
metadat['bed'] = dep_m[:nrec] * ft
metadat['heading'] = metadat['heading'][:nrec]
metadat['lon'] = lon[:nrec]
metadat['lat'] = lat[:nrec]
metadat['dist_m'] = dist_m[:nrec]
metadat['dep_m'] = dep_m[:nrec]
metadat['pix_m'] = 1 / ft
metadat['bed'] = metadat['bed'][:nrec]
metadat['c'] = c
metadat['t'] = t
if model == 2:
metadat['f'] = f * 2
else:
metadat['f'] = f
metadat['spd'] = metadat['spd'][:nrec]
metadat['time_s'] = metadat['time_s'][:nrec]
metadat['e'] = metadat['e'][:nrec]
metadat['n'] = metadat['n'][:nrec]
metadat['es'] = metadat['es'][:nrec]
metadat['ns'] = metadat['ns'][:nrec]
try:
metadat['caltime'] = metadat['caltime'][:nrec]
except:
metadat['caltime'] = metadat['caltime']
savemat(os.path.normpath(os.path.join(sonpath, base + 'meta.mat')),
metadat,
oned_as='row')
f = open(os.path.normpath(os.path.join(sonpath, base + 'rawdat.csv')),
'wt')
writer = csv.writer(f)
writer.writerow(
('longitude', 'latitude', 'easting', 'northing', 'depth (m)',
'distance (m)', 'instr. heading (deg)', 'heading (deg.)'))
for i in range(0, nrec):
writer.writerow(
(float(lon[i]), float(lat[i]), float(es[i]), float(ns[i]),
float(dep_m[i]), float(dist_m[i]),
float(metadat['instr_heading'][i]), float(metadat['heading'][i])))
f.close()
del lat, lon, dep_m #, dist_m
if doplot == 1:
plot_pos(sonpath, metadat, es, ns)
if 'dwnlow_fp' in locals():
plot_dwnlow(dwnlow_fp, chunkmode, sonpath)
if 'dwnhi_fp' in locals():
plot_dwnhi(dwnhi_fp, chunkmode, sonpath)
if os.name == 'posix': # true if linux/mac
elapsed = (time.time() - start)
else: # windows
elapsed = (time.clock() - start)
print("Processing took " + str(elapsed) + "seconds to analyse")
print("Done!")
print("===================================================")
r_file.close()
print ("\ndone")
break
def main():
global a, sgkdir, pr_s, result_queue, wr_s
pr_p = threading.Thread(target=pr)
wr_p = threading.Thread(target=wr)
pr_s = True
wr_s = True
pr_p.start()
wr_p.start()
for target_file in get_file(sgkdir):
b = config_handler()
a = 0
print "Start handle %s" % target_file
for line in open(target_file, "rb"):
a += 1
line_temp = line.strip()
if line_temp != "":
line_temp = b.handle(line_temp)
result_queue.append(line_temp)
print "\nCompleted: %s" % target_file
wr_s = False
pr_s = False
exit(0)
if __name__ == "__main__":
start = time.clock()
main()
all_layouts.append([onto_basis2, onto_basis22, pur_p])
print 'End purpose:', pur_p, '--------------------------------------------------------------'
#
print 'Start process page:---'
process_termo(termo, username, pur_p, start_c, '', all_layouts)
#=========================
if len(entry_doc) > 0: return
start_c = 0
import time
startTT = time.clock()
def remp(s):
r = ''
i = len(s) - 1
pos = 0
while i >= 0:
if s[i] == '\\' or s[i] == '/':
pos = i
break
r = s[i] + r
i -= 1
return s[:pos + 1]
'b0': b0,
'N': N,
'nw': 0.2, # net worth of the central bank
'pibar': 1.6, # constraint on central bank losses
'epeg': epeg,
'ebar': ebar,
'gamma': gamma,
'lam': lam,
'g_val': g_val,
'c_states': (0, 1), # (low perm, low transitory, high transitory)
'c_transition': interest_transition,
'c_istar_values': (ilow, ihigh)
}
sns.set_style("whitegrid") # optional seabird settings
t1 = time.clock()
model = gs.Model(**par)
print("time : {} seconds".format(time.clock() - t1))
gs.do_plots(model, save_to_file=False, file_name='benchmark_figure.pdf')
robustness = {}
labels = {}
markers = ['^--', 'o-', 's--']
robustness['ebar'] = []
labels['ebar'] = []
for ebar_val, m in zip([0.65, 0.7], markers):
par2 = par.copy()
par2['ebar'] = ebar_val
robustness['ebar'].append(gs.Model(**par2))
lab = {}
def kk(self, energy=None, mu=None, z=None, edge='K', how='scalar', mback_kws=None):
"""
Convert mu(E) data into f'(E) and f"(E). f"(E) is made by
matching mu(E) to the tabulated values of the imaginary part
of the scattering factor (Cromer-Liberman), f'(E) is then
obtained by performing a differential Kramers-Kronig transform
on the matched f"(E).
Attributes
energy: energy array
mu: array with mu(E) data
z: Z number of absorber
edge: absorption edge, usually 'K' or 'L3'
mback_kws: arguments for the mback algorithm
Returns
self.f1, self.f2: CL values over on the input energy grid
self.fp, self.fpp: matched and KK transformed data on the input energy grid
References:
* Cromer-Liberman: http://dx.doi.org/10.1063/1.1674266
* KK computation: Ohta and Ishida, Applied Spectroscopy 42:6 (1988) 952-957
* diffKK implementation: http://dx.doi.org/10.1103/PhysRevB.58.11215
* MBACK (Weng, Waldo, Penner-Hahn): http://dx.doi.org/10.1086/303711
* Lee and Xiang: http://dx.doi.org/10.1088/0004-637X/702/2/970
"""
if type(energy).__name__ == 'ndarray': self.energy = energy
if type(mu).__name__ == 'ndarray': self.mu = mu
if z != None: self.z = z
if edge != None: self.edge = edge
if mback_kws != None: self.mback_kws = mback_kws
if self.z == None:
Exception("Z for absorber not provided for diffKK")
if self.edge == None:
Exception("absorption edge not provided for diffKK")
mb_kws = dict(order=3, z=self.z, edge=self.edge, e0=None, emin=None, emax=None,
whiteline=False, leexiang=False, tables='chantler',
fit_erfc=False, return_f1=True)
if self.mback_kws is not None:
mb_kws.update(self.mback_kws)
start = time.clock()
mback(self.energy, self.mu, group=self, _larch=self._larch, **mb_kws)
## interpolate matched data onto an even grid with an even number of elements (about 1 eV)
npts = int(self.energy[-1] - self.energy[0]) + (int(self.energy[-1] - self.energy[0])%2)
self.grid = np.linspace(self.energy[0], self.energy[-1], npts)
fpp = interp(self.energy, self.f2-self.fpp, self.grid, fill_value=0.0)
## do difference KK
if repr(how).startswith('sca'):
fp = kkmclr_sca(self.grid, fpp)
else:
fp = kkmclr(self.grid, fpp)
## interpolate back to original grid and add diffKK result to f1 to make fp array
self.fp = self.f1 + interp(self.grid, fp, self.energy, fill_value=0.0)
## clean up group
#for att in ('normalization_function', 'weight', 'grid'):
# if hasattr(self, att): delattr(self, att)
finish = time.clock()
self.time_elapsed = float(finish-start)
ac = karatsuba(int1_split[0], int2_split[0])
bd = karatsuba(int1_split[1], int2_split[1])
ab = np.sum(int1_split[0:2])
cd = np.sum(int2_split[0:2])
apb_dpc = karatsuba(ab, cd) - ac - bd
return np.power(power, 2) * ac + power * apb_dpc + bd
###Ejemplos
karatsuba(1234, 5678)
karatsuba(12341234, 234)
######################################################################################
## Pruebas de estress
######################################################################################
## Facil
time_start = time.clock()
karatsuba(1234, 5678)
time_elapsed = (time.clock() - time_start)
time_start = time.clock()
1234 * 5678
time_elapsed = (time.clock() - time_start)
## medio
time_start = time.clock()
karatsuba(12341234123412341234, 1241241234123412341234)
time_elapsed = (time.clock() - time_start)
time_start = time.clock()
12341234123412341234 * 1241241234123412341234
time_elapsed = (time.clock() - time_start)
def parse_series(self, data, **kwargs):
log.debug('Parsing series: `%s` [options: %s]', data, kwargs)
guessit_options = self._guessit_options(kwargs)
valid = True
if kwargs.get('name'):
expected_titles = [kwargs['name']]
if kwargs.get('alternate_names'):
expected_titles.extend(kwargs['alternate_names'])
# apostrophe support
expected_titles = [title.replace('\'', '(?:\'|\\\'|\\\\\'|-|)?') for title in expected_titles]
guessit_options['expected_title'] = ['re:' + title for title in expected_titles]
if kwargs.get('id_regexps'):
guessit_options['id_regexps'] = kwargs.get('id_regexps')
start = time.clock()
# If no series name is provided, we don't tell guessit what kind of match we are looking for
# This prevents guessit from determining that too general of matches are series
parse_type = 'episode' if kwargs.get('name') else None
if parse_type:
guessit_options['type'] = parse_type
# NOTE: Guessit expects str on PY3 and unicode on PY2 hence the use of future.utils.native
try:
guess_result = guessit_api.guessit(native(data), options=guessit_options)
except GuessitException:
log.warning('Parsing %s with guessit failed. Most likely a unicode error.', data)
guess_result = {}
if guess_result.get('type') != 'episode':
valid = False
name = kwargs.get('name')
country = guess_result.get('country')
if not name:
name = guess_result.get('title')
if country and hasattr(country, 'alpha2'):
name += ' (%s)' % country.alpha2
elif guess_result.matches['title']:
# Make sure the name match is up to FlexGet standards
# Check there is no unmatched cruft before the matched name
title_start = guess_result.matches['title'][0].start
title_end = guess_result.matches['title'][0].end
if title_start != 0:
try:
pre_title = max((match[0].end for match in guess_result.matches.values() if
match[0].end <= title_start))
except ValueError:
pre_title = 0
for char in reversed(data[pre_title:title_start]):
if char.isalnum() or char.isdigit():
return SeriesParseResult(data=data, valid=False)
if char.isspace() or char in '._':
continue
else:
break
# Check the name doesn't end mid-word (guessit might put the border before or after the space after title)
if data[title_end - 1].isalnum() and len(data) <= title_end or \
not self._is_valid_name(data, guessit_options=guessit_options):
valid = False
# If we are in exact mode, make sure there is nothing after the title
if kwargs.get('strict_name'):
post_title = sys.maxsize
for match_type, matches in guess_result.matches.items():
if match_type in ['season', 'episode', 'date', 'regexpId']:
if matches[0].start < title_end:
continue
post_title = min(post_title, matches[0].start)
if matches[0].parent:
post_title = min(post_title, matches[0].parent.start)
for char in data[title_end:post_title]:
if char.isalnum() or char.isdigit():
valid = False
else:
valid = False
season = guess_result.get('season')
episode = guess_result.get('episode')
if episode is None and 'part' in guess_result:
episode = guess_result['part']
if isinstance(episode, list):
# guessit >=2.1.4 returns a list for multi-packs, but we just want the first one and the number of eps
episode = episode[0]
date = guess_result.get('date')
quality = self._quality(guess_result)
proper_count = self._proper_count(guess_result)
group = guess_result.get('release_group')
# Validate group with from_group
if not self._is_valid_groups(group, guessit_options.get('allow_groups', [])):
valid = False
# Validate country, TODO: LEGACY
if country and name.endswith(')'):
p_start = name.rfind('(')
if p_start != -1:
parenthetical = re.escape(name[p_start + 1:-1])
if parenthetical and parenthetical.lower() != str(country).lower():
valid = False
special = guess_result.get('episode_details', '').lower() == 'special'
if 'episode' not in guess_result.values_list:
episodes = len(guess_result.values_list.get('part', []))
else:
episodes = len(guess_result.values_list['episode'])
if episodes > 3:
valid = False
identified_by = kwargs.get('identified_by', 'auto')
identifier_type, identifier = None, None
if identified_by in ['date', 'auto']:
if date:
identifier_type = 'date'
identifier = date
if not identifier_type and identified_by in ['ep', 'auto']:
if episode is not None:
if season is None and kwargs.get('allow_seasonless', True):
if 'part' in guess_result:
season = 1
else:
episode_raw = guess_result.matches['episode'][0].initiator.raw
if episode_raw and any(c.isalpha() and c.lower() != 'v' for c in episode_raw):
season = 1
if season is not None:
identifier_type = 'ep'
identifier = (season, episode)
if not identifier_type and identified_by in ['id', 'auto']:
if guess_result.matches['regexpId']:
identifier_type = 'id'
identifier = '-'.join(match.value for match in guess_result.matches['regexpId'])
if not identifier_type and identified_by in ['sequence', 'auto']:
if episode is not None:
identifier_type = 'sequence'
identifier = episode
if (not identifier_type or guessit_options.get('prefer_specials')) and (special or
guessit_options.get('assume_special')):
identifier_type = 'special'
identifier = guess_result.get('episode_title', 'special')
if not identifier_type:
valid = False
# TODO: Legacy - Complete == invalid
if 'complete' in normalize_component(guess_result.get('other')):
valid = False
parsed = SeriesParseResult(
data=data,
name=name,
episodes=episodes,
identified_by=identified_by,
id=identifier,
id_type=identifier_type,
quality=quality,
proper_count=proper_count,
special=special,
group=group,
valid=valid
)
end = time.clock()
log.debug('Parsing result: %s (in %s ms)', parsed, (end - start) * 1000)
return parsed
quant_engine = create_engine(
'mysql+pymysql://{user}:{password}@{host}/{db}?charset={charset}'.format(**ConfigQuant))
# create source engine
spider_engine = create_engine(
'mysql+pymysql://{user}:{password}@{host}/{db}?charset={charset}'.format(**ConfigSpider2))
chunk_size = 10
# updateFull(quant_engine, spider_engine, chunk_size)
# supplementForAdjQuote(quant_engine)
updateIncrm(quant_engine, spider_engine)
if __name__ == '__main__':
# create target engine
quant_engine = create_engine(
'mysql+pymysql://{user}:{password}@{host}/{db}?charset={charset}'.format(**ConfigQuant))
# create source engine
spider_engine = create_engine(
'mysql+pymysql://{user}:{password}@{host}/{db}?charset={charset}'.format(**ConfigSpider2))
chunk_size = 10
t_start = time.clock()
# updateFull(quant_engine, spider_engine, chunk_size)
# updateIncrm(quant_engine, spider_engine)
print(time.clock() - t_start)
num_rep * 1.0 / size * 1.0
) / 3600.0, 'time grown:', par_vals['dt'] * par_vals['nstep'] * N
dx = num_rep / size
rem = np.mod(num_rep, size)
if rank >= size - rem: # this makes sure that it distributes the remainder as equally as possible.
start = dx * (size - rem) + (dx + 1) * (rank + rem - size)
stop = start + dx + 1
else:
start = dx * rank
stop = start + dx
if rank == size - 1:
stop = num_rep
print 'I am {0}, my start is {1}, stop {2}'.format(str(rank), str(start),
str(stop))
tic = time.clock()
for i0 in xrange(start, stop): # variable number of repeats for each core
c = g.discr_gen(par_vals)
temp = [obj for obj in c if obj.exists]
for i1 in range(X[1]):
# tic = time.clock()
temp0, temp1 = g.starting_popn_seeded_1(temp,
par_vals,
discr_time=True)
if np.mod(i1, save_freq) == 0:
np.save(
'../../Documents/data_storage/March17_dilution_symmetric_3_savedpop_model_{0}_rep_{1}_it_{2}_asymm'
.format(str(par_vals['modeltype']), str(i0),
str(i1)), temp1)
new_c = g.discr_time_1(par_vals, temp0)
mothers = [obj.mother for obj in new_c[0] if obj.exists]
B = np.eye( n, n )
Y = np.eye( n, 3 )
# X = sp.rand( n, 3 )
xfile = {100 : 'X.txt', 1000 : 'X2.txt', 10000 : 'X3.txt'}
X = np.fromfile( xfile[n], dtype = np.float64, sep = ' ' )
X.shape = (n, 3)
ivals = [1./vals[0]]
def precond( x ):
invA = spdiags( ivals, 0, n, n )
y = invA * x
if issparse( y ):
y = y.toarray()
return as2d( y )
precond = spdiags( ivals, 0, n, n )
# precond = None
tt = time.clock()
# B = None
eigs, vecs = lobpcg( X, A, B, blockVectorY = Y,
M = precond,
residualTolerance = 1e-4, maxIterations = 40,
largest = False, verbosityLevel = 1 )
print('solution time:', time.clock() - tt)
print(vecs)
print(eigs)
def equil_solver(w):
'''
This is a function that calculates the optimal policy
for k' and the stationary distribution for a given wage
and is used to get the equilibrium solution
'''
# operating profits, op
sizez = len(z_grid)
op = np.zeros((sizez, sizek))
for i in range(sizez):
for j in range(sizek):
op[i,j] = ((1 - a_l) * ((a_l / w) ** (a_l / (1 - a_l))) *
((kgrid[j] ** a_k) ** (1 / (1 - a_l))) * (z_grid[i] ** (1/(1 - a_l))))
# firm cash flow, e
e = np.zeros((sizez, sizek, sizek))
for i in range(sizez):
for j in range(sizek):
for k in range(sizek):
e[i, j, k] = (op[i,j] - kgrid[k] + ((1 - delta) * kgrid[j]) -
((psi / 2) * ((kgrid[k] - ((1 - delta) * kgrid[j])) ** 2)
/ kgrid[j]))
# value function iteration
VFtol = 1e-6
VFdist = 7.0
VFmaxiter = 3000
V = np.zeros((sizez, sizek))
Vmat = np.zeros((sizez, sizek, sizek))
Vstore = np.zeros((sizez, sizek, VFmaxiter))
VFiter = 1
start_time = time.clock()
while VFdist > VFtol and VFiter < VFmaxiter:
TV = V
Vmat = VFI_loop(V, e, betafirm, sizez, sizek, Vmat, pi)
Vstore[:, :, VFiter] = V.reshape(sizez, sizek,)
V = Vmat.max(axis=2)
PF = np.argmax(Vmat, axis=2)
Vstore[:,:, i] = V
VFdist = (np.absolute(V - TV)).max()
VFiter += 1
VFI_time = time.clock() - start_time
if VFiter < VFmaxiter:
print('Value function converged after this many iterations:', VFiter)
else:
print('Value function did not converge')
print('VFI took ', VFI_time, ' seconds to solve')
VF = V
# optimal capital stock k'
optK = kgrid[PF]
# stationary distribution
Gamma = np.ones((sizez, sizek)) * (1 / (sizek * sizez))
SDtol = 1e-12
SDdist = 7
SDiter = 0
SDmaxiter = 1000
while SDdist > SDtol and SDmaxiter > SDiter:
HGamma = SD_loop(PF, pi, Gamma, sizez, sizek)
SDdist = (np.absolute(HGamma - Gamma)).max()
Gamma = HGamma
SDiter += 1
if SDiter < SDmaxiter:
print('Stationary distribution converged after this many iterations: ',
SDiter)
else:
print('Stationary distribution did not converge')
return optK, Gamma
"""
该脚本演示了在MXNet中使用ART的简单示例。
该示例在MNIST数据集上训练一个小模型，并使用快速梯度符号方法创建对抗性示例。
这里我们使用ART分类器来训练模型，也可以为ART分类器提供一个预先训练好的模型。
选择这些参数是为了减少脚本的计算需求，而不是为了提高准确性。"""
import mxnet
from mxnet.gluon.nn import Conv2D, MaxPool2D, Flatten, Dense
import numpy as np
from art.attacks import FastGradientMethod
from art.classifiers import MXClassifier
from art.utils import load_mnist
import time
start = time.clock()
# Step 1: 加载MNIST数据集 28x28的灰度图 70000张手写数字图(6:1)
(x_train, y_train), (x_test,
y_test), min_pixel_value, max_pixel_value = load_mnist()
# Step 1a: 把轴转换到MXNet的NCHW格式
x_train = np.swapaxes(x_train, 1, 3)
x_test = np.swapaxes(x_test, 1, 3)
# Step 2: 创建模型
model = mxnet.gluon.nn.Sequential()
with model.name_scope():
model.add(Conv2D(channels=4, kernel_size=5, activation='relu'))
def main():
# Parsing options
parser = argparse.ArgumentParser(description='Hgt candidate evaluation.')
parser.add_argument('bamfile', help="BAM alignments file for both acceptor and donor, sorted via qname")
parser.add_argument('candfile', help="File with inter-chr translocation breakpoints")
parser.add_argument('acceptor', help="Name of acceptor reference (gi as in fasta)")
parser.add_argument('donor', help="Name of donor reference (gi as in fasta)")
# optional arguments (has default values)
parser.add_argument('--phagefile', default=None, help="SAM alignments file for phage database")
parser.add_argument('-o','--outfile', default="hgt_eval.vcf", help="Output VCF file with filtered, evaluated candidates")
parser.add_argument('-min', dest='min_size', default=100, type=int, help="minimal HGT size, default 100")
parser.add_argument('-max', dest='max_size', default=50000, type=int, help="maximal HGT size, default 50000")
parser.add_argument('--tolerance', default=20, type=int, help="Position range to remove duplicates, default 20")
parser.add_argument('--pair-support', dest='paired_reads', default=True, action='store_false', help="Turn on/off paired reads support, default TRUE")
parser.add_argument('--num-boot-regions', dest='boot_num', default=100, type=int, help="Number of sampled regions in acceptor for bootstrapping expected number of pairs and coverage for filtering, default 100")
parser.add_argument('--boot-sens', dest='boot_sens', default=95, type=int, choices=range(0, 100), help="Percent of cases for the candidate region to exceed values of sampled regions, default 95 percent")
options = parser.parse_args()
# Extracting parameters
hgt_minLength = options.min_size
hgt_maxLength = options.max_size
bf = pysam.Samfile(options.bamfile,'rb')
if (options.phagefile is not None):
psf = pysam.Samfile(options.phagefile,'r')
acc_tid = bf.gettid(options.acceptor)
don_tid = bf.gettid(options.donor)
options.out_all = options.outfile.strip('vcf')+'tsv'
print ('acc_tid', acc_tid, options.acceptor)
print ('don_tid', don_tid, options.donor)
print ('hgt_minLength', hgt_minLength)
print ('hgt_maxLength', hgt_maxLength)
print ('paired_reads', options.paired_reads)
print ('num_boot_regions', options.boot_num)
print ('boot_sensitivity', options.boot_sens)
# Getting acceptor genome length
options.acc_length = bf.lengths[acc_tid]
options.don_length = bf.lengths[don_tid]
# Extracting coverages
covs = [np.zeros((l,)) for l in bf.lengths]
num_matches = 0 # number of read matches, unused by now
for read in bf:
if not read.is_unmapped:
r_start = read.pos # start position
r_end = read.pos + read.qlen # end
covs[read.tid][r_start:r_end] += 1
num_matches += 1
bf.reset()
acc_cov = covs[acc_tid]
don_cov = covs[don_tid]
# Extracting phage read IDs, if any
phage_list = []
if (options.phagefile != None):
phage_list = [qname for qname in read_phage_pairs(psf)]
# Extracting candidate positions from candifate file
cand_list = [cand for cand in get_candidates(options.candfile, options.acceptor, options.donor)]
#indices = range(len(acc_pos))
#indices.sort(key=acc_pos.__getitem__)
cand_list.sort(key=operator.attrgetter('acc_pos'))
tstart = time.clock()
# Pair up single boundary candidates to HGT candidates conforming to size constraints
hgt_list = []
for ps in range(0, len(cand_list)):
cand_start = cand_list[ps]
acc_start = cand_start.acc_pos
don_start_temp = cand_start.don_pos
for pe in range(ps, len(cand_list)):
cand_end = cand_list[pe]
acc_end = cand_end.acc_pos + 1
don_end_temp = cand_end.don_pos + 1
# Exchange don_start/end so that don_start < don_end (we don't care about the orientation for now)
if (don_end_temp < don_start_temp):
don_start = don_end_temp
don_end = don_start_temp
else:
don_start = don_start_temp
don_end = don_end_temp
# Avoid similar hgt entries within tolerance
if (duplicate_entry(hgt_list, options.tolerance, acc_start, acc_end, don_start, don_end, cand_end.split_support)):
continue
# Continue if acceptor HGT region exceeds max length
if (abs(acc_end - acc_start) > hgt_maxLength):
break
if (abs(don_end - don_start) > hgt_minLength and abs(don_end - don_start) < hgt_maxLength):
split_support = (cand_start.split_support + cand_end.split_support)
hgt_list.append(HGT(acc_start, acc_end, cand_start.acc_base, cand_end.acc_base, don_start, don_end, split_support, options, acc_cov, don_cov))
print (time.clock() - tstart)
# Get (all/primary) read pairs which map to both donor and acceptor and add them to hgt attributes
if (options.paired_reads):
#for aln1, aln2 in read_pairs(bf):
for aln1, aln2 in read_primary_pairs(bf):
if aln1.is_unmapped:
continue
if aln2.is_unmapped:
continue
if (aln1.tid == don_tid) and (aln2.tid == don_tid):
for hgt in hgt_list:
hgt.add_don_pair_if_matching(aln1, aln2, phage_list)
if (aln1.tid != acc_tid) or (aln2.tid != don_tid):
aln1, aln2 = aln2, aln1 # Ensures that aln1 belongs to acceptor and aln2 to donor
if (aln1.tid != acc_tid) or (aln2.tid != don_tid):
continue
for hgt in hgt_list:
hgt.add_pair_if_matching(aln1, aln2, phage_list)
print ("total sample time ", time.clock() - tstart)
# Output results
print ("writing output")
with open(options.outfile, 'w') as vcf_out, open(options.out_all, 'w') as output:
# VCF header
writevcf = csv.writer(vcf_out, delimiter = '\t')
writevcf.writerow(['##fileformat=VCFv4.2'])
writevcf.writerow(['##source=DAISY'])
writevcf.writerow(['##INFO=<ID=EVENT,Number=1,Type=String,Description=\"Event identifier for breakends.\">'])
writevcf.writerow(['##contig=<ID='+options.acceptor+'>'])
writevcf.writerow(['##contig=<ID='+options.donor+'>'])
writevcf.writerow(['#CHROM', 'POS', 'ID', 'REF', 'ALT', 'QUAL', 'FILTER', 'INFO', 'FORMAT'])
# TSV header
writetsv = csv.writer(output, delimiter = '\t')
writetsv.writerow(['#AS: Acceptor start position'])
writetsv.writerow(['#AE: Acceptor end position'])
writetsv.writerow(['#DS: Donor start position'])
writetsv.writerow(['#DE: Donor end position'])
writetsv.writerow(['#MC: Mean coverage in region'])
writetsv.writerow(['#Split: Total number split-reads per region (including duplicates!)'])
writetsv.writerow(['#PS-S: Pairs spanning HGT boundaries'])
writetsv.writerow(['#PS-W: Pairs within HGT boundaries'])
writetsv.writerow(['#Phage: PS-S and PS-W reads mapping to phage database'])
writetsv.writerow(['#BS:MC/PS-S/PS-W: Percent of bootstrapped random regions with MC/PS-S/PS-W smaller than candidate'])
if (options.paired_reads):
writetsv.writerow(['AS', 'AE', 'MC', 'BS:MC', 'DS', 'DE', 'MC', 'Split', 'PS-S', 'PS-W', 'Phage', 'BS:MC', 'BS:PS-S', 'BS:PS-W'])
else:
writetsv.writerow(['AS', 'AE', 'MC', 'BS:MC', 'DS', 'DE', 'MC', 'Split', 'BS:MC'])
# write candidates
hgt_vcf_counter = 1
# Define sensitivity values for candidates to be reported in VCF
sens = float(options.boot_sens)/float(100)
sens_acc = 1.0 - sens
thresh = sens * float(options.boot_num)
for hgt in hgt_list:
# evaluate bootstrap
acc_cov_test = sum(1 for i in range(len(list(hgt.boot_acc_coverage_list))) if hgt.acc_mean > hgt.boot_acc_coverage_list[i])
don_cov_test = sum(1 for i in range(len(list(hgt.boot_don_coverage_list))) if hgt.don_mean > hgt.boot_don_coverage_list[i])
if (options.paired_reads):
# write all candidates to tsv file
cross_pair_test = sum(1 for i in range(len(list(hgt.boot_crossing_pairs_list))) if hgt.pair_support > hgt.boot_crossing_pairs_list[i])
don_pair_test = sum(1 for i in range(len(list(hgt.boot_don_pairs_list))) if hgt.don_pair_support > hgt.boot_don_pairs_list[i])
phage_support = 0
if ((hgt.pair_support + hgt.don_pair_support) > 0):
phage_support = float(hgt.phage_hits)/float((hgt.pair_support + hgt.don_pair_support))
writetsv.writerow([hgt.acc_start, hgt.acc_end, "%.2f" % hgt.acc_mean, acc_cov_test, hgt.don_start, hgt.don_end, "%.2f" % hgt.don_mean, hgt.split_support, hgt.pair_support, hgt.don_pair_support, "%.4f" % phage_support, don_cov_test, cross_pair_test, don_pair_test])
else:
writetsv.writerow([hgt.acc_start, hgt.acc_end, "%.2f" % hgt.acc_mean, acc_cov_test, hgt.don_start, hgt.don_end, "%.2f" % hgt.don_mean, hgt.split_support, don_cov_test])
# Write only canidates to VCF file that passed the filter (boot_sens)
if (options.boot_sens > 0):
if (acc_cov_test < thresh) and (acc_cov_test > sens_acc * options.boot_num): continue
if (options.paired_reads):
if (cross_pair_test < thresh) or (don_pair_test < thresh): continue
if (don_cov_test < thresh): continue
# write filtered candidates to VCF file
writevcf.writerow([options.acceptor, hgt.acc_start, 'BND_'+str(hgt_vcf_counter)+'_1', hgt.acc_start_base, hgt.acc_start_base+'['+options.donor+':'+str(hgt.don_start)+'[', 'PASS', 'SVTYPE=BND;EVENT=HGT'+str(hgt_vcf_counter), '.', '1'])
writevcf.writerow([options.acceptor, hgt.acc_end, 'BND_'+str(hgt_vcf_counter)+'_2', hgt.acc_end_base, ']'+options.donor+':'+str(hgt.don_end)+']'+hgt.acc_end_base, 'PASS', 'SVTYPE=BND;EVENT=HGT'+str(hgt_vcf_counter), '.', '1'])
hgt_vcf_counter += 1
if (hgt_vcf_counter == 1):
print ('No canidates written to VCF, try to rerun with lower sampling sensitivity (--boot_sens)')
def grand(w):
'''
This is a grand loop that first calculates the firm's optimal decision rule, the HH's
consumption and labor choices, then aggregates individual choice to the overall economy, and finally
calculates the distance between aggregate labor demand and supply
'''
# operating profits, op
sizez = len(z_grid)
op = np.zeros((sizez, sizek))
for i in range(sizez):
for j in range(sizek):
op[i,j] = ((1 - a_l) * ((a_l / w) ** (a_l / (1 - a_l))) *
((kgrid[j] ** a_k) ** (1 / (1 - a_l))) * (z_grid[i] ** (1/(1 - a_l))))
# firm cash flow, e
e = np.zeros((sizez, sizek, sizek))
for i in range(sizez):
for j in range(sizek):
for k in range(sizek):
e[i, j, k] = (op[i,j] - kgrid[k] + ((1 - delta) * kgrid[j]) -
((psi / 2) * ((kgrid[k] - ((1 - delta) * kgrid[j])) ** 2)
/ kgrid[j]))
# value function iteration
VFtol = 1e-6
VFdist = 7.0
VFmaxiter = 3000
V = np.zeros((sizez, sizek))
Vmat = np.zeros((sizez, sizek, sizek))
Vstore = np.zeros((sizez, sizek, VFmaxiter))
VFiter = 1
start_time = time.clock()
while VFdist > VFtol and VFiter < VFmaxiter:
TV = V
Vmat = VFI_loop(V, e, betafirm, sizez, sizek, Vmat, pi)
Vstore[:, :, VFiter] = V.reshape(sizez, sizek,)
V = Vmat.max(axis=2)
PF = np.argmax(Vmat, axis=2)
Vstore[:,:, i] = V
VFdist = (np.absolute(V - TV)).max()
VFiter += 1
VFI_time = time.clock() - start_time
if VFiter < VFmaxiter:
print('Value function converged after this many iterations:', VFiter)
else:
print('Value function did not converge')
print('VFI took ', VFI_time, ' seconds to solve')
VF = V
# optimal capital stock k'
optK = kgrid[PF]
# stationary distribution
Gamma = np.ones((sizez, sizek)) * (1 / (sizek * sizez))
SDtol = 1e-12
SDdist = 7
SDiter = 0
SDmaxiter = 1000
while SDdist > SDtol and SDmaxiter > SDiter:
HGamma = SD_loop(PF, pi, Gamma, sizez, sizek)
SDdist = (np.absolute(HGamma - Gamma)).max()
Gamma = HGamma
SDiter += 1
if SDiter < SDmaxiter:
print('Stationary distribution converged after this many iterations: ',SDiter)
else:
print('Stationary distribution did not converge')
# optimal investment for an individual firm
opti = optK - (1 - delta) * kgrid
# labor demand for an individual firm
ld = np.zeros((sizez, sizek))
for i in range(sizez):
for j in range(sizek):
ld[i,j] = (((a_l / w) ** (1 / (1 - a_l))) *
((kgrid[j] ** a_k) ** (1 / (1 - a_l))) * (z_grid[i] ** (1/(1 - a_l))))
# adjustment costs
adj_cost = psi / 2 * np.multiply((opti)**2, 1 / kgrid)
# output per firm
y = (np.multiply(np.multiply((ld) ** a_l, kgrid ** a_k), np.transpose([z_grid])))
# aggregate labor demand
LD = np.multiply(ld, Gamma).sum()
# aggregate investment
I = np.multiply(opti, Gamma).sum()
# aggregate adjustment costs
ADJ = np.multiply(adj_cost, Gamma).sum()
# aggregate output
Y = np.multiply(y, Gamma).sum()
# aggregate consumption
C = Y - I - ADJ
# aggregate labor supply
LS = w / (h * C)
# distance between aggregate labor demand and aggregate labor supply
dist = abs(LD - LS)
print('|Ld-Ls|:', dist)
print('wage:', w)
return dist