def process_references():
if not DocSettings.args.quiet:
print("Processing References...")
roots = DocState.roots
i = 0
for i in xrange(0, len(roots)):
progressbar(i + 1, len(roots))
process_references_root(roots[i])
def build_compound_output():
if not DocSettings.args.quiet:
print("Building Output...")
pages = DocState.pages
i = 0
for page in pages:
progressbar(i + 1, len(pages))
generate_compound_doc(page)
i += 1
def process_compounds():
if not DocSettings.args.quiet:
print("Processing Compounds...")
compounds = DocState.input_xml
i = 0
for compound in compounds:
progressbar(i + 1, len(compounds))
gather_compound_doc(compound)
i += 1
def test_id_ref(path):
dom = ET.parse(path)
root = dom.getroot()
refs = root.findall(".//*[@refid]")
for i in range(1, len(refs)):
ref = refs[i]
progressbar(i + 1, len(refs))
#id = ref.get("refid")
refobj = ref.get("ref")
assert refobj
def progressbars(f=True, next=None, name=None):
if isinstance(f, _progressbar_wrapper_sum):
return f
if callable(f):
next = f
f = False
if f in [None, False]:
return _progressbar_wrapper_sum(next=next, name=name)
else:
if f is True:
return _progressbar_wrapper_sum(bar=progressbar(), next=next, name=name)
elif isinstance(f, six.string_types):
return _progressbar_wrapper_sum(bar=progressbar(f), next=next, name=name)
else:
return _progressbar_wrapper_sum(next=next, name=name)
def progressbars(f=True, next=None, name=None):
if isinstance(f, _progressbar_wrapper_sum):
return f
if callable(f):
next = f
f = False
if f in [None, False]:
return _progressbar_wrapper_sum(next=next, name=name)
else:
if f is True:
return _progressbar_wrapper_sum(bar=progressbar(), next=next, name=name)
elif isinstance(f, six.string_types):
return _progressbar_wrapper_sum(bar=progressbar(f), next=next, name=name)
else:
return _progressbar_wrapper_sum(next=next, name=name)
def create_persons():
aux = []
for _ in progressbar(range(50), 'Persons'):
data = get_person()
obj = Person(**data)
aux.append(obj)
Person.objects.bulk_create(aux)
def progressbars(f=True, next=None, name=None):
if callable(f):
next = f
f = False
if f in [None, False]:
return _progressbar_wrapper_sum(next=next, name=name)
else:
if f is True:
return _progressbar_wrapper_sum(bar=progressbar(),
next=next,
name=name)
else:
return _progressbar_wrapper_sum(next=next, name=name)
def scan_input():
filenames = [f for f in listdir("xml") if isfile(join("xml", f))]
if not DocSettings.args.quiet:
print("Scanning input")
compounds = DocState.input_xml = []
roots = DocState.roots = []
for i in xrange(0, len(filenames)):
fname = filenames[i]
progressbar(i + 1, len(filenames))
try:
extension = splitext(fname)[1]
if extension != ".xml":
continue
dom = ET.parse(join("xml", fname))
root = dom.getroot()
roots.append(root)
compound = root.find("compounddef")
assert root is not None, "Root is None"
assert dom is not None, "Dom is None"
if compound is not None:
compounds.append(compound)
DocState.register_compound(compound)
except Exception as e:
print(fname)
raise e
def begin(self):
begin_time=time.clock()
im = Image.open(self.img_in).convert("RGB")
w, h = im.size
print("w:%d,h:%d" % (w,h))
print("r:%d-%d,g:%d-%d,b:%d-%d"
% (self.red_s,self.red_e,
self.green_s,self.green_e,
self.blue_s,self.blue_e))
for j in range(0,h):
progressbar(j+1, h)
for i in range(0,w):
r,g,b = im.getpixel((i,j))
if ( r >= self.red_s and r <= self.red_e
and g >= self.green_s and g <= self.green_e
and b >= self.blue_s and b <= self.blue_e):
#print ("get pix i:%d,j:%d rgb:%d %d %d"%(i,j,r,g,b) )
r = self.red_n
g = self.green_n
b = self.blue_n
im.putpixel((i,j), (r,g,b))
im.save(self.img_out)
end_time=time.clock()
print("done:%d"%(end_time - begin_time))
def ztosigma(vin,Z,zcroco):
#
#
# Do a vertical interpolation from z levels to sigma CROCO levels
#
#
######################################################################
#
#
#
[N,M,L]=np.shape(zcroco)
[Nz]=np.shape(Z)
#
# Find the grid position of the nearest vertical levels
#
i1=np.arange(0,L)
j1=np.arange(0,M)
[imat,jmat]=np.meshgrid(i1,j1)
VAR=np.reshape(vin,Nz*M*L)
vout=np.zeros((N,M,L))
#
# Loop on the sigma layers
#
for ks in progressbar(range(N),' Sigma layer : ', 40):
sigmalev=zcroco[ks,:,:]
thezlevs=np.zeros((M,L),dtype=int)-1
for kz in range(Nz):
thezlevs[np.where(sigmalev>Z[kz])]=thezlevs[np.where(sigmalev>Z[kz])]+1
pos= L*M*thezlevs + L*jmat + imat
z1=Z[thezlevs]
z2=Z[thezlevs+1]
v1=VAR[pos]
v2=VAR[pos+1]
vout[ks,:,:]=(((v1-v2)*sigmalev+v2*z1-v1*z2)/(z1-z2))
return vout
def progressbar_callable(name="processing", max_value=1):
bar = progressbar(name, max_value=max_value)
return _progressbar_wrapper(bar)
])
return predict, train
predict, train = compile_model()
X, Y = get_dataset()
# Dataset error checking:
if np.shape(X)[0] == np.shape(Y)[0]:
D = np.shape(X)[0]
else:
print "ERROR: DATASET IS NOT VALID! Number of examples does not match number of labels."
# initialize progressbar class
progress = progressbar(iterations)
cost_list = []
for i in range(iterations):
cost = train(X,Y)
cost_list.append(cost)
progress.show(i)
# Plot the cost
plt.plot(cost_list)
pylab.show()
predictions = predict(X)
for x in range(np.shape(Y)[0]):
print "Actual value: ", Y[x], "| predicted value: ", predictions[x]
sniff_thread = threading.Thread(target=sniffer)
sniff_thread.start()
PACKETS_TO_SEND = 10000
# Send packets with some delay. Error if there are more than ERROR_THRESHOLD consecutive errors.
# No delay would be a better simulation, but it seems that there are race conditions that need
# to be worked out (or not, if they don't actually affect the soundness of the data).
widgets = [
RotatingMarker(),
Percentage(),
' ', SimpleProgress(format='(%s)' % SimpleProgress.DEFAULT_FORMAT,),
' ', Bar(marker='=', left='[', right=']'),
' ', Timer(),
' ', AdaptiveETA()
]
for i in progressbar(range(PACKETS_TO_SEND), marker='=', widgets=widgets):
if not run:
print('\nEncountered an error. Check log file for details. Exiting.', file=sys.stderr)
break
send_rand()
if not NODELAY_TEST:
time.sleep(0.001 * DELAY_MSEC)
print(errs)
if run:
print('\nTest completed successfully. Exiting.', file=sys.stderr)
sys.exit(0)
def interp3d_uv(nc,tndx_glo,Nzgoodmin,depth,z_rho,cosa,sina,\
iminU,imaxU,jminU,jmaxU,LonU,LatU,coefU,elemU,\
iminV,imaxV,jminV,jmaxV,LonV,LatV,coefV,elemV):
#
# Do a full interpolation of a 3d variable from GLORYS to a CROCO sigma grid
#
# 1 - Horizontal interpolation on each GLORYS levels
# 2 - Vertical Interpolation from z to CROCO sigma levels
#
#
######################################################################
#
#
#
[N,M,L]=np.shape(z_rho)
[Nz]=np.shape(depth)
comp_horzinterp = 1 # if 1 compute horizontal interpolations - 0 use saved matrices (for debugging)
if comp_horzinterp==1:
print('Horizontal interpolation of u and v over z levels')
u3d=np.zeros((Nz,M,L-1))
v3d=np.zeros((Nz,M-1,L))
kgood=-1
for k in progressbar(range(Nz),' uv : ', 40):
(u2d,Nzgood_u) = interp_tracers(nc,'u',tndx_glo,k,iminU,imaxU,jminU,jmaxU,LonU,LatU,coefU,elemU)
(v2d,Nzgood_v) = interp_tracers(nc,'v',tndx_glo,k,iminV,imaxV,jminV,jmaxV,LonV,LatV,coefV,elemV)
Nzgood=np.min((Nzgood_u,Nzgood_v))
if Nzgood>Nzgoodmin:
kgood=kgood+1
#
# Rotation and put to u-points and v-points
#
u3d[kgood,:,:]=rho2u_2d(u2d*cosa+v2d*sina)
v3d[kgood,:,:]=rho2v_2d(v2d*cosa-u2d*sina)
u3d=u3d[0:kgood,:,:]
v3d=v3d[0:kgood,:,:]
depth=depth[0:kgood]
np.savez('u3d.npz',u3d=u3d,v3d=v3d,depth=depth)
else:
print('Load matrices...')
data=np.load('u3d.npz')
u3d = data['u3d']
v3d = data['v3d']
depth = data['depth']
[Nz]=np.shape(depth)
Z=-depth
#
#----------------------------------------------------
# Vertical interpolation
#----------------------------------------------------
#
print('Vertical interpolation')
#
# Add a layer below the bottom and above the surface to avoid vertical extrapolations
# and flip the matrices upside down (Z[Nz]=surface)
#
Z=np.flipud(np.concatenate(([100.],Z,[-10000.])))
[Nz]=np.shape(Z)
u3d=np.flipud(vgrd.add2layers(u3d))
v3d=np.flipud(vgrd.add2layers(v3d))
#
# Do the vertical interpolations
#
uout=vgrd.ztosigma(u3d,Z,rho2u_3d(z_rho))
vout=vgrd.ztosigma(v3d,Z,rho2v_3d(z_rho))
return uout,vout
'pitch_high': args.pitch_high,
'save_blend': args.save_blend,
'use_90turn': args.use_90turn
}
# give parameters to each job
jobs = [job.copy() for i in range(args.num_sessions)]
for i, job in enumerate(jobs):
job['job_id'] = i
job['main_model'] = main_models[i % len(main_models)]
job['occl_models'] = sample(occl_models_pool, args.num_occluding)
logging.debug(job['occl_models'])
dataset_writer = DatasetWriter(args.out_db_file, overwrite=True)
# workhorse
progressbar = progressbar.ProgressBar(max_value=len(jobs))
if args.mode == 'SEQUENTIAL':
for job in progressbar(jobs):
patch_entries = run_patches_job(job)
write_results(dataset_writer, patch_entries, args.use_90turn)
elif args.mode == 'PARALLEL':
pool = multiprocessing.Pool(processes=5)
logging.info('the pool has %d workers' % pool._processes)
for patch_entries in progressbar(pool.imap(run_patches_job, jobs)):
write_results(dataset_writer, patch_entries, args.use_90turn)
pool.close()
pool.join()
dataset_writer.close()
def get_tri_coef(X, Y, newX, newY, verbose=0):
"""
Inputs:
origin lon and lat 2d arrays (X,Y)
child lon and lat 2d arrays (newX,newY)
Ouputs:
elem - pointers to 2d gridded data (at lonp,latp locations) from
which the interpolation is computed (3 for each child point)
coef - linear interpolation coefficients
Use:
To subsequently interpolate data from Fp to Fc, the following
will work: Fc = sum(coef.*Fp(elem),3); This line should come in place of all
griddata calls. Since it avoids repeated triangulations and tsearches (that are done
with every call to griddata) it should be much faster.
"""
Xp = np.array([X.ravel(), Y.ravel()]).T
Xc = np.array([newX.ravel(), newY.ravel()]).T
#Compute Delaunay triangulation
if verbose==1: tstart = tm.time()
tri = Delaunay(Xp)
if verbose==1: print('Delaunay Triangulation', tm.time()-tstart)
#Compute enclosing simplex and barycentric coordinate (similar to tsearchn in MATLAB)
npts = Xc.shape[0]
p = np.zeros((npts,3))
points = tri.points[tri.vertices[tri.find_simplex(Xc)]]
if verbose==1: tstart = tm.time()
for i in progressbar(range(npts),' Get_tri_coef: ', 40):
if verbose==1: print(np.float(i)/npts)
if tri.find_simplex(Xc[i])==-1: #Point outside triangulation
p[i,:] = p[i,:] * np.nan
else:
if verbose==1: tstart = tm.time()
A = np.append(np.ones((3,1)),points[i] ,axis=1)
if verbose==1: print('append A', tm.time()-tstart)
if verbose==1: tstart = tm.time()
B = np.append(1., Xc[i])
if verbose==1: print('append B', tm.time()-tstart)
if verbose==1: tstart = tm.time()
p[i,:] = np.linalg.lstsq(A.T,B.T)[0]
if verbose==1: print('solve', tm.time()-tstart)
if verbose==1: print('Coef. computation 1', tm.time()-tstart)
if verbose==1: tstart = tm.time()
elem = np.reshape(tri.vertices[tri.find_simplex(Xc)],(newX.shape[0],newY.shape[1],3))
coef = np.reshape(p,(newX.shape[0],newY.shape[1],3))
if verbose==1: print('Coef. computation 2', tm.time()-tstart)
return(elem,coef)
def interp3d(nc,vname,tndx_glo,Nzgoodmin,depth,z_rho,imin,imax,jmin,jmax,Lon,Lat,coef,elem):
#
# Do a full interpolation of a 3d variable from GLORYS to a CROCO sigma grid
#
# 1 - Horizontal interpolation on each GLORYS levels
# 2 - Vertical Interpolation from z to CROCO sigma levels
#
#
######################################################################
#
#
#
[N,M,L]=np.shape(z_rho)
[Nz]=np.shape(depth)
comp_horzinterp = 1 # if 1 compute horizontal interpolations - 0 use saved matrices (for debugging)
if comp_horzinterp==1:
print('Horizontal interpolation over z levels')
t3d=np.zeros((Nz,M,L))
kgood=-1
for k in progressbar(range(Nz),vname+': ', 40):
(t2d,Nzgood) = interp_tracers(nc,vname,tndx_glo,k,imin,imax,jmin,jmax,Lon,Lat,coef,elem)
if Nzgood>Nzgoodmin:
kgood=kgood+1
t3d[kgood,:,:]=t2d
t3d=t3d[0:kgood,:,:]
depth=depth[0:kgood]
np.savez('t3d.npz',t3d=t3d,depth=depth)
else:
print('Load matrix...')
data=np.load('t3d.npz')
t3d = data['t3d']
depth = data['depth']
[Nz]=np.shape(depth)
Z=-depth
#
#----------------------------------------------------
# Vertical interpolation
#----------------------------------------------------
#
print('Vertical interpolation')
#
# Add a layer below the bottom and above the surface to avoid vertical extrapolations
# and flip the matrices upside down (Z[Nz]=surface)
#
Z=np.flipud(np.concatenate(([100.],Z,[-10000.])))
[Nz]=np.shape(Z)
t3d=np.flipud(vgrd.add2layers(t3d))
#
# Do the vertical interpolations
#
vout=vgrd.ztosigma(t3d,Z,z_rho)
return vout
def main():
browser.get(url)
wait = 0
print('scrolling...')
lenOfPage = browser.execute_script(
"window.scrollTo(0, document.body.scrollHeight);var lenOfPage=document.body.scrollHeight;return lenOfPage;"
)
scroll = True
while scroll == True:
lastCount = lenOfPage
time.sleep(4)
wait += 1
lenOfPage = browser.execute_script(
"window.scrollTo(0, document.body.scrollHeight);var lenOfPage=document.body.scrollHeight;return lenOfPage;"
)
browser.page_source
soup = Soup(browser.page_source, "html.parser")
# get all browser post url
if wait > 10:
for i in soup.find_all('a', href=True):
if i['href'].startswith('/p/'):
print('Link found : {0}'.format(i['href']))
if i['href'] not in media_url:
media_url.append(IGurl + i['href'])
if lastCount == lenOfPage:
scroll = False
if wait == 15:
scroll = False
print('scrolling page :' + str(wait))
# browser.execute_script("window.scrollTo(0,4000);")
# time.sleep(SCROLL_PAUSE_TIME)
# wait +=1
# print('scrolling page :' + str(wait))
# if wait == 40:
# break
print('scrolling finish...')
#獲得每個貼文的img
count = 0
for i in media_url:
browser.get(i)
browser.page_source
soup = Soup(browser.page_source, "html.parser")
time.sleep(2)
try:
WebDriverWait(browser, 3).until(
EC.presence_of_element_located((By.CLASS_NAME, '_6CZji')))
button = browser.find_elements_by_class_name('_6CZji')[0]
except:
button = None
# 只有1張
if button == None:
soup = Soup(browser.page_source, "html.parser")
img_frame = soup.find(class_="KL4Bh")
try:
new_img = img_frame.img.get('src')
if (new_img != None):
finalall_url.append(new_img)
count += 1
print(count)
except:
pass
# 進入post之後，每一秒按一次直到沒有下一頁按鈕為止
# 並將得到的URL儲存在finalall_url之中
while button != None:
soup = Soup(browser.page_source, "html.parser")
time.sleep(1)
try:
img_frame = soup.find_all(class_="Ckrof")
for i in img_frame:
try:
new_img = i.img.get('src')
if (new_img != None) & (new_img not in finalall_url):
finalall_url.append(new_img)
count += 1
print(count)
except:
pass
button.click()
except:
break
# 儲存檔案
if not os.path.exists("marina"):
os.mkdir("marina")
num = 402
for i in finalall_url:
image = requests.get(i)
with open("marina\\" + "marinaImg" + str(num) + ".jpg",
"wb") as file: # 開啟資料夾及命名圖片檔
file.write(image.content)
num += 1
progressbar(num, count + num)
time.sleep(0.25)
# pbar.finish()
print('\n照片已儲存...')
from genFnClass import createCsharpFile
import sys
from os import listdir, makedirs, path
from os.path import isfile, join
from progressbar import *
# python ./genFnClasses.py $TES3MP_PATH/apps/openmw-mp/Script/Functions $OUT/lib
if __name__ == '__main__':
cppfilesDir = sys.argv[1]
outdir = sys.argv[2]
hppfiles = [
f for f in listdir(cppfilesDir)
if isfile(join(cppfilesDir, f)) and f.endswith('.hpp')
]
if not path.exists(outdir):
makedirs(outdir)
for i in progressbar(range(len(hppfiles))):
cppfile = join(cppfilesDir, hppfiles[i])
outfile = join(outdir, hppfiles[i].split('.')[0] + '.cs')
if hppfiles[i] == "Timer.hpp" or hppfiles[i] == "Public.hpp":
continue
createCsharpFile("TES3MPSharp", cppfile, outfile)
import sys
from os.path import join
from os import walk, makedirs
import progressbar
from shutil import copyfile
srcDir = sys.argv[1]
dstDir = sys.argv[2]
dirs = [join(root, dir) for root, dirs, files in walk(srcDir) for dir in dirs]
for dir in dirs:
newDir = join(dstDir, dir.replace(srcDir, ""))
makedirs(newDir, exist_ok=True)
txtFiles = [
join(root, file) for root, dirs, files in walk(srcDir) for file in files
if file.endswith(".txt")
]
#print(dirs)
for txtFile in progressbar(txtFiles):
dstTxtFile = join(dstDir, txtFile.replace(srcDir, ""))
copyfile(txtFile, dstTxtFile)
def example32():
for i in progressbar(range(10), redirect_stdout=True):
print()
print('Some text', i)
time.sleep(0.1)
def progressbar_callable(title="processing", max_value=1):
bar = progressbar(title=title, max_value=max_value)
return _progressbar_wrapper(bar)
