def back(self, xlist, midrslt, outrslt, anslist, w_one, w_two, srclass,
dropclass, normclass_mid, normclass_out):
#ylist.shape = [10, 100]
#anslist.shape = [100, 10]
#softmax + cross_en back-propagation
en_ak = (outrslt - anslist.T) / 100
#en_ak.shape = [10,100]
#batch-norm back-propagation
(en_midrslt, en_gamma_out, en_beta_out) = normclass_out.backward(en_ak)
#out back-propagation
(en_midrslt, en_w_two, en_b_two) = \
Diff.diff(en_ak, w_two, midrslt)
#dropout back-propagation
pre_drop = dropclass.backward(en_midrslt)
pre_sig = srclass.backward(pre_drop)
#batch-norm back-propagation
(en_summid, en_gamma_mid,
en_beta_mid) = normclass_mid.backward(pre_sig)
#middle back-propagation
(en_x, en_w_one, en_b_one) = \
Diff.diff(en_summid, w_one, xlist)
return (en_x, en_w_one, en_w_two, en_b_one, en_b_two, en_gamma_mid,
en_beta_mid, en_gamma_out, en_beta_out)
def diff(self, **kwargs):
"""
Compile a difference list of files between src and dst directories
"""
if self.__validate():
# TODO: filter kwargs
self.diffstngs.update(kwargs)
self.origdiff = Diff(self.src, self.dst, **self.diffstngs)
self.trimdiff = self.origdiff.copy()
self.__hasrundiff = True
self.__diffcurrent = True
def mem():
form = ReciteForm()
if form.validate_on_submit():
b = Bible()
actual = b.get_verse(request.args.get("book").lower(), request.args.get("chapter"), request.args.get("verse"), "esv")
typed = form.text.data
d = Diff(actual, typed)
html = d.generate_html()
return html
return render_template('mem.html', form=form, book=request.args.get("book"), chapter=request.args.get("chapter"), verse=request.args.get("verse"))
def main(self, file1path, file2path):
xlsPre = xlrd.open_workbook(file1path.__str__())
xlsSuf = xlrd.open_workbook(file2path.__str__())
#存储tabel转换为list后的结果，按照行的格式，形如[(sheetname,[sheetPre],[sheetSuf]),]
sheets = []
#存储diff后的结果，形如[(表名，改动情况，行的diff，列的diff)]，-1：删除，0：未改动，1：新增，2：改动
#增删改的结构为，[(表的增删改，行的改动，列的改动，单元格的改动）]
#行列的改动的结构为，[（增删，行号）,]
#单元格的改动的结构为，[(行号，列号)，]
diffrst = []
sheetnamesPre = xlsPre.sheet_names()
sheetnamesSuf = xlsSuf.sheet_names()
# print u"There are %s sheets."%(len(sheetnamesSuf))
diffmain = Diff()
for sheetname in sheetnamesPre:
if sheetname in sheetnamesSuf:
# print u"Start processing: ",sheetname
rowsPre = self.sheet2ListByRow(xlsPre.sheet_by_name(sheetname))
rowsSuf = self.sheet2ListByRow(xlsSuf.sheet_by_name(sheetname))
sheets.append((sheetname, rowsPre, rowsSuf))
if rowsPre == rowsSuf:
diffrst.append((sheetname, 0, rowsPre, rowsSuf))
else:
# print u"old sheet has %s rows. " % len(rowsPre), u"new sheet has %s rows" % len(rowsSuf)
diffrows = diffmain.diff_main(rowsPre, rowsSuf, True)
# print u"finish rows diff"
colsPre = self.sheet2ListByCol(
xlsPre.sheet_by_name(sheetname))
colsSuf = self.sheet2ListByCol(
xlsSuf.sheet_by_name(sheetname))
# print u"old sheet has %s columns. " % len(colsSuf), u"new sheet has %s columns" % len(colsSuf)
diffcols = diffmain.diff_main(colsPre, colsSuf, True)
# print u"finish columns diff"
#diffmeg=self.diffMerge(diffrows,diffcols)
diffrst.append((sheetname, 2, diffrows, diffcols))
else:
# print u"Start processing: ", sheetname
rowsPre = self.sheet2ListByRow(xlsPre.sheet_by_name(sheetname))
sheets.append((sheetname, rowsPre, []))
diffrst.append((sheetname, -1, rowsPre, []))
for sheetname in sheetnamesSuf:
if sheetname not in sheetnamesPre:
# print u"Start processing: ", sheetname
rowsPre = self.sheet2ListByRow(xlsSuf.sheet_by_name(sheetname))
sheets.append((sheetname, [], rowsPre))
diffrst.append((sheetname, 1, [], rowsPre))
diffmerge = self.diffMerge(diffrst)
tables = self.makeTable(sheets, diffmerge)
return tables
def test_TC5(self):
data = {
'a': {
'a1': 1,
'a2': 2
},
'b': {
'b1': {
'b11': 1,
'b12': 2
}
},
'c': {
'd': 4
}
}
data_before = copy.deepcopy(data)
data['a']['a2'] = 7
del (data['b'])
try_result = {'a': {'a2': 7}, 'b': {'b1': {'b11': 1, 'b12': 2}}}
result = Diff().diff_two_dict(data_before, data, {})
self.assertEqual(try_result, result)
def setUp(self):
self.old_dict = {"dict1":{"dict2": {"x":"A","z":"d"}}}
self.new_dict = {"dict1":{"dict2": {"x":"C","y":"B"}}}
self.old_flat = {"a":"a", "z":"z"}
self.new_flat = {"a":"f", "b":"b"}
self.oldsame = {"dict1":{"dict2": {"x":"A","z":"d"}}}
self.newsame = {"dict1":{"dict2": {"x":"A","z":"d"}}}
self.old = {"result":{
"first name":"bob",
"last name":"segar",
"phone":{"home":"615-555-4433",
"mobile":"615-555-4444"},
"title":["programmer", "dev-ops"],
"salary":{"starting":80000,
"20150303":90000}}}
self.new = {"result":{
"first name":"bob",
"last name":"segar",
"phone":{"mobile":"615-555-4444"},
"title":["python developer", "dev-ops"],
"salary":{"starting":80000,
"20150303":90000},
"clearance":"level Z"}}
self.d = Diff()
def faireDiff(self, agenda1, agenda2):
"""
Lance une comparaison entre I{agenda1} et I{agenda2}.
@param self: L'argument implicite.
@type agenda1: L{Agenda}
@param agenda1: le premier agenda à comparer.
@type agenda2: L{Agenda}
@param agenda2: le second agenda à comparer.
@rtype: L{Diff}
@return: Renvoi un objet Diff contenant les différences.
"""
if agenda1 is None or agenda2 is None:
raise ValueError()
#if
monDiff = Diff(agenda1, agenda2)
monDiff.comparer()
return monDiff
def test_TC12(self):
data = {'a': 1, 'b': 2}
data_before = copy.deepcopy(data)
try_result = {}
result = Diff().diff_two_dict(data_before, data, {})
self.assertEqual(try_result, result)
def test_TC13(self):
data = {}
data_before = copy.deepcopy(data)
data['a'] = {'a1': {'a2': 3}}
try_result = {'a': {'a1': {'a2': 3}}}
result = Diff().diff_two_dict(data_before, data, {})
self.assertEqual(try_result, result)
def test_TC4(self):
data = {'a': 1, 'b': {'b1': 2}, 'c': {'c1': {'c2': 3, 'c3': 5}}}
data_before = copy.deepcopy(data)
data['c']['c1']['c4'] = {'c5': 3}
try_result = {'c': {'c1': {'c4': {'c5': 3}}}}
result = Diff().diff_two_dict(data_before, data, {})
self.assertEqual(try_result, result)
def diff(config):
"""
This argument will run DYC on DIFF patch only
"""
diff = Diff(config.plain)
uncommitted = diff.uncommitted
paths = [idx.get('path') for idx in uncommitted]
if len(uncommitted):
dyc = DYC(config.plain)
dyc.prepare(files=paths)
dyc.process_methods(diff_only=True, changes=uncommitted)
def diff(self, **kwargs):
"""
Compile a difference list of files between src and dst directories
"""
if self.__validate():
# TODO: filter kwargs
self.diffstngs.update(kwargs)
self.origdiff = Diff(self.src, self.dst, **self.diffstngs)
self.trimdiff = self.origdiff.copy()
self.__hasrundiff = True
self.__diffcurrent = True
def test_TC3(self):
data = {'a': 1, 'b': 2, 'c': {'c1': 3, 'c2': 5}}
data_before = copy.deepcopy(data)
data['a'] = 5
data['c']['c2'] = 9
try_result = {'a': 5, 'c': {'c2': 9}}
result = Diff().diff_two_dict(data_before, data, {})
self.assertEqual(try_result, result)
def test_TC1(self):
'''
Text example
'''
data = {'a': {'a1': 1, 'a2': 2}, 'b': {'b1': {'b11': 1, 'b12': 2}}}
data_before = copy.deepcopy(data)
data['a']['a1'] = 3
data['b']['b1']['b11'] = 5
try_result = {'a': {'a1': 3}, 'b': {'b1': {'b11': 5}}}
result = Diff().diff_two_dict(data_before, data, {})
self.assertEqual(try_result, result)
def parseDiff(self, diffLines):
diff = Diff(diffLines[0])
expectedOpType = self.OP_TYPE_DELETE
if self.isDiffOpTypeAppend(diff.getOpType()):
expectedOpType = self.OP_TYPE_APPEND
for lineNo in range(1, len(diffLines)):
line = diffLines[lineNo]
if self.isDiffLineDelete(
line) and expectedOpType == self.OP_TYPE_DELETE:
diff.addDelete(line)
elif self.isDiffLineBreak(
line) and diff.getOpType() == self.OP_TYPE_CHANGE:
expectedOpType = self.OP_TYPE_APPEND
elif self.isDiffLineAppend(
line) and expectedOpType == self.OP_TYPE_APPEND:
diff.addAppend(line)
else:
#TODO: Implement custom errors
raise ValueError
return diff
def test_TC6(self):
data = {
'a': {
'a1': 1,
'a2': 3
},
'b': {
'b1': 5,
'b2': {
'b3': {
'b4': 5
}
}
}
}
data_before = copy.deepcopy(data)
data['a']['a2'] = 7
del (data['b']['b2'])
try_result = {'a': {'a2': 7}, 'b': {'b2': {'b3': {'b4': 5}}}}
result = Diff().diff_two_dict(data_before, data, {})
self.assertEqual(try_result, result)
def backward(self, en_y):
pre_sig = self.srclass.backward(en_y)
#print(pre_sig.shape)
(en_x, en_w, en_b) = Diff.diff(pre_sig.reshape(20, -1), self.w,
self.x) #reshape <- ayashii
return (en_w, en_b)
import numpy as np
import pandas as pd
from diff import Diff
if __name__ == '__main__':
df1 = pd.DataFrame({
'x': [None, 2, np.NaN, 3, np.Inf],
'y': ['', 2, np.NaN, None, '1']
})
df2 = pd.DataFrame({'x': [1, 2, 3, 4], 'y': ['a', 'b', 'c', 'd']})
diff = Diff()
diff(df1, df2)
def judge(self, sub):
lang = sub.lang
mem_policy = True if (jcnf.POLICY[lang] == 'ALL'
or jcnf.POLICY[lang] == 'MEM') else False
page_size = resource.getpagesize()
case_cnt = min(len(sub.case_lim), sub.case_cnt)
sub.case_done = 0
case_id = 0
for case_id in range(case_cnt):
sub.case_res.append({
'res': jcnf.JUDGE_RES['init'],
'time': 0,
'mem': 0,
})
exec_cmd = jcnf.EXEC_CMD[lang]
exec_param = jcnf.EXEC_PARAM[lang]
if lang == 'java':
mem_lim = 0
for case_id in xrange(case_cnt):
mem_lim = max(mem_lim, sub.case_lim[case_id]['mem'])
exec_param.insert(1, '-Xmx' + str(mem_lim / 1024 + 1) + 'm')
for case_id in xrange(case_cnt):
lim = sub.case_lim[case_id]
res = sub.case_res[case_id]
exec_pid = os.fork()
if exec_pid == 0:
for rk, rv in jcnf.EXEC_RLIM[lang].items():
resource.setrlimit(rk, rv)
try:
exec_i = open(jcnf.getCasePathI(sub.pid, case_id), 'r')
exec_o = open(jcnf.getExecPathO(), 'w')
except:
#sys.stderr.write('file read error')
raise Exception('cannot handle input or output file')
lim['time'] = max(lim['time'], jcnf.EXEC_MIN_TL[lang])
lim['time'] = int(lim['time'] * jcnf.EXEC_TL_RATIO[lang])
if lang == 'java':
lim['time'] *= 3
rlimt = (lim['time'] - 1) // 1000 + 2
resource.setrlimit(resource.RLIMIT_CPU, (rlimt, rlimt))
rlimm = jcnf.EXEC_MAX_MEM
if mem_policy: # java uses virtual machine
resource.setrlimit(resource.RLIMIT_AS, (rlimm, rlimm))
os.dup2(exec_i.fileno(), 0)
os.dup2(exec_o.fileno(), 1)
#TOTO: ptrace
pt_ret = cptrace.ptrace(cptrace.PTRACE_TRACEME, 0)
if (pt_ret == -1):
#sys.stderr.write('warning: ptrace error')
raise Exception('child process cannot be ptraced')
if exec_i:
exec_i.close()
if exec_o:
exec_o.close()
os._exit(1)
os.execvp(exec_cmd, exec_param)
sys.stderr.write('warning: something wrong')
if exec_i:
exec_i.close()
if exec_o:
exec_o.close()
os._exit(1)
else:
stat_info_file = str(exec_pid).join(['/proc/', '/statm'])
res['mem'] = 0
res['time'] = 0
t_prev = jclk()
eax_prev = 142857
insyscall = False
def killProc():
try:
os.kill(exec_pid, signal.SIGKILL)
except OSError:
pass
else:
res['res'] = jcnf.JUDGE_RES['re']
threading.Timer((lim['time'] - 1) // 1000 + 10,
killProc).start()
try:
while res['res'] == jcnf.JUDGE_RES['init']:
#while True:
exec_status = os.wait4(exec_pid, 0)
if res['res'] != jcnf.JUDGE_RES['init']:
break
t_now = jclk()
res['time'] += t_now - t_prev
t_prev = t_now
DEBUG_CNT = 0
#res['mem'] = exec_status[2].ru_minflt-360
res['mem'] = exec_status[2].ru_maxrss
if os.WIFSIGNALED(exec_status[1]):
#strange exited or tle?
if res['time'] * 1000 > lim['time']:
res['res'] = jcnf.JUDGE_RES['tle']
break
elif os.WIFEXITED(exec_status[1]):
#normally exited , ok
break
elif os.WIFSTOPPED(exec_status[1]):
#sigtrap by ptra ce
exec_sig = os.WSTOPSIG(exec_status[1])
if exec_sig != signal.SIGTRAP:
res['res'] = jcnf.JUDGE_RES['re']
#print exec_status[0], exec_status[1], 'hehe', exec_sig
cptrace.ptrace(cptrace.PTRACE_KILL, exec_pid)
#strange exited?
break
eax_now = cptrace.ptrace(
cptrace.PTRACE_PEEKUSER, exec_pid,
4 * cptrace.ORIG_EAX
) #when used in 64bit system, it should be 8*xxxx, so it is recommended to make it a const in conf
if jcnf.POLICY[lang] == 'ALL':
if jcnf.SYSCALL[eax_now][
0] == 0: #prohibited syscall
res['res'] = jcnf.JUDGE_RES['re']
cptrace.ptrace(
cptrace.PTRACE_KILL, exec_pid
) #deprecated! should be implemented in another way
break
else:
#TODO extend implementation
pass
if eax_now != eax_prev and eax_now != -1:
insyscall = False
if eax_now != -1:
if insyscall:
DEBUG_CNT += 1
#if eax_now==45 or eax_now==90 or eax_now==91:
try:
stat_info = open(stat_info_file, 'r')
mem_now = int(
stat_info.read().split(' ')[5]
) #automatically to long when exceed
#res['mem'] = max(res['mem'], mem_now)
stat_info.close()
except:
pass
insyscall = False
else:
insyscall = True
if mem_policy and res['mem'] > lim[
'mem']: #res['mem']*page_size>lim['mem']*1024:
res['res'] = jcnf.JUDGE_RES['mle']
cptrace.ptrace(
cptrace.PTRACE_KILL, exec_pid
) #deprecated! should be implemented in another way
break
if res['time'] * 1000 > lim['time']:
res['res'] = jcnf.JUDGE_RES['tle']
cptrace.ptrace(
cptrace.PTRACE_KILL, exec_pid
) #deprecated! should be implemented in another way
break
if eax_now != -1:
eax_prev = eax_now
t_prev = jclk()
else:
#sys.stderr.write('unknown status')
pass
#TODO: also check total time limit?
if res['res'] == jcnf.JUDGE_RES['tle']:
#TODO: write log
cptrace.ptrace(
cptrace.PTRACE_KILL, exec_pid
) #deprecated! should be implemented in another way
else:
cptrace.ptrace(cptrace.PTRACE_SYSCALL, exec_pid)
except:
pass
try:
os.wait()
os.kill(exec_pid, signal.SIGKILL)
except Exception, e:
if JDEBUG: print 'cannot kill', Exception, e
pass
res['mem'] = int(res['mem'])
res['time'] = int(res['time'] * 1000)
if res['res'] == jcnf.JUDGE_RES['init']:
if os.WIFSIGNALED(exec_status[1]):
res['res'] = jcnf.JUDGE_RES['re']
elif os.WIFSTOPPED(exec_status[1]) and os.WSTOPSIG(
exec_status[1]) != signal.SIGTRAP:
res['res'] = jcnf.JUDGE_RES['re']
if res['res'] == jcnf.JUDGE_RES['init']:
df = Diff()
res['res'] = df.diff(jcnf.getCasePathO(sub.pid, case_id),
jcnf.getExecPathO())
sub.case_done += 1
#sub.mem += res['mem']
sub.mem = max(sub.mem, res['mem'])
sub.time += res['time']
if res['res'] == jcnf.JUDGE_RES['init']:
res['res'] = jcnf.JUDGE_RES['se']
# Need to calculate the scores of all test data, and thus we cannot break out when judging
# if sub.block and res['res']!=jcnf.JUDGE_RES['ac']:
# break
t_prev = jclk()
sub.status = jcnf.SUB_STATUS['done']
def ColumnModelCal(self):
# Calculate angle between wind direction and canyon orientation (theta_S) [deg]
theta_S = (360+abs(self.theta_can-self.ForcWindDir))%90
# Road roughness
z0g = 0.05
# Roof roughness
z0r = 0.15
# gas constant dry air [J kg^-1 K^-1]
r = 287.04
rcp = r / self.Cp
# Define explicit and implicit parts of source and sink terms
srex_vx = numpy.zeros(self.nz) # Explicit part of x component of horizontal wind speed [m s^-2]
srim_vx = numpy.zeros(self.nz) # Implicit part of x component of horizontal wind speed [s^-1]
srex_vy = numpy.zeros(self.nz) # Explicit part of y component of horizontal wind speed [m s^-2]
srim_vy = numpy.zeros(self.nz) # Implicit part of y component of horizontal wind speed [s^-1]
srex_tke = numpy.zeros(self.nz) # Explicit part of turbulent kinetic energy [m^2 s^-3]
srim_tke = numpy.zeros(self.nz) # Implicit part of turbulent kinetic energy [s^-1]
srex_th = numpy.zeros(self.nz) # Explicit part of potential temperature [K s^-1]
srim_th = numpy.zeros(self.nz) # Implicit part of potential temperature [s^-1]
srex_qn = numpy.zeros(self.nz) # Explicit part of specific humidity [K s^-1] ?????
srim_qn = numpy.zeros(self.nz) # Implicit part of specific humidity [s^-1] ?????
srex_th_veg = numpy.zeros(self.nz) # Explicit part of potential temperature caused by vegetation
srex_qn_veg = numpy.zeros(self.nz) # Explicit part of specific humidity caused by vegetation
Tveg = numpy.zeros(self.nz)
# Apply boundary conditions at the top of the domain using vertical diffusion model(VDM) outputs
self.th[self.nz-1] = self.ForcTemp
self.qn[self.nz-1] = self.ForcHum
self.vx[0] = 0.001
self.vy[0] = 0.001
self.tke[0] = 0.00001
# Calculate bulk Richardson number (Ri_b):
# Ri_b = (g*H/((Uroof - Ustreet)^2+(Vroof - Vstreet)^2))*(Troof - Tstreet)/Tavg (equation 6, Aliabadi et al., 2018)
delU = ((self.vx[0]-self.vx[self.nz_u+1])**2+(self.vy[0]-self.vy[self.nz_u+1])**2)
# Denominator of the fraction must not be zero. So, a minimum value for denominator is considered
delU = max(delU,0.1)
#For calculation of Rib, option 1: surface temperature difference
#delT = self.RoofTemp-self.RoadTemp
#For calculation of Rib, option 2: air temperature difference
delT = self.th[self.nz_u+1]-self.th[1]
Ri_b = ((self.g*self.hmean)/delU)*(delT/numpy.mean(self.th[0:self.nz_u]))
# Calculate turbulent diffusion coefficient (Km) [m^2 s^-1]
TurbDiff = CdTurb(self.nz, self.Ck, self.tke, self.dlk,self.it,Ri_b,self.var_sens)
Km = TurbDiff.TurbCoeff()
# Road surface temperature [K]
ptg = self.RoadTemp
# Wall surface temperature [K]
ptw = self.WallTemp
# Roof surface temperature [K]
ptr = self.RoofTemp
# Call "BuildingCol" to calculate sink and source terms in momentum, temperature and turbulent kinetic energy (TKE)
# equations which are caused by building
BuildingCoef = BuildingCol(self.nz, self.dz, self.dt, self.vol, (1-self.VegCoverage), self.lambdap, self.lambdaf,
self.hmean, self.Ck, self.Cp, self.th0, self.vx, self.vy, self.th, self.Cdrag,
ptg,ptr, ptw, self.rho,self.nz_u, self.pb, self.ss,self.g,z0g,z0r,self.SensHt_HVAC,self.HVAC_street_frac,self.HVAC_atm_frac)
# Calculate shear production [m^2 s^-3] in TKE equation. (Term II of equation 5.2, Krayenhoff 2014, PhD thesis)
Shear_Source = Shear(self.nz, self.dz, self.vx, self.vy, Km)
sh = Shear_Source.ShearProd()
# Calculate buoyant production [m^2 s^-3] in TKE equation. (Term IX of equation 5.2, Krayenhoff 2014, PhD thesis)
Buoyancy_Source = Buoyancy(self.nz, self.dz, self.th, Km, self.th0, self.prandtl, self.g)
bu = Buoyancy_Source.BuoProd()
# Calculate dissipation (td) [s^-1] in TKE equation. (Term VI of equation 5.2, Krayenhoff 2014, PhD thesis)
# parameterization of dissipation is based on Nazarian's code. (https://github.com/nenazarian/MLUCM/blob/master/Column_Model/column_lkPro.f90)
td = numpy.zeros(self.nz)
for i in range(0, self.nz):
if self.dls[i] != 0:
td[i] = -self.Ceps*(math.sqrt(self.tke[i]))/self.dls[i]
else:
td[i] = 0
sh[i] = sh[i]*self.sf[i]
bu[i] = bu[i]*self.sf[i]
# Return sink and source terms caused by buildings
srex_vx_h, srex_vy_h, srex_tke_h, srex_th_h, srim_vx_v, srim_vy_v, srex_tke_v, srim_th_v, srex_th_v, sff,swf, ustarCol = BuildingCoef.BuildingDrag()
# Friction velocity (Aliabadi et al, 2018)
ustar = 0.07 * self.ForcWind + 0.12
# Calculate pressure gradient
C_dpdx = 5.4
dpdx = C_dpdx*self.rho[self.nz-1]*(ustar**2)*math.cos(math.radians(theta_S))/(self.dz*self.nz)
dpdy = C_dpdx*self.rho[self.nz-1]*(ustar**2)*math.sin(math.radians(theta_S))/(self.dz*self.nz)
# Latent heat of vaporization [J kg^-1]
latent = 2.45e+06
# Latent heat of vaporization [J mol^-1](Campbell and Norman,1998)
latent2 = 44100
# The average surface and boundary-layer conductance for humidity for the whole leaf
gvs = 0.330
# Set leaf dimension of trees
leaf_width = 0.05
leaf_dim = 0.72 * leaf_width
# Air pressure [Pa]
pr = 101300
# Total neighbourhood foliage clumping [non dimensional]
omega = 1
# Molar heat capacity [J mol^-1 K^-1](Campbell and Norman, 1998)
cp_mol = 29.3
# Drag coefficient for vegetation foliage
cdv = 0.2
omega_drag = 0.34
# Calculate source and sink terms caused by trees and then calculate total source and sink terms
for i in range(0, self.nz):
# source/sink terms of specific humidity
wind = numpy.sqrt(self.vx[i] ** 2 + self.vy[i] ** 2)
# Boundary-layer conductance for vapor (p. 101 Campbell and Norman, 1998)
gva = 1.4 * 0.147 * numpy.sqrt(wind / leaf_dim)
# Overall vapour conductance for leaves [mol m^-2 s^-1] (equation 14.2, Campbell and Norman, 1998):
gv = gvs * gva / (gvs + gva)
# Conductance for heat [mol m^-2 s^-1]
gHa = 1.4 * 0.135 * numpy.sqrt(wind / leaf_dim)
# Since a leaf has two sides in parallel, gHa should be multiplied by 2
gHa = gHa * 2
# Convert potential air temperature to real temperature [K]
# potential temperature = real temperature * (P0/P)^(R/cp)
tair = self.th[i] / (pr / 1.e+5) ** (-rcp)
# Convert absolute humidity to vapour pressure [Pa]
eair = self.qn[i] * pr / 0.622
# Saturation vapor pressure [Pa] (equation 7.5.2d, Stull 1988)
es = 611.2 * numpy.exp(17.67 * (tair - 273.16) / (tair - 29.66))
D = es - eair
desdT = 0.622 * latent * es / r / (tair) ** 2
s = desdT / pr
# Calculate terms in transport equations caused by trees. "wt" is term in temperature equation adn "wt_drag"
# is term in TKE and momentum equations. It is assumed there is no vegetation above average building height
if self.dz * i > max(self.h_LAD):
wt = 0 # [m^2 m^-3]
wt_drag = 0 # [m^2 m^-3]
else:
wt = self.f_LAD(self.dz * i) * omega * (1 - self.lambdap) / self.vol[i] # [m^2 m^-3]
wt_drag = self.f_LAD(self.dz * i) * omega_drag * (1. - self.lambdap) / self.vol[i] # [m^2 m^-3]
# Stefan-Boltzmann constant [W m^-2 K^-4]
sigma = 5.67e-8
gam = 6.66e-4
# Emissivity of leaves surface
emveg = 0.95
# Total fraction scattered by leaves: reflected & transmitted
albv_u = 0.5
fact = 1
# Total radiation absorbed by leaves [W m^-2]
Rabs = (1-albv_u)*self.S_t+self.L_t*emveg
gr = 4 * emveg * sigma * tair ** 3 / cp_mol
gr = gr * 2. * omega * fact
sides = 2. * omega * fact
gHr = gHa + gr
gamst = gam * gHr / gv
# Calculate temperature of vegetation [K]
tveg_tmp = tair+gamst/(s+gamst)*((Rabs-sides*emveg*sigma*(tair**4))/gHr/cp_mol-D/pr/gamst)
Tveg[i] = tveg_tmp
# Calculate terms in temperature and humidity equations caused by trees.
if self.dz * i > max(self.h_LAD):
srex_th_veg[i] = 0
srex_qn_veg[i] = 0
else:
srex_th_veg[i] = cp_mol*gHa*tveg_tmp*wt/self.Cp/self.rho[i]
srex_qn_veg[i] = (latent2*gv*(s*(tveg_tmp-tair)+es/pr))*wt/self.rho[i]/latent
# Calculate total explicit terms
# Explicit term in x momentum equation [m s^-2] = fluxes from horizontal surfaces + pressure gradient
# pressure gradient is zero, because boundary conditions are forced by vertical diffusion model
srex_vx[i] = srex_vx_h[i]+dpdx
# Explicit term in y momentum equation [m s^-2] = fluxes from horizontal surfaces + pressure gradient
# pressure gradient is zero, because boundary conditions are forced by vertical diffusion model
srex_vy[i] = srex_vy_h[i]+dpdy
# Explicit term in TKE equation [m^2 s^-3] = terms from urban horizontal surfaces [??????] +
# terms from walls [m^2 s^-3] + shear production [m^2 s^-3] + buoyant production [m^2 s^-3] +
# term caused by vegetation [m^2 s^-3]
srex_tke[i] = srex_tke_h[i] + srex_tke_v[i] + sh[i] + bu[i] + cdv*wind**3.*wt_drag
# Explicit term in temperature equation [K s^-1] = term from urban horizontal surfaces [K s^-1] +
# term from walls [K s^-1] + term caused by vegetation [K s^-1]
srex_th[i] = srex_th_h[i] + srex_th_v[i] + srex_th_veg[i] #+ 4*rho_abs*kbs*(1-self.lambdap)*self.L_abs/self.rho/self.Cp/self.vol[i]
# Explicit term in humidity equation [K s^-1] = term caused by latent heat from vegetation [K s^-1]
srex_qn[i] = srex_qn[i] + srex_qn_veg[i]
# Calculate total Implicit terms
# Implicit term in x momentum equation [s^-1] = term from walls [s^-1] - term caused by vegetation [s^-1]
srim_vx[i] = srim_vx_v[i]-cdv*wind*wt_drag
# Implicit term in y momentum equation [s^-1] = term from walls [s^-1] - term caused by vegetation [s^-1]
srim_vy[i] = srim_vy_v[i]-cdv*wind*wt_drag
# Implicit term in TKE equation [s^-1] = dissipation [s^-1] - term caused by vegetation [s^-1]
srim_tke[i] = td[i]-6.5*cdv*wind*wt_drag
# Implicit term in temperature equation [s^-1] = term from wall [s^-1] - term caused by vegetation [s^-1]
srim_th[i] = srim_th_v[i]-cp_mol*gHa*wt/self.Cp/self.rho[i]
# Implicit term in humidity equation [s^-1] = term caused by latent heat from vegetation [s^-1]
srim_qn[i] = srim_qn[i]-latent2*gv*(pr/0.622)/pr*wt/self.rho[i]/latent
# Solve transport equations
# Set type of boundary conditions (B.Cs):
# Neumann boundary condition (Flux): iz = 1
# Dirichlet boundary condition (Constant value): iz = 2
# Sol.Solver(B.C. at the bottom of domain)
Sol = Diff(self.nz, self.dt, self.sf, self.vol, self.dz, self.rho)
# Solve x component of momentum equation
A_vx = Sol.Solver21(2, 1, self.vx, srim_vx, srex_vx,Km)[0]
RHS_vx = Sol.Solver21(2, 1, self.vx, srim_vx, srex_vx,Km)[1]
Inv_vx = Invert(self.nz, A_vx, RHS_vx)
self.vx = Inv_vx.Output()
# Solve y component of momentum equation
A_vy = Sol.Solver21(2, 1, self.vy, srim_vy, srex_vy,Km)[0]
RHS_vy = Sol.Solver21(2, 1, self.vy, srim_vy, srex_vy,Km)[1]
Inv_vy = Invert(self.nz, A_vy, RHS_vy)
self.vy = Inv_vy.Output()
# Solve TKE equation
A_tke = Sol.Solver21(2, 1, self.tke, srim_tke, srex_tke,Km)[0]
RHS_tke = Sol.Solver21(2, 1, self.tke, srim_tke, srex_tke,Km)[1]
Inv_tke = Invert(self.nz, A_tke, RHS_tke)
self.tke = Inv_tke.Output()
# Solve temperature equation
A_th = Sol.Solver12(1, 2, self.th, srim_th, srex_th,Km/self.prandtl)[0]
RHS_th = Sol.Solver12(1, 2, self.th, srim_th, srex_th,Km/self.prandtl)[1]
Inv_th = Invert(self.nz, A_th, RHS_th)
self.th = Inv_th.Output()
# Solve specific humidity equation
A_qn = Sol.Solver12(1, 2, self.qn, srim_qn, srex_qn,Km/self.schmidt)[0]
RHS_qn = Sol.Solver12(1, 2, self.qn, srim_qn, srex_qn,Km/self.schmidt)[1]
Inv_qn = Invert(self.nz, A_qn, RHS_qn)
self.qn = Inv_qn.Output()
# Set a minimum value for kinetic energy which avoid trapping of heat at street level
for i in range(0, self.nz):
if self.tke[i] < 1e-3:
self.tke[i] = 1e-3
return self.vx,self.vy,self.tke,self.th,self.qn, ustarCol,Km,tveg_tmp,Ri_b,Tveg
proc_list = []
for samp in allSamps:
proc = Process(target=basicAnalyze,
args=(
samp,
args.species,
args.threads,
args.alignTool,
))
proc_list.append(proc)
for p in proc_list:
p.start()
for p in proc_list:
p.join()
PlotCorrelation(allSamps, args.threads)
Matrix(allSamps)
try:
Diff(len(conSamps), args.species)
DiffTransFig()
except:
print('Diff error !')
finally:
Cluster(allSamps)
Enrich()
rMATS(conSamps, expSamps, args.species)
rmats2sashimiplot(conSamps, expSamps)
print('分析结束')
t1 = time.time()
print('\n' + '#' * 60)
def test_TC11(self):
try_result = {}
result = Diff().diff_two_dict({}, {}, {})
self.assertEqual(try_result, result)
class Sync(object):
"""
Sync is a class for synchronizing or updating one directory with another.
The class operates one directionally, so a true sync would require multiple
Syncs that do not purge files.
Optionally, a file path list can be supplied to limit the sync between
the src and dst. This file list can either be relative paths or include
the src or dst prefixes in their file paths.
The basic setup is to pass a src and dst path then run the ``diff`` method.
This will generate a Diff instance that will provide information about
which files would be created/updated/purged if the sync was run. The diff
can then be trimmed using ``difftrim``. Once the difference reflects the
desired changes, run the ``sync`` or ``update`` methods depending on if
files/dirs should be created and updated, or only updated.
TODO: describe the diff settings and run settings here
"""
def __init__(self, src=None, dst=None, **kwargs):
self.src = None if src is None else os.path.normpath(src)
self.dst = None if dst is None else os.path.normpath(dst)
self.origdiff = None
self.trimdiff = None
self.ops = ['create', 'update', 'purge']
self.diffstngs = {
'filters': [],
'excludes': ['.DS_Store', 'Thumbs.db', '.place-holder'],
'filelist': [],
'regexfilters': False,
'includedirs': True,
'timeprecision': 3,
'recursive': True,
'newer': True,
'forceUpdate': False,
'sizeLimit': 0,
}
self.runstngs = {
'maketarget': True,
'trimmed': True,
'create': False,
'update': False,
'purge': False,
'forceOwnership': False,
'errorsToDebug': False,
}
self.progressfnc = None
self.progresscheck = None
self.progressamt = 0
self.stats = {
'stime': 0.0,
'etime': 0.0,
'creates': [],
'createfails': [],
'updates': [],
'updatefails': [],
'purges': [],
'purgefails': [],
}
self.__hasrun = False
self.__hasrundiff = False
self.__diffcurrent = False
for k, v in kwargs.items():
if k in self.diffstngs.keys():
self.diffstngs[k] = v
elif k in self.runstngs.keys():
self.runstngs[k] = v
def getopts(self):
"""
Return a list of options and their values
"""
result = {}
for k in self.diffstngs.keys():
result[k] = self.diffstngs[k]
for k in self.runstngs.keys():
result[k] = self.runstngs[k]
return result
def __validate(self):
if self.src is None or self.dst is None:
return False
return True
def diff(self, **kwargs):
"""
Compile a difference list of files between src and dst directories
"""
if self.__validate():
# TODO: filter kwargs
self.diffstngs.update(kwargs)
self.origdiff = Diff(self.src, self.dst, **self.diffstngs)
self.trimdiff = self.origdiff.copy()
self.__hasrundiff = True
self.__diffcurrent = True
def difftrim(self, create=[], update=[], purge=[]):
"""
Removes items from ``origdiff`` and saves the results in ``trimdiff``
``create`` -- a list of full paths to remove from the create diff list
``update`` -- a list of full paths to remove from the update diff list
``purge`` -- a list of full paths to remove from the purge diff list
"""
for path in create:
self.trimdiff.remove_create(path)
for path in update:
self.trimdiff.remove_update(path)
for path in purge:
self.trimdiff.remove_purge(path)
def sync(self, refreshDiff=False, dry_run=False, **kwargs):
"""
Copy any new files and update any existing files from src to dst
directories using the compiled dirdiff list
"""
self.runstngs['create'] = True
self.runstngs['update'] = True
self.runstngs.update(kwargs)
self.run(refreshDiff=refreshDiff, dry_run=dry_run)
def update(self, refreshDiff=False, dry_run=False, **kwargs):
"""
Update only files that already exist in both src and dst
"""
self.runstngs['create'] = False
self.runstngs['update'] = True
self.runstngs.update(kwargs)
self.run(refreshDiff=refreshDiff, dry_run=dry_run)
def run(self, refreshDiff=False, dry_run=False, **kwargs):
"""
Run the directory sync given the current run settings
``refreshDiff`` -- re-runs diff() after syncing
"""
try:
if kwargs.has_key('no_update'):
LOG.warning(
'Filesync Deprecation Warning: \'no_update\' is no longer supported, use \'refreshDiff\' instead'
)
refreshDiff = not kwargs['no_update']
if not self.__diffcurrent:
LOG.warning(
'diff is not current; it\'s recommended to run diff again before updating/synching'
)
self.stats['stime'] = time.time()
self.__run(dry_run=dry_run)
self.stats['etime'] = time.time()
self.__hasrun = True
self.__diffcurrent = False
if refreshDiff:
self.diff()
except:
LOG.exception("Exception running filesync")
def __run(self, dry_run=False):
# reset stats
self.stats['creates'] = []
self.stats['createfails'] = []
self.stats['updates'] = []
self.stats['updatefails'] = []
self.stats['purges'] = []
self.stats['purgefails'] = []
# determine the diff to use (trimmed or untrimmed)
d = self.trimdiff if self.runstngs['trimmed'] else self.origdiff
if d is None:
return
return self.runwithdiff(d, dry_run)
def runwithdiff(self, diff, dry_run=False):
if not isinstance(diff, Diff):
raise TypeError('expected Diff, got {0}'.format(
type(diff).__name__))
# run through all 'create' files
if self.runstngs['create']:
LOG.debug('Creating')
if LOG.getEffectiveLevel() <= logging.DEBUG:
ROOTLOG.indent += 1
items = sorted(diff.create.items())
for path, files in items:
if self.progresscheck is not None:
if not self.progresscheck():
return
relpath = os.path.relpath(path, self.src)
if relpath == '.':
relpath = ''
srcdir = os.path.join(self.src, relpath)
dstdir = os.path.join(self.dst, relpath)
# make the destination dir if it doesn't exist
if not os.path.isdir(dstdir):
self.__makedirs(dstdir, self.stats['creates'],
self.stats['createfails'], dry_run)
for f in files:
srcp = os.path.join(srcdir, f)
dstp = os.path.join(dstdir, f)
if os.path.isdir(srcp):
self.__copydir(srcp, dstp, self.stats['creates'],
self.stats['createfails'], dry_run)
elif os.path.isfile(srcp):
self.__copy(srcp, dstp, self.stats['creates'],
self.stats['createfails'], dry_run)
if LOG.getEffectiveLevel() <= logging.DEBUG:
ROOTLOG.indent -= 1
# run through all 'update' files
if self.runstngs['update']:
LOG.debug('Updating')
if LOG.getEffectiveLevel() <= logging.DEBUG:
ROOTLOG.indent += 1
items = sorted(diff.update.items())
for path, files in items:
if self.progresscheck is not None:
if not self.progresscheck():
return
relpath = os.path.relpath(path, self.src)
if relpath == '.':
relpath = ''
srcdir = os.path.join(self.src, relpath)
dstdir = os.path.join(self.dst, relpath)
for f in files:
# updates never include dirs
srcp = os.path.join(srcdir, f)
dstp = os.path.join(dstdir, f)
self.__copy(srcp, dstp, self.stats['updates'],
self.stats['updatefails'], dry_run)
if LOG.getEffectiveLevel() <= logging.DEBUG:
ROOTLOG.indent -= 1
# run through all 'purge' files
if self.runstngs['purge']:
LOG.debug('Purging')
if LOG.getEffectiveLevel() <= logging.DEBUG:
ROOTLOG.indent += 1
items = sorted(diff.purge.items())
for path, files in items:
if self.progresscheck is not None:
if not self.progresscheck():
return
relpath = os.path.relpath(path, self.dst)
if relpath == '.':
relpath = ''
dstdir = os.path.join(self.dst, relpath)
for f in files:
dstp = os.path.join(dstdir, f)
if os.path.isdir(dstp):
self.__rmdir(dstp, self.stats['purges'],
self.stats['purgefails'], dry_run)
elif os.path.isfile(dstp):
self.__remove(dstp, self.stats['purges'],
self.stats['purgefails'], dry_run)
else:
LOG.debug('file/folder not found: {0}'.format(dstp))
if LOG.getEffectiveLevel() <= logging.DEBUG:
ROOTLOG.indent -= 1
if self.progressfnc:
self.progressfnc("Sync Complete", 100)
def __makedirs(self, dir_, passes=None, fails=None, dry_run=False):
"""
Make the given dir_ including any parent dirs
Append dir_ to ``fails`` on error
"""
try:
os.makedirs(dir_)
except Exception as e:
LOG.error(e)
if fails is not None:
fails.append(dir_)
else:
if passes is not None:
passes.append(dir_)
LOG.debug('made dirs: {0}'.format(dir_))
def __getProgPercent(self):
"""
Get a progress percentage and return it.
"""
self.progressamt += 1
return float(self.progressamt) / float(self.origdiff.totalcount) * 100
def __copydir(self, src, dst, passes=None, fails=None, dry_run=False):
"""
Make the given dst directory and copy stats from src
Append dst to ``fails`` on error
"""
if self.progressfnc:
self.progressfnc('Copying to {0}'.format(dst),
self.__getProgPercent())
try:
if not dry_run:
os.mkdir(dst)
except Exception as e:
LOG.error(e)
if fails is not None:
fails.append(dst)
else:
if os.path.exists(dst) and self.runstngs['forceOwnership']:
# make writable
fileAtt = os.stat(dst)[0]
if (not fileAtt & stat.S_IWRITE):
try:
os.chmod(dst, stat.S_IWRITE)
except Exception as e:
LOG.error(
'could not make file writable {0}: {1}'.format(
dst, e))
return False
shutil.copystat(src, dst)
if passes is not None:
passes.append(dst)
LOG.debug('Created Directory: {0}'.format(dst))
def __copy(self, src, dst, passes=None, fails=None, dry_run=False):
"""
Copy the given src file to dst
Append dst to ``fails`` on error
"""
if self.progressfnc:
self.progressfnc('Copying {0} -> {1}'.format(src, dst),
self.__getProgPercent())
try:
if not dry_run:
if os.path.exists(dst) and self.runstngs['forceOwnership']:
# make writable
fileAtt = os.stat(dst)[0]
if (not fileAtt & stat.S_IWRITE):
try:
os.chmod(dst, stat.S_IWRITE)
except Exception as e:
LOG.error(
'Could not make file writable {0}: {1}'.format(
dst, e))
return False
shutil.copy2(src, dst)
except (IOError, OSError) as e:
if self.runstngs['errorsToDebug']:
LOG.debug(e)
else:
LOG.error(e)
if fails is not None:
fails.append(dst)
else:
if passes is not None:
passes.append(dst)
LOG.debug('Copied: {0}'.format(dst))
def __rmdir(self, dir_, passes=None, fails=None, dry_run=False):
"""
Remove the given dir_.
Append dir_ to ``fails`` on error
"""
if self.progressfnc:
self.progressfnc('Deleting {0}'.format(dir_),
self.__getProgPercent())
if not os.path.isdir(dir_):
LOG.warning('Directory does not exist: {0}'.format(dir_))
return
try:
if not dry_run:
shutil.rmtree(dir_)
except Exception as e:
LOG.error(e)
if fails is not None:
fails.append(dir_)
else:
if passes is not None:
passes.append(dir_)
LOG.debug('Removed dir: {0}'.format(dir_))
def __remove(self, f, passes=None, fails=None, dry_run=False):
"""
Delete the given file
Append f to ``fails`` on error
"""
if self.progressfnc:
self.progressfnc('Deleting {0}'.format(f), self.__getProgPercent())
if not os.path.isfile(f):
LOG.warning('File does not exist: {0}'.format(f))
return
try:
if not dry_run:
os.remove(f)
except OSError as e:
LOG.error(e)
if fails is not None:
fails.append(f)
else:
if passes is not None:
passes.append(f)
LOG.debug('Deleted: {0}'.format(f))
def report(self, diff=False):
if not self.__hasrun or diff:
if not self.__diffcurrent:
LOG.warning('diff is not current')
return self.diffreport()
else:
self.runreport()
def diffreport(self, **kwargs):
"""
Return a report from the current diff
``trimmed`` -- if True, returns the trimmed diff's report
"""
if self.origdiff is None:
return
if self.runstngs['trimmed']:
return self.trimdiff.report(**kwargs)
else:
return self.origdiff.report(**kwargs)
def runreport(self):
"""
Print a report for the last update/sync/run.
"""
if self.src is None:
LOG.info('No source path is defined')
return
if self.dst is None:
LOG.info('No destination path is defined')
return
# build title
title = 'Sync report ({0} -> {1}):'.format(self.src, self.dst)
dashes = '-' * len(title)
result = '\n{0}\n{1}\n'.format(title, dashes)
# loop through all attributes
attrs = ['create', 'update', 'purge']
for attr in attrs:
fails = self.stats['{0}fails'.format(attr)]
passes = self.stats['{0}s'.format(attr)]
result += ('\n{attr}: ({0})\n'.format(len(passes),
attr=(attr.title() +
' Passes')))
result += ('{attr}: ({0})\n'.format(len(fails),
attr=(attr.title() +
' Fails')))
for item in fails:
result += (' {0}\n'.format(item))
LOG.info(result)
return result
class TestDiff(unittest.TestCase):
def setUp(self):
self.old_dict = {"dict1":{"dict2": {"x":"A","z":"d"}}}
self.new_dict = {"dict1":{"dict2": {"x":"C","y":"B"}}}
self.old_flat = {"a":"a", "z":"z"}
self.new_flat = {"a":"f", "b":"b"}
self.oldsame = {"dict1":{"dict2": {"x":"A","z":"d"}}}
self.newsame = {"dict1":{"dict2": {"x":"A","z":"d"}}}
self.old = {"result":{
"first name":"bob",
"last name":"segar",
"phone":{"home":"615-555-4433",
"mobile":"615-555-4444"},
"title":["programmer", "dev-ops"],
"salary":{"starting":80000,
"20150303":90000}}}
self.new = {"result":{
"first name":"bob",
"last name":"segar",
"phone":{"mobile":"615-555-4444"},
"title":["python developer", "dev-ops"],
"salary":{"starting":80000,
"20150303":90000},
"clearance":"level Z"}}
self.d = Diff()
def test_added_logic(self):
result = self.d.added(self.old_dict, self.new_dict)
expected = [{"operation": "ADDED", "field": "dict1.dict2.y", "new": "B"}]
self.assertEqual(result, expected)
def test_added_flat_dict(self):
result = self.d.added(self.old_flat, self.new_flat)
expected = [{'field': 'b', 'operation': 'ADDED', 'new': 'b'}]
self.assertEqual(result, expected)
def test_modified_logic(self):
result = self.d.modified(self.old_dict, self.new_dict)
expected = [{"operation": "MODIFIED", "field": "dict1.dict2.x", "old": "A", "new": "C"}]
self.assertEqual(result, expected)
def test_modified_flat_dict(self):
result = self.d.modified(self.old_flat, self.new_flat)
expected = [{'field': 'a', 'operation': 'MODIFIED', 'new': 'f', 'old': 'a'}]
self.assertEqual(result, expected)
def test_deleted_logic(self):
result = self.d.deleted(self.old_dict, self.new_dict)
expected = [{"operation": "DELETED", "field": "dict1.dict2.z", "old": "d"}]
self.assertEqual(result, expected)
def test_deleted_flat_dict(self):
result = self.d.deleted(self.old_flat, self.new_flat)
expected = [{'field': 'z', 'operation': 'DELETED', 'old': 'z'}]
self.assertEqual(result, expected)
def test_difference_logic(self):
result = self.d.combine_results(self.old_dict, self.new_dict)
expected = [
{"operation": "ADDED", "field": "dict1.dict2.y", "new": "B"},
{"operation": "MODIFIED", "field": "dict1.dict2.x", "old": "A", "new": "C"},
{"operation": "DELETED", "field": "dict1.dict2.z", "old": "d"}
]
self.assertEqual(result, expected)
def test_no_difference(self):
result = self.d.combine_results(self.oldsame, self.newsame)
expected = []
self.assertEqual(result, expected)
class Sync(object):
"""
Sync is a class for synchronizing or updating one directory with another.
The class operates one directionally, so a true sync would require multiple
Syncs that do not purge files.
Optionally, a file path list can be supplied to limit the sync between
the src and dst. This file list can either be relative paths or include
the src or dst prefixes in their file paths.
The basic setup is to pass a src and dst path then run the ``diff`` method.
This will generate a Diff instance that will provide information about
which files would be created/updated/purged if the sync was run. The diff
can then be trimmed using ``difftrim``. Once the difference reflects the
desired changes, run the ``sync`` or ``update`` methods depending on if
files/dirs should be created and updated, or only updated.
TODO: describe the diff settings and run settings here
"""
def __init__(self, src=None, dst=None, **kwargs):
self.src = None if src is None else os.path.normpath(src)
self.dst = None if dst is None else os.path.normpath(dst)
self.origdiff = None
self.trimdiff = None
self.ops = ['create', 'update', 'purge']
self.diffstngs = {
'filters':[],
'excludes':['.DS_Store','Thumbs.db','.place-holder'],
'filelist':[],
'regexfilters':False,
'includedirs':True,
'timeprecision':3,
'recursive':True,
'newer':True,
'forceUpdate':False,
'sizeLimit':0,
}
self.runstngs = {
'maketarget':True,
'trimmed':True,
'create':False,
'update':False,
'purge':False,
'forceOwnership':False,
'errorsToDebug':False,
}
self.progressfnc = None
self.progresscheck = None
self.progressamt = 0
self.stats = {
'stime':0.0,
'etime':0.0,
'creates':[],
'createfails':[],
'updates':[],
'updatefails':[],
'purges':[],
'purgefails':[],
}
self.__hasrun = False
self.__hasrundiff = False
self.__diffcurrent = False
for k, v in kwargs.items():
if k in self.diffstngs.keys():
self.diffstngs[k] = v
elif k in self.runstngs.keys():
self.runstngs[k] = v
def getopts(self):
"""
Return a list of options and their values
"""
result = {}
for k in self.diffstngs.keys():
result[k] = self.diffstngs[k]
for k in self.runstngs.keys():
result[k] = self.runstngs[k]
return result
def __validate(self):
if self.src is None or self.dst is None:
return False
return True
def diff(self, **kwargs):
"""
Compile a difference list of files between src and dst directories
"""
if self.__validate():
# TODO: filter kwargs
self.diffstngs.update(kwargs)
self.origdiff = Diff(self.src, self.dst, **self.diffstngs)
self.trimdiff = self.origdiff.copy()
self.__hasrundiff = True
self.__diffcurrent = True
def difftrim(self, create=[], update=[], purge=[]):
"""
Removes items from ``origdiff`` and saves the results in ``trimdiff``
``create`` -- a list of full paths to remove from the create diff list
``update`` -- a list of full paths to remove from the update diff list
``purge`` -- a list of full paths to remove from the purge diff list
"""
for path in create:
self.trimdiff.remove_create(path)
for path in update:
self.trimdiff.remove_update(path)
for path in purge:
self.trimdiff.remove_purge(path)
def sync(self, refreshDiff=False, dry_run=False, **kwargs):
"""
Copy any new files and update any existing files from src to dst
directories using the compiled dirdiff list
"""
self.runstngs['create'] = True
self.runstngs['update'] = True
self.runstngs.update(kwargs)
self.run(refreshDiff=refreshDiff, dry_run=dry_run)
def update(self, refreshDiff=False, dry_run=False, **kwargs):
"""
Update only files that already exist in both src and dst
"""
self.runstngs['create'] = False
self.runstngs['update'] = True
self.runstngs.update(kwargs)
self.run(refreshDiff=refreshDiff, dry_run=dry_run)
def run(self, refreshDiff=False, dry_run=False, **kwargs):
"""
Run the directory sync given the current run settings
``refreshDiff`` -- re-runs diff() after syncing
"""
try:
if kwargs.has_key('no_update'):
LOG.warning('Filesync Deprecation Warning: \'no_update\' is no longer supported, use \'refreshDiff\' instead')
refreshDiff = not kwargs['no_update']
if not self.__diffcurrent:
LOG.warning('diff is not current; it\'s recommended to run diff again before updating/synching')
self.stats['stime'] = time.time()
self.__run(dry_run=dry_run)
self.stats['etime'] = time.time()
self.__hasrun = True
self.__diffcurrent = False
if refreshDiff:
self.diff()
except:
LOG.exception("Exception running filesync")
def __run(self, dry_run=False):
# reset stats
self.stats['creates'] = []
self.stats['createfails'] = []
self.stats['updates'] = []
self.stats['updatefails'] = []
self.stats['purges'] = []
self.stats['purgefails'] = []
# determine the diff to use (trimmed or untrimmed)
d = self.trimdiff if self.runstngs['trimmed'] else self.origdiff
if d is None:
return
return self.runwithdiff(d, dry_run)
def runwithdiff(self, diff, dry_run=False):
if not isinstance(diff, Diff):
raise TypeError('expected Diff, got {0}'.format(type(diff).__name__))
# run through all 'create' files
if self.runstngs['create']:
LOG.debug('Creating')
if LOG.getEffectiveLevel() <= logging.DEBUG:
ROOTLOG.indent += 1
items = sorted(diff.create.items())
for path, files in items:
if self.progresscheck is not None:
if not self.progresscheck():
return
relpath = os.path.relpath(path, self.src)
if relpath == '.':
relpath = ''
srcdir = os.path.join(self.src, relpath)
dstdir = os.path.join(self.dst, relpath)
# make the destination dir if it doesn't exist
if not os.path.isdir(dstdir):
self.__makedirs(dstdir, self.stats['creates'], self.stats['createfails'], dry_run)
for f in files:
srcp = os.path.join(srcdir, f)
dstp = os.path.join(dstdir, f)
if os.path.isdir(srcp):
self.__copydir(srcp, dstp, self.stats['creates'], self.stats['createfails'], dry_run)
elif os.path.isfile(srcp):
self.__copy(srcp, dstp, self.stats['creates'], self.stats['createfails'], dry_run)
if LOG.getEffectiveLevel() <= logging.DEBUG:
ROOTLOG.indent -= 1
# run through all 'update' files
if self.runstngs['update']:
LOG.debug('Updating')
if LOG.getEffectiveLevel() <= logging.DEBUG:
ROOTLOG.indent += 1
items = sorted(diff.update.items())
for path, files in items:
if self.progresscheck is not None:
if not self.progresscheck():
return
relpath = os.path.relpath(path, self.src)
if relpath == '.':
relpath = ''
srcdir = os.path.join(self.src, relpath)
dstdir = os.path.join(self.dst, relpath)
for f in files:
# updates never include dirs
srcp = os.path.join(srcdir, f)
dstp = os.path.join(dstdir, f)
self.__copy(srcp, dstp, self.stats['updates'], self.stats['updatefails'], dry_run)
if LOG.getEffectiveLevel() <= logging.DEBUG:
ROOTLOG.indent -= 1
# run through all 'purge' files
if self.runstngs['purge']:
LOG.debug('Purging')
if LOG.getEffectiveLevel() <= logging.DEBUG:
ROOTLOG.indent += 1
items = sorted(diff.purge.items())
for path, files in items:
if self.progresscheck is not None:
if not self.progresscheck():
return
relpath = os.path.relpath(path, self.dst)
if relpath == '.':
relpath = ''
dstdir = os.path.join(self.dst, relpath)
for f in files:
dstp = os.path.join(dstdir, f)
if os.path.isdir(dstp):
self.__rmdir(dstp, self.stats['purges'], self.stats['purgefails'], dry_run)
elif os.path.isfile(dstp):
self.__remove(dstp, self.stats['purges'], self.stats['purgefails'], dry_run)
else:
LOG.debug('file/folder not found: {0}'.format(dstp))
if LOG.getEffectiveLevel() <= logging.DEBUG:
ROOTLOG.indent -= 1
if self.progressfnc:
self.progressfnc("Sync Complete", 100)
def __makedirs(self, dir_, passes=None, fails=None, dry_run=False):
"""
Make the given dir_ including any parent dirs
Append dir_ to ``fails`` on error
"""
try:
os.makedirs(dir_)
except Exception as e:
LOG.error(e)
if fails is not None:
fails.append(dir_)
else:
if passes is not None:
passes.append(dir_)
LOG.debug('made dirs: {0}'.format(dir_))
def __getProgPercent(self):
"""
Get a progress percentage and return it.
"""
self.progressamt += 1
return float(self.progressamt) / float(self.origdiff.totalcount) * 100
def __copydir(self, src, dst, passes=None, fails=None, dry_run=False):
"""
Make the given dst directory and copy stats from src
Append dst to ``fails`` on error
"""
if self.progressfnc:
self.progressfnc('Copying to {0}'.format(dst), self.__getProgPercent())
try:
if not dry_run:
os.mkdir(dst)
except Exception as e:
LOG.error(e)
if fails is not None:
fails.append(dst)
else:
if os.path.exists(dst) and self.runstngs['forceOwnership']:
# make writable
fileAtt = os.stat(dst)[0]
if (not fileAtt & stat.S_IWRITE):
try:
os.chmod(dst, stat.S_IWRITE)
except Exception as e:
LOG.error('could not make file writable {0}: {1}'.format(dst, e))
return False
shutil.copystat(src, dst)
if passes is not None:
passes.append(dst)
LOG.debug('Created Directory: {0}'.format(dst))
def __copy(self, src, dst, passes=None, fails=None, dry_run=False):
"""
Copy the given src file to dst
Append dst to ``fails`` on error
"""
if self.progressfnc:
self.progressfnc('Copying {0} -> {1}'.format(src, dst), self.__getProgPercent())
try:
if not dry_run:
if os.path.exists(dst) and self.runstngs['forceOwnership']:
# make writable
fileAtt = os.stat(dst)[0]
if (not fileAtt & stat.S_IWRITE):
try:
os.chmod(dst, stat.S_IWRITE)
except Exception as e:
LOG.error('Could not make file writable {0}: {1}'.format(dst, e))
return False
shutil.copy2(src, dst)
except (IOError, OSError) as e:
if self.runstngs['errorsToDebug']:
LOG.debug(e)
else:
LOG.error(e)
if fails is not None:
fails.append(dst)
else:
if passes is not None:
passes.append(dst)
LOG.debug('Copied: {0}'.format(dst))
def __rmdir(self, dir_, passes=None, fails=None, dry_run=False):
"""
Remove the given dir_.
Append dir_ to ``fails`` on error
"""
if self.progressfnc:
self.progressfnc('Deleting {0}'.format(dir_), self.__getProgPercent())
if not os.path.isdir(dir_):
LOG.warning('Directory does not exist: {0}'.format(dir_))
return
try:
if not dry_run:
shutil.rmtree(dir_)
except Exception as e:
LOG.error(e)
if fails is not None:
fails.append(dir_)
else:
if passes is not None:
passes.append(dir_)
LOG.debug('Removed dir: {0}'.format(dir_))
def __remove(self, f, passes=None, fails=None, dry_run=False):
"""
Delete the given file
Append f to ``fails`` on error
"""
if self.progressfnc:
self.progressfnc('Deleting {0}'.format(f), self.__getProgPercent())
if not os.path.isfile(f):
LOG.warning('File does not exist: {0}'.format(f))
return
try:
if not dry_run:
os.remove(f)
except OSError as e:
LOG.error(e)
if fails is not None:
fails.append(f)
else:
if passes is not None:
passes.append(f)
LOG.debug('Deleted: {0}'.format(f))
def report(self, diff=False):
if not self.__hasrun or diff:
if not self.__diffcurrent:
LOG.warning('diff is not current')
return self.diffreport()
else:
self.runreport()
def diffreport(self, **kwargs):
"""
Return a report from the current diff
``trimmed`` -- if True, returns the trimmed diff's report
"""
if self.origdiff is None:
return
if self.runstngs['trimmed']:
return self.trimdiff.report(**kwargs)
else:
return self.origdiff.report(**kwargs)
def runreport(self):
"""
Print a report for the last update/sync/run.
"""
if self.src is None:
LOG.info('No source path is defined')
return
if self.dst is None:
LOG.info('No destination path is defined')
return
# build title
title = 'Sync report ({0} -> {1}):'.format(self.src, self.dst)
dashes = '-'*len(title)
result = '\n{0}\n{1}\n'.format(title, dashes)
# loop through all attributes
attrs = ['create', 'update', 'purge']
for attr in attrs:
fails = self.stats['{0}fails'.format(attr)]
passes = self.stats['{0}s'.format(attr)]
result += ('\n{attr}: ({0})\n'.format(len(passes), attr=(attr.title() + ' Passes')))
result += ('{attr}: ({0})\n'.format(len(fails), attr=(attr.title() + ' Fails')))
for item in fails:
result += (' {0}\n'.format(item))
LOG.info(result)
return result
def test_TC10(self):
with self.assertRaises(TypeError):
Diff().diff_two_dict(5, {}, {})
def judge(self, sub):
lang = sub.lang
mem_policy = True if (jcnf.POLICY[lang]=='ALL' or jcnf.POLICY[lang]=='MEM') else False
page_size = resource.getpagesize()
case_cnt = min(len(sub.case_lim), sub.case_cnt)
sub.case_done = 0
case_id = 0
for case_id in range(case_cnt):
sub.case_res.append({
'res':jcnf.JUDGE_RES['init'],
'time':0,
'mem':0,
})
exec_cmd = jcnf.EXEC_CMD[lang]
exec_param = jcnf.EXEC_PARAM[lang]
if lang == 'java':
mem_lim = 0
for case_id in xrange(case_cnt):
mem_lim = max(mem_lim, sub.case_lim[case_id]['mem'])
exec_param.insert(1, '-Xmx'+str(mem_lim/1024+1)+'m')
print exec_cmd
print exec_param
for case_id in xrange(case_cnt):
lim = sub.case_lim[case_id]
res = sub.case_res[case_id]
exec_pid = os.fork()
if exec_pid == 0:
for rk, rv in jcnf.EXEC_RLIM[lang].items():
resource.setrlimit(rk, rv)
try:
exec_i = open(jcnf.getCasePathI(sub.pid, case_id), 'r')
exec_o = open(jcnf.getExecPathO(), 'w')
except:
#sys.stderr.write('file read error')
raise Exception('cannot handle input or output file');
lim['time'] = max(lim['time'], jcnf.EXEC_MIN_TL[lang])
lim['time'] = int(lim['time']*jcnf.EXEC_TL_RATIO[lang])
rlimt = (lim['time']-1)//1000+2
resource.setrlimit(resource.RLIMIT_CPU, (rlimt, rlimt));
rlimm = jcnf.EXEC_MAX_MEM
if mem_policy: # java uses virtual machine
resource.setrlimit(resource.RLIMIT_AS, (rlimm, rlimm));
os.dup2(exec_i.fileno(), 0)
os.dup2(exec_o.fileno(), 1)
#TOTO: ptrace
pt_ret = cptrace.ptrace(cptrace.PTRACE_TRACEME, 0)
if (pt_ret == -1):
#sys.stderr.write('warning: ptrace error')
raise Exception('child process cannot be ptraced');
if exec_i:
exec_i.close()
if exec_o:
exec_o.close()
os._exit(1)
os.execvp(exec_cmd, exec_param)
sys.stderr.write('warning: something wrong')
if exec_i:
exec_i.close()
if exec_o:
exec_o.close()
os._exit(1)
else:
stat_info_file = str(exec_pid).join(['/proc/', '/statm'])
res['mem'] = 0
res['time'] = 0
t_prev = jclk()
eax_prev = 142857
insyscall = False
while res['res']==jcnf.JUDGE_RES['init']:
#while True:
exec_status = os.waitpid(-1, 0)
if exec_status[0] != exec_pid:
continue
t_now = jclk()
res['time'] += t_now-t_prev
#t_prev = t_now
DEBUG_CNT = 0
if os.WIFSIGNALED(exec_status[1]):
#strange exited or tle?
if res['time']*1000>lim['time']:
res['res'] = jcnf.JUDGE_RES['tle']
break
elif os.WIFEXITED(exec_status[1]):
#normally exited, ok
break
elif os.WIFSTOPPED(exec_status[1]):
#sigtrap by ptrace
exec_sig = os.WSTOPSIG(exec_status[1])
if exec_sig != signal.SIGTRAP:
res['res'] = jcnf.JUDGE_RES['re']
#print exec_status[0], exec_status[1], 'hehe', exec_sig
cptrace.ptrace(cptrace.PTRACE_KILL, exec_pid)
#strange exited?
break
eax_now = cptrace.ptrace(cptrace.PTRACE_PEEKUSER, exec_pid, 4*cptrace.ORIG_EAX) #when used in 64bit system, it should be 8*xxxx, so it is recommended to make it a const in conf
if jcnf.POLICY[lang]=='ALL':
if jcnf.SYSCALL[eax_now][0] == 0: #prohibited syscall
res['res'] = jcnf.JUDGE_RES['re']
cptrace.ptrace(cptrace.PTRACE_KILL, exec_pid) #deprecated! should be implemented in another way
break
else:
#TODO extend implementation
pass
if eax_now!=eax_prev and eax_now!=-1:
insyscall = False
if eax_now!=-1:
if insyscall:
DEBUG_CNT+=1
#if eax_now==45 or eax_now==90 or eax_now==91:
try:
stat_info = open(stat_info_file, 'r')
mem_now = int(stat_info.read().split(' ')[5]) #automatically to long when exceed
res['mem'] = max(res['mem'], mem_now)
stat_info.close()
except:
pass
insyscall = False
else:
insyscall = True
if mem_policy and res['mem']*page_size>lim['mem']*1024:
res['res'] = jcnf.JUDGE_RES['mle']
cptrace.ptrace(cptrace.PTRACE_KILL, exec_pid) #deprecated! should be implemented in another way
break
if res['time']*1000>lim['time']:
res['res'] = jcnf.JUDGE_RES['tle']
cptrace.ptrace(cptrace.PTRACE_KILL, exec_pid) #deprecated! should be implemented in another way
break
if eax_now!=-1:
eax_prev = eax_now
t_prev = jclk()
else:
#sys.stderr.write('unknown status')
pass
#TODO: also check total time limit?
if res['res'] == jcnf.JUDGE_RES['tle']:
#TODO: write log
cptrace.ptrace(cptrace.PTRACE_KILL, exec_pid) #deprecated! should be implemented in another way
else:
cptrace.ptrace(cptrace.PTRACE_SYSCALL, exec_pid)
try:
os.wait()
os.kill(exec_pid, signal.SIGKILL)
except Exception, e:
if JDEBUG: print 'cannot kill', Exception, e
pass
res['mem'] = int(res['mem']*page_size/1024)
res['time'] = int(res['time']*1000)
if res['res'] == jcnf.JUDGE_RES['init']:
if os.WIFSIGNALED(exec_status[1]):
res['res'] = jcnf.JUDGE_RES['re']
elif os.WIFSTOPPED(exec_status[1]) and os.WSTOPSIG(exec_status[1])!=signal.SIGTRAP:
res['res'] = jcnf.JUDGE_RES['re']
if res['res'] == jcnf.JUDGE_RES['init']:
df = Diff()
res['res'] = df.diff(jcnf.getCasePathO(sub.pid, case_id), jcnf.getExecPathO())
sub.case_done += 1
#sub.mem += res['mem']
sub.mem = max(sub.mem, res['mem'])
sub.time += res['time']
if res['res'] == jcnf.JUDGE_RES['init']:
res['res'] = jcnf.JUDGE_RES['se']
# Need to calculate the scores of all test data, and thus we cannot break out when judging
# if sub.block and res['res']!=jcnf.JUDGE_RES['ac']:
# break
t_prev = jclk()
def main(diff_file, output_file, ignore_files):
d = Diff(diff_file, ignore_files.split(","))
d.export(output_file)