def big2littlePTS(self, fileName, foleName):
# ~~ Openning files ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
fle = open(fileName, 'rb')
fole = open(foleName, 'wb')
print ' +> writing the volumes-file: ', foleName
# ~~ Read/Write dimensions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
n1 = self.NPOIN3 + 1
minvol = self.minvol * np.ones(self.NPOIN3, dtype=np.float32)
# ~~ Read volumes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
pbar = ProgressBar(maxval=self.HYDROIT).start()
for i in range(self.HYDROIT):
l, it = unpack('>ii', fle.read(4 + 4))
VOLUME = np.asarray(
unpack('>' + str(self.NPOIN3) + 'f',
fle.read(4 * self.NPOIN3)))
VOLUME = np.maximum(VOLUME, minvol)
fle.seek(4, 1)
if it >= self.tfrom and it <= self.tstop:
pbar.write(' ~> read iteration: ' + str(it), i)
fole.write(pack('<ii', 4 * n1, it - self.HYDRO00))
fole.write(pack('<' + str(self.NPOIN3) + 'f', *(VOLUME)))
fole.write(pack('<i', 4 * n1))
else:
pbar.write(' ~> ignore iteration: ' + str(it), i)
pbar.update(i)
pbar.finish()
fle.close()
fole.close()
def removeDuplangles(self):
ibar = 0
pbar = ProgressBar(maxval=self.npoin).start()
ip = 0
while ip < len(self.poly):
ibar += len(self.poly[ip])
lb = len(self.poly[ip])
self.poly[ip], self.type[ip] = removeDuplangles(
self.poly[ip], self.type[ip])
la = len(self.poly[ip])
if la < lb:
pbar.write(
' +> removed ' + str(lb - la) + ' points of ' +
str(lb) + ' from polygon ' + str(ip + 1), ibar)
if self.poly[ip] == []:
pbar.write(' +> removed entire polygon ' + str(ip + 1),
ibar)
self.poly.pop(ip)
self.type.pop(ip)
else:
ip += 1
pbar.update(ibar)
pbar.finish()
return self.poly, self.type
def big2littleARE(self, fileName, foleName):
# ~~ Openning files ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
fle = open(fileName, 'rb')
fole = open(foleName, 'wb')
print ' +> writing the areas-file: ', foleName
# ~~ Read/Write dimensions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
NSEG2 = (3 * self.geo.NELEM3 + self.conlim.NPTFR) / 2
MBND2 = np.count_nonzero(self.conlim.BOR['lih'] != 2)
n3 = (NSEG2 + MBND2)
#n4 = 2*( self.geo.NPLAN-1 )*self.geo.NPOIN3
# ~~ Read volumes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
pbar = ProgressBar(maxval=self.HYDROIT).start()
for i in range(self.HYDROIT):
l, it = unpack('>ii', fle.read(4 + 4))
AREAS = np.asarray(unpack('>' + str(n3) + 'f', fle.read(4 * n3)))
fle.seek(4, 1)
if it >= self.tfrom and it <= self.tstop:
pbar.write(' ~> read iteration: ' + str(it), i)
fole.write(pack('<ii', 4 * n3, it - self.HYDRO00))
fole.write(pack('<' + str(n3) + 'f', *(AREAS)))
fole.write(pack('<i', 4 * n3))
else:
pbar.write(' ~> ignore iteration: ' + str(it), i)
pbar.update(i)
pbar.finish()
fle.close()
fole.close()
def big2little_pts(self, file_name, fole_name):
# ~~ Openning files ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
fle = open(file_name, 'rb')
fole = open(fole_name, 'wb')
print(' +> writing the volumes-file: ' + fole_name)
# ~~ Read/Write dimensions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
n_1 = self.npoin3 + 1
minvol = self.minvol * np.ones(self.npoin3, dtype=np.float32)
# ~~ Read volumes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
pbar = ProgressBar(maxval=self.hydroit).start()
for i in range(self.hydroit):
_, itime = unpack('>ii', fle.read(4 + 4))
volume = np.asarray(
unpack('>' + str(self.npoin3) + 'f',
fle.read(4 * self.npoin3)))
volume = np.maximum(volume, minvol)
fle.seek(4, 1)
if itime >= self.tfrom and itime <= self.tstop:
pbar.write(' ~> read iteration: ' + str(itime), i)
fole.write(pack('<ii', 4 * n_1, itime - self.hydro00))
fole.write(pack('<' + str(self.npoin3) + 'f', *(volume)))
fole.write(pack('<i', 4 * n_1))
else:
pbar.write(' ~> ignore iteration: ' + str(itime), i)
pbar.update(i)
pbar.finish()
fle.close()
fole.close()
def big2little_are(self, file_name, fole_name):
# ~~ Openning files ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
fle = open(file_name, 'rb')
fole = open(fole_name, 'wb')
print(' +> writing the areas-file: ' + fole_name)
# ~~ Read/Write dimensions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
nseg2 = (3 * self.geo.nelem3 + self.conlim.nptfr) / 2
mbnd2 = np.count_nonzero(self.conlim.bor['lih'] != 2)
n_3 = (nseg2 + mbnd2)
# ~~ Read volumes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
pbar = ProgressBar(maxval=self.hydroit).start()
for i in range(self.hydroit):
_, itime = unpack('>ii', fle.read(4 + 4))
areas = np.asarray(unpack('>' + str(n_3) + 'f', fle.read(4 * n_3)))
fle.seek(4, 1)
if itime >= self.tfrom and itime <= self.tstop:
pbar.write(' ~> read iteration: ' + str(itime), i)
fole.write(pack('<ii', 4 * n_3, itime - self.hydro00))
fole.write(pack('<' + str(n_3) + 'f', *(areas)))
fole.write(pack('<i', 4 * n_3))
else:
pbar.write(' ~> ignore iteration: ' + str(itime), i)
pbar.update(i)
pbar.finish()
fle.close()
fole.close()
def copyFiles(pi,po):
ld = listdir(pi)
ibar = 0; pbar = ProgressBar(maxval=len(ld)).start()
for f in ld:
if path.isfile(path.join(pi,f)): shutil.copy(path.join(pi,f),po)
pbar.update(ibar); ibar += 1
pbar.finish()
return
def addFileContent(file,lines):
SrcF = open(file,'ab')
ibar = 0; pbar = ProgressBar(maxval=len(lines)).start()
for line in lines[0:]:
ibar += 1; pbar.update(ibar)
SrcF.write('\n'+(line.rstrip()).replace('\r','').replace('\n\n','\n'))
pbar.finish()
SrcF.close()
return
def putFileContent(file,lines):
if path.exists(file): remove(file)
SrcF = open(file,'wb')
if len(lines)>0:
ibar = 0; pbar = ProgressBar(maxval=len(lines)).start()
SrcF.write((lines[0].rstrip()).replace('\r','').replace('\n\n','\n'))
for line in lines[1:]:
pbar.update(ibar); ibar += 1
SrcF.write('\n'+(line.rstrip()).replace('\r','').replace('\n\n','\n'))
pbar.finish()
SrcF.close()
return
def _generation(self):
clones = list(self.all_clones.clones())
num_clones = len(clones)
pb = ProgressBar(ncr(num_clones, 4) + ncr(num_clones, 3))
sys.stderr.write("Starting generation {} with {} clones\n".format(self.current_generation, num_clones))
for i in range(num_clones):
for j in range(i+1, num_clones):
for k in range(j+1, num_clones):
for l in range(k+1, num_clones):
box = Box([clones[i], clones[j], clones[k], clones[l]], self.current_generation)
self._process(box)
if self.done:
return
pb.increment()
for i in range(num_clones):
for j in range(i+1, num_clones):
for k in range(j+1, num_clones):
clone1 = clones[i]
clone2 = clones[j]
clone3 = clones[k]
box = Box([clones[i], clones[j], clones[k]], self.current_generation)
self._process(box)
if self.done:
return
pb.increment()
pb.clear()
def smoothSubsampleAngle(self, angle):
ibar = 0
pbar = ProgressBar(maxval=self.npoin).start()
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~/^\~~~~~~~~
# subsampling by anlgle \_/
ip = 0
while ip < len(self.poly):
ibar += len(self.poly[ip])
lb = len(self.poly[ip])
self.poly[ip], self.type[ip] = subsampleAngle(
self.poly[ip], self.type[ip], angle)
la = len(self.poly[ip])
if la < lb:
pbar.write(
' +> removed ' + str(lb - la) + ' points of ' +
str(lb) + ' from polygon ' + str(ip + 1), ibar)
if self.poly[ip] == []:
pbar.write(' +> removed entire polygon ' + str(ip + 1),
ibar)
self.poly.pop(ip)
self.type.pop(ip)
else:
ip += 1
#pbar.update(ibar)
pbar.finish()
return self.poly, self.type
def get_neighbours_slf(ikle, meshx, meshy, showbar=True):
"""
Return a list containing for each element the list of elements that are
neighbours to that element
@param ikle (np.array) Connectivity table
@param meshx (np.array) X coordinates of the mesh points
@param meshy (np.array) Y coordinates of the mesh points
@param showbar (boolean) If True display a progress bar
@returns (list) The list of neighbours
@param
"""
try:
from matplotlib.tri import Triangulation
neighbours = Triangulation(meshx, meshy, ikle).get_cpp_triangulation()\
.get_neighbors()
except ImportError:
insiders = {}
bounders = {}
ibar = 0
if showbar:
pbar = ProgressBar(maxval=(3 * len(ikle))).start()
for elem, i in zip(ikle, range(len(ikle))):
n_k = bounders.keys()
for k in [0, 1, 2]:
ibar += 1
if showbar:
pbar.update(ibar)
if (elem[k], elem[(k + 1) % 3]) not in n_k:
bounders.update({(elem[(k + 1) % 3], elem[k]): i})
else:
j = bounders[(elem[k], elem[(k + 1) % 3])]
insiders.update({(elem[k], elem[(k + 1) % 3]): [i, j]})
del bounders[(elem[k], elem[(k + 1) % 3])]
ibar = 0
neighbours = -np.ones((len(ikle), 3), dtype=np.int)
for elem, i in zip(ikle, range(len(ikle))):
for k in [0, 1, 2]:
ibar += 1
if showbar:
pbar.update(ibar)
if (elem[k], elem[(k + 1) % 3]) in insiders:
elem_a, elem_b = insiders[(elem[k], elem[(k + 1) % 3])]
if elem_a == i:
neighbours[i][k] = elem_b
if elem_b == i:
neighbours[i][k] = elem_a
if (elem[(k + 1) % 3], elem[k]) in insiders:
elem_a, elem_b = insiders[(elem[(k + 1) % 3], elem[k])]
if elem_a == i:
neighbours[i][k] = elem_b
if elem_b == i:
neighbours[i][k] = elem_a
if showbar:
pbar.finish()
return neighbours
def valid_epoch(self, data_loader):
pbar = ProgressBar(n_total=len(data_loader), desc='Evaluating')
self.entity_score.reset()
valid_loss = AverageMeter()
#output_file = jsonlines.open("data/case_out.jsonl","w")
for step, batch in enumerate(data_loader):
batch = tuple(t.to(self.device) for t in batch)
input_ids, input_mask, trigger_mask, segment_ids, label_ids, input_lens, one_hot_labels = batch
if not self.trigger:
trigger_mask = None
if not self.partial:
one_hot_labels = None
input_lens = input_lens.cpu().detach().numpy().tolist()
self.model.eval()
with torch.no_grad():
features, loss = self.model(input_ids,
segment_ids,
input_mask,
trigger_mask,
label_ids,
input_lens,
one_hot_labels=one_hot_labels)
tags, _ = self.model.crf._obtain_labels(
features, self.id2label, input_lens)
valid_loss.update(val=loss.item(), n=input_ids.size(0))
pbar(step=step, info={"loss": loss.item()})
label_ids = label_ids.to('cpu').numpy().tolist()
for i, label in enumerate(label_ids):
temp_1 = []
temp_2 = []
for j, m in enumerate(label):
if j == 0:
continue
elif j == input_lens[i] - 1: # 控制结束的位置
r = self.entity_score.update(pred_paths=[temp_2],
label_paths=[temp_1])
r["input_ids"] = input_ids[i, :].to(
"cpu").numpy().tolist()
#output_file.write(r)
break
else:
temp_1.append(self.id2label[label_ids[i][j]])
try:
temp_2.append(tags[i][j])
except Exception as e:
print(i, j)
valid_info, ex_valid_info, class_info = self.entity_score.result()
ex_info = {f'{key}': value for key, value in ex_valid_info.items()}
info = {f'{key}': value for key, value in valid_info.items()}
info['valid_loss'] = valid_loss.avg
if 'cuda' in str(self.device):
torch.cuda.empty_cache()
return info, ex_info, class_info
def makeClockwise(self):
ibar = 0
pbar = ProgressBar(maxval=self.npoin).start()
for ip in range(len(self.poly)):
ibar += len(self.poly[ip])
if self.type[ip] != 0:
if not isClockwise(self.poly[ip]):
pbar.write(
' +> turned clockwise polygon ' + str(ip + 1), ibar)
self.poly[ip] = np.flipud(self.poly[ip])
pbar.update(ibar)
pbar.finish()
return self.poly
def downloadECMWF(self):
result = self.connection.result()
fileName = self.request.get("target")
# ~> tries connecting 3 times before stopping
tries = 0
while True:
# ~> downloading file by blocks
http = urllib2.urlopen(result["href"])
f = open(fileName, "wb")
ibar = 0
pbar = ProgressBar(maxval=result["size"]).start()
while True:
chunk = http.read(1024 * 1024)
if not chunk: break
f.write(chunk)
ibar += len(chunk)
pbar.update(ibar)
f.flush()
f.close()
pbar.finish()
# ~> have I got everything ?
if ibar == result["size"]: break
if tries == 3:
print " ... exhausted the number of download trials.\nYou may wish to attempt this again later."
sys.exit()
print " ... trying to download the data once more ..."
tries += 1
def get_edges_slf(ikle, meshx, meshy, showbar=True):
"""
Returns the list of edges of the mesh
@param ikle (np.array) Connectivity table
@param meshx (np.array) X coordinates of the mesh points
@param meshy (np.array) Y coordinates of the mesh points
@param showbar (boolean) If True display a progress bar
@returns (list) The list of edges
"""
try:
from matplotlib.tri import Triangulation
edges = Triangulation(meshx, meshy, ikle).get_cpp_triangulation()\
.get_edges()
except ImportError:
edges = []
ibar = 0
if showbar:
pbar = ProgressBar(maxval=len(ikle)).start()
for elem in ikle:
ibar += 1
if showbar:
pbar.update(ibar)
if [elem[0], elem[1]] not in edges:
edges.append([elem[1], elem[0]])
if [elem[1], elem[2]] not in edges:
edges.append([elem[2], elem[1]])
if [elem[2], elem[0]] not in edges:
edges.append([elem[0], elem[2]])
if showbar:
pbar.finish()
return edges
def putContent(self, fileName, showbar=True):
# ~~> netcdf reader
ecmwfdata = netcdf.netcdf_file(self.request.get("target"), 'r')
# ~~> new SELAFIN writer
self.slf2d.fole = {}
self.slf2d.fole.update({'hook': open(fileName, 'wb')})
self.slf2d.fole.update({'name': fileName})
self.slf2d.fole.update({'endian': ">"}) # big endian
self.slf2d.fole.update({'float': ('f', 4)}) # single precision
print ' +> Write SELAFIN header'
self.appendHeaderECMWF(ecmwfdata)
print ' +> Write SELAFIN core'
ibar = 0
if showbar:
pbar = ProgressBar(maxval=len(self.slf2d.tags['times'])).start()
for t in range(len(self.slf2d.tags['times'])):
self.appendCoreTimeECMWF(t)
self.appendCoreVarsECMWF(ecmwfdata, ibar)
ibar += 1
if showbar: pbar.update(ibar)
self.slf2d.fole['hook'].close()
if showbar: pbar.finish()
def put_file_content(fle, lines):
"""
put line to file
@param fle (string) file
@param lines (string) adding line
"""
if path.exists(fle):
remove(fle)
src_file = open(fle, 'wb')
if len(lines) > 0:
ibar = 0
pbar = ProgressBar(maxval=len(lines)).start()
src_file.write(bytes((lines[0].rstrip()).replace('\r', '')\
.replace('\n\n', '\n'),
'utf-8'))
for line in lines[1:]:
pbar.update(ibar)
ibar += 1
src_file.write(bytes('\n'+(line.rstrip()).replace('\r', '')\
.replace('\n\n', '\n'),
'utf-8'))
pbar.finish()
src_file.close()
return
def fit(self, x, y, steps=0, batch_size=32):
batch_size = 32 if batch_size == None else batch_size
num_batches = x.shape[0] // batch_size
for i, p in enumerate(self.particles):
local_score = p.get_score(x, y)
if local_score < self.global_best_score:
self.global_best_score = local_score
self.global_best_weights = p.get_best_weights()
print("PSO -- Initial best score {:0.4f}".format(self.global_best_score))
bar = ProgressBar(steps, updates=20)
for i in range(steps):
for j in range(num_batches):
x_ = x[j*batch_size:(j+1)*batch_size,:]
y_ = y[j*batch_size:(j+1)*batch_size]
for p in self.particles:
local_score = p.step(x_, y_, self.global_best_weights)
if local_score < self.global_best_score:
self.global_best_score = local_score
self.global_best_weights = p.get_best_weights()
bar.update(i)
bar.done()
def put_content(self, file_name, showbar=True):
self.open_grib(file_name)
print(' +> Write Selafin header')
self.append_header_grib()
print(' +> Write Selafin core')
if showbar:
pbar = ProgressBar(maxval=len(self.dataset)).start()
for itime in range(len(self.dataset)):
seconds = int(self.slf2d.tags['times'][itime])
date = (datetime(*self.slf2d.datetime) +
timedelta(seconds=seconds)).timetuple()[0:6]
print(" - {}-{}-{} {}:{}:{}".format(date[2], date[1],
date[0], date[3],
date[4], date[5]))
self.append_core_time_grib(itime)
self.append_core_vars_grib(itime)
if showbar:
pbar.update(itime)
if showbar:
pbar.finish()
self.close_grib()
def copyFiles(pi,po):
ld = listdir(pi)
ibar = 0; pbar = ProgressBar(maxval=len(ld)).start()
for f in ld:
if path.isfile(path.join(pi,f)): shutil.copy(path.join(pi,f),po)
pbar.update(ibar); ibar += 1
pbar.finish()
return
def measure(self, cls, name, timeout=None, tries=None, processes=None):
timeout = timeout if timeout is not None else self.timeout
tries = tries if tries is not None else self.tries
processes = processes if processes is not None else self.processes
pb = ProgressBar(maximum=tries, width=30, prefix="{self.name:25}",
suffix=" {self.last_progress}/{self.maximum}")
measure_result = list()
for no_cpu in processes:
pb.name = "{:s} {:d} {:s}".format(name, no_cpu, 'core' if no_cpu == 1 else 'cores')
results = list()
for i in range(0, tries):
if print_output:
pb.progress(i)
targets = [cls() for j in range(0, no_cpu)]
with timer.measured("{:s} {:d}".format(name, i), False):
# start processes
for target in targets:
target.start()
# wait for timeout
time.sleep(timeout)
# send exit status
for target in targets:
target.shutdown()
# join threads
for target in targets:
target.join()
tmp = dict()
tmp['duration'] = timer.time()
tmp['value'] = sum(pluck(targets, 'result.value'))
tmp['exit'] = not max(pluck(targets, 'terminated'))
results.append(tmp)
if print_output:
pb.end()
result = dict()
result['processes'] = no_cpu
result['exit'] = min(pluck(results, 'exit'))
result['duration'] = sum(pluck(results, 'duration')) / float(tries)
result['value'] = sum(pluck(results, 'value')) / float(tries)
result['performance'] = result['value'] / result['duration']
if human_format:
result['value'] = human_readable(result['value'])
result['performance'] = human_readable(result['performance'])
measure_result.append(result)
return measure_result
def smoothSubdivise(self, weight=0.5):
ibar = 0
pbar = ProgressBar(maxval=self.npoin).start()
for ip in range(len(self.poly)):
ibar += len(self.poly[ip])
lb = len(self.poly[ip])
self.poly[ip], self.type[ip] = smoothSubdivise(
self.poly[ip], self.type[ip], weight)
la = len(self.poly[ip])
if la > lb:
pbar.write(
' +> added ' + str(la - lb) + ' points to polygon ' +
str(ip + 1), ibar)
pbar.update(ibar)
pbar.finish()
return self.poly, self.type
def addFileContent(file,lines):
SrcF = open(file,'ab')
ibar = 0; pbar = ProgressBar(maxval=len(lines)).start()
for line in lines[0:]:
ibar += 1; pbar.update(ibar)
SrcF.write('\n'+(line.rstrip()).replace('\r','').replace('\n\n','\n'))
pbar.finish()
SrcF.close()
return
def get_series(self, nodes, vars_indexes=None, showbar=True):
"""
Return the value for a list of nodes on variables given in vars_indexes
for each time step
@param nodes (list) list of nodes for which to extract data
@param vars_indexes (list) List of variables to extract data for
@param showbar (boolean) If True display a showbar for the progress
"""
f = self.file['hook']
endian = self.file['endian']
ftype, fsize = self.file['float']
if vars_indexes is None:
vars_indexes = self.varindex
# ~~ Ordering the nodes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# This assumes that nodes starts at 1
onodes = np.sort(np.array(zip(range(len(nodes)), nodes),
dtype=[('0', int), ('1', int)]),
order='1')
# ~~ Extract time profiles ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
if fsize == 4:
z = np.zeros(
(len(vars_indexes), len(nodes), len(self.tags['cores'])),
dtype=np.float32)
else:
z = np.zeros(
(len(vars_indexes), len(nodes), len(self.tags['cores'])),
dtype=np.float64)
f.seek(self.tags['cores'][0])
if showbar:
pbar = ProgressBar(maxval=len(self.tags['cores'])).start()
for time in range(len(self.tags['cores'])):
f.seek(self.tags['cores'][time])
f.seek(4 + fsize + 4, 1)
if showbar:
pbar.update(time)
for ivar in range(self.nvar):
f.seek(4, 1)
if ivar in vars_indexes:
jnod = onodes[0]
f.seek(fsize * (jnod[1] - 1), 1)
z[vars_indexes.index(ivar), jnod[0], time] = \
unpack(endian+ftype, f.read(fsize))[0]
for inod in onodes[1:]:
f.seek(fsize * (inod[1] - jnod[1] - 1), 1)
z[vars_indexes.index(ivar), inod[0], time] = \
unpack(endian+ftype, f.read(fsize))[0]
jnod = inod
f.seek(fsize * self.npoin3 - fsize * jnod[1], 1)
else:
f.seek(fsize * self.npoin3, 1)
f.seek(4, 1)
if showbar:
pbar.finish()
return z
def put_content(self, file_name, showbar=True):
# ~~> Output file header
self.slf.fole.update({'hook': open(file_name, 'wb')})
self.slf.fole['name'] = file_name
self.slf.nbv1 = 0
self.slf.nbv2 = 0
self.slf.varnames = []
self.slf.varunits = []
for calc in self.calcs:
for cname, cunit in calc['vars']:
self.slf.varnames.append(cname)
self.slf.varunits.append(cunit)
self.slf.nbv1 += 1
self.slf.append_header_slf()
# ~~> Time stepping
print('\n > Input signal based on:')
print(' - ' + str(self.slf.nvar / 2) + ' direction(s)')
print(' - for the following periods:' +
repr(self.slf.tags['times']))
# ~~> Time stepping
print('\n > Going through time ('+str(len(self.history))+\
'time steps) :')
pbar = ProgressBar(maxval=len(self.history)).start()
for t_i in range(len(self.history)):
ttime = self.history[t_i]
# ~~> Initialise vari to 0 for each time step
vari = []
t_0 = self.slf.tags['times'][0]
for calc, icalc in zip(self.calcs, range(len(self.calcs))):
fct, args = calc['init']
vari.append(fct(vari, args, t_0, t_0))
# ~~> Sweeps through wave periods, adding up all components of
# free surface into "vari" for a given time
# in this particular case self.slf.tags['times'] holds
# wave periods, not times
for t_p in range(len(self.slf.tags['times'])):
vrs = self.get_palues(t_p)
for icalc, calc in enumerate(self.calcs):
fct, args = calc['calc']
times = self.slf.tags['times']
vari[icalc] = fct(vrs, args, ttime, times[t_p],
times[len(times) - 1], vari[icalc])
# ~~> Print time record
self.slf.append_core_time_slf(float(ttime))
for icalc, calc in enumerate(self.calcs):
fct = calc['stop']
self.slf.append_core_vars_slf(fct(t_0, ttime, vari[icalc]))
pbar.update(t_i)
pbar.finish()
self.slf.fole['hook'].close()
def putFileContent(file,lines):
if path.exists(file): remove(file)
SrcF = open(file,'wb')
if len(lines)>0:
ibar = 0; pbar = ProgressBar(maxval=len(lines)).start()
SrcF.write((lines[0].rstrip()).replace('\r','').replace('\n\n','\n'))
for line in lines[1:]:
pbar.update(ibar); ibar += 1
SrcF.write('\n'+(line.rstrip()).replace('\r','').replace('\n\n','\n'))
pbar.finish()
SrcF.close()
return
def append_core_vars_grib(self, itime):
if self.typ == 'wave':
pass
# ~~> WAVE HEIGHT == 'swh'
# ~~> SIGNIFICANT WAVE PERIOD == 'mwp'
# ~~> MEAN WAVE DIRECTION == 'mwd'
elif self.typ == 'oper':
var2d = np.zeros((self.slf2d.nvar, self.slf2d.npoin2),
dtype=np.float)
grbs = pygrib.open(self.dataset[itime])
for grb in grbs:
if grb.indicatorOfParameter in self.variables:
jvar = self.variables.index(grb.indicatorOfParameter)
var2d[jvar, :] = np.ravel(grb.values.T)
grbs.close()
for jvar in range(self.slf2d.nvar):
self.slf2d.append_core_vars_slf([var2d[jvar, :]])
elif self.typ == 'spec':
spec = np.zeros(
(self.nb_direct, self.nb_freq, self.nx1d, self.ny1d),
dtype=np.float)
grbs = pygrib.open(self.dataset[itime])
ibar = 0
maxval = self.nb_direct * self.nb_freq
pbar = ProgressBar(maxval=maxval).start()
for grb in grbs:
i_i = 0
data = grb.values.data
data[np.where(np.absolute(data) <= 0.001)] = np.nan
data[np.where(data == self.missing_value)] = np.nan
data = 10.**data
data[np.isnan(data)] = 0.
for i_y in range(len(self.masky)):
if self.masky[i_y]:
spec[grb.directionNumber-1, grb.frequencyNumber-1, :,
i_i] = \
data[i_y][self.maskx]
i_i += 1
ibar += 1
pbar.update(ibar)
pbar.finish()
grbs.close()
for i_x in range(self.nx1d):
for i_y in range(self.ny1d):
self.slf2d.append_core_vars_slf([np.ravel\
(spec[:, :, i_x, i_y].T)])
def train_epoch(self, data_loader):
pbar = ProgressBar(n_total=len(data_loader), desc='Training')
tr_loss = AverageMeter()
for step, batch in enumerate(data_loader):
self.model.train()
batch = tuple(t.to(self.device) for t in batch)
input_ids, input_mask, trigger_mask, segment_ids, label_ids, input_lens, one_hot_labels = batch
if not self.partial:
one_hot_labels = None
if not self.trigger:
trigger_mask = None
input_lens = input_lens.cpu().detach().numpy().tolist()
_, loss, = self.model(input_ids, segment_ids, input_mask,
trigger_mask, label_ids, input_lens,
one_hot_labels)
if len(self.n_gpu.split(",")) >= 2:
loss = loss.mean()
if self.gradient_accumulation_steps > 1:
loss = loss / self.gradient_accumulation_steps
loss.backward()
clip_grad_norm_(self.model.parameters(), self.grad_clip)
if (step + 1) % self.gradient_accumulation_steps == 0:
self.optimizer.step()
self.optimizer.zero_grad()
self.global_step += 1
tr_loss.update(loss.item(), n=1)
self.tb_logger.scalar_summary("loss", tr_loss.avg,
self.global_step)
pbar(step=step, info={'loss': loss.item()})
# if step%5==0:
# self.logger.info("step:{},loss={:.4f}".format(self.global_step,loss.item()))
info = {'loss': tr_loss.avg}
if "cuda" in str(self.device):
torch.cuda.empty_cache()
return info
def removeDuplilines(self):
ibar = 0
pbar = ProgressBar(maxval=self.npoin).start()
ip = 0
while ip < len(self.poly):
ibar += len(self.poly[ip])
p, t = removeDuplilines(self.poly[ip], self.type[ip])
pbar.trace() # /!\ the call requires sub-progress bar
if len(p) > 1:
pbar.maxval -= len(self.poly[ip])
ibar -= len(self.poly[ip])
self.poly.pop(ip)
self.type.pop(ip)
for po, to in zip(p, t):
pbar.maxval += len(po)
self.poly.append(po)
self.type.append(to)
else:
ip += 1
pbar.update(ibar)
pbar.finish()
return self.poly, self.type
def copy_files(src, dest):
"""
Copy all the files within src
@param src (string) source directory
@param dest (string) target directory
"""
l_d = listdir(src)
ibar = 0
pbar = ProgressBar(maxval=len(l_d)).start()
for f in l_d:
if path.isfile(path.join(src, f)):
shutil.copy(path.join(src, f), dest)
pbar.update(ibar)
ibar += 1
pbar.finish()
def put_content(self, file_name):
self.slf.fole.update({'hook': open(file_name, 'wb')})
self.slf.file['name'] = file_name
pbar = ProgressBar(maxval=len(self.slf.tags['times'])).start()
self.slf.append_header_slf()
# ~~> Time stepping
for itime in range(len(self.slf.tags['times'])):
self.slf.append_core_time_slf(itime)
vrs = self.get_palues(itime)
self.slf.append_core_vars_slf(vrs)
for fct, args in self.calcs:
self.slf.append_core_vars_slf(fct(vrs, args))
pbar.update(itime)
pbar.finish()
self.slf.fole['hook'].close()
def put_content(self, file_name, showbar=True):
# ~~> Sweep through time steps, saving "vari"
vari = []
initiate = True
if showbar:
pbar = ProgressBar(maxval=len(self.slf.tags['times'])).start()
t_0 = self.slf.tags['times'][0]
for itime in range(len(self.slf.tags['times'])):
vrs = self.get_palues(itime)
if initiate:
for icalc, calc in enumerate(self.calcs):
fct, args = calc['init']
vari.append(
fct(vrs, args, t_0, self.slf.tags['times'][itime]))
initiate = False
else:
for icalc, calc in enumerate(self.calcs):
fct, args = calc['calc']
vari[icalc] = fct(vrs, args, t_0,
self.slf.tags['times'][itime],
vari[icalc])
if showbar:
pbar.update(itime)
if showbar:
pbar.finish()
# ~~> Header
self.slf.fole.update({'hook': open(file_name, 'wb')})
self.slf.fole['name'] = file_name
self.slf.nbv1 = 0
self.slf.nbv2 = 0
self.slf.varnames = []
self.slf.varunits = []
for calc in self.calcs:
for cname, cunit in calc['vars']:
self.slf.varnames.append(cname)
self.slf.varunits.append(cunit)
self.slf.nbv1 += 1
self.slf.append_header_slf()
# ~~> Core
# TODO: writing only for first timestep
self.slf.append_core_time_slf(0)
for icalc, calc in enumerate(self.calcs):
fct = calc['stop']
itime = t_0
self.slf.append_core_vars_slf(fct(t_0, itime, vari[icalc]))
self.slf.fole['hook'].close()
def add_file_content(fle, lines):
"""
Add line to file
@param fle (string) file
@param lines (string) adding line
"""
src_file = open(fle, 'ab')
ibar = 0
pbar = ProgressBar(maxval=len(lines)).start()
for line in lines[0:]:
ibar += 1
pbar.update(ibar)
src_file.write(bytes('\n'+(line.rstrip()).replace('\r', '')\
.replace('\n\n', '\n'),
'utf-8'))
pbar.finish()
src_file.close()
return
def put_content(self, file_name, showbar=True):
"""
Write content of the object into a Serafin file
@param file_name (string) Name of the serafin file
@param showbar (boolean) If True displays a showbar
"""
self.fole.update({'name': file_name})
self.fole.update({'hook': open(file_name, 'wb')})
ibar = 0
if showbar:
pbar = ProgressBar(maxval=len(self.tags['times'])).start()
self.append_header_slf()
for time in range(len(self.tags['times'])):
ibar += 1
self.append_core_time_slf(time)
self.append_core_vars_slf(self.get_values(time))
if showbar:
pbar.update(ibar)
self.fole['hook'].close()
if showbar:
pbar.finish()
def create_sample_dump(self, articles=None):
""" Creates smaller sample dump from large dump of given language
:articles: list/set of unicodes [optional]
Titles of articles you want to appear in sample. Number of titles
is arbitrary, if there are too many of them, some will be ommited,
if there are too few, smaller dump will be created and message
will be displayed.
"""
# TODO: check that all items of articles are unicodes, not just str
# find parent dump
parent = self.get_parent_corpus()
# select sample path
sample_path = self.get_dump_path()
# find the namespace
namespace = parent.get_namespace()
# articles specified
if articles:
specific_sample = True
articles = set(articles)
else:
specific_sample = False
# create qualified names (= names with namespaces) for tags we need
TEXT_TAG = qualified_name('text', namespace)
TITLE_TAG = qualified_name('title', namespace)
REDIRECT_TAG = qualified_name('redirect', namespace)
NS_TAG = qualified_name('ns', namespace)
# iterate through xml and build a sample file
with parent._open_dump() as dump_file:
context = etree.iterparse(dump_file, events=('end',))
# create root under which we will add sample articles
sample_root = etree.Element('mediawiki', nsmap={None: namespace})
skip = True
pages = 0
last_title = None
if specific_sample:
# in case of specific sample, it's easily possible
# that we will need to go through the whole dump
# -> meassure progress as a ratio of processed part
progressbar = ProgressBar(parent.get_dump_length())
else:
progressbar = ProgressBar(self.sample_size())
for event, elem in context:
if elem.tag == REDIRECT_TAG:
# ignore redirect pages
skip = True
elif elem.tag == NS_TAG:
last_ns = elem.text
if elem.text != WikiCorpus.ARTICLE_NS:
skip = True
elif elem.tag == TITLE_TAG:
# remember the title
last_title = elem.text
elif elem.tag == TEXT_TAG:
if skip:
skip = False
continue
# if articles are not specified, take any article,
# if they are specified, check if this is wanted article
if not specific_sample or last_title in articles:
# build page node with title and text subelements
page_node = etree.Element('page')
title_node = etree.SubElement(page_node, 'title')
title_node.text = last_title
ns_node = etree.SubElement(page_node, 'ns')
ns_node.text = last_ns
page_node.append(deepcopy(elem)) # text
# append this node to sample articles
sample_root.append(page_node)
pages += 1
if specific_sample:
articles.remove(last_title)
if pages == self.sample_size():
break
# progress update
if specific_sample:
progressbar.update(dump_file.tell())
else:
progressbar.update(pages)
# cleanup
elem.clear()
#while elem.getprevious() is not None:
# del elem.getparent()[0]
for ancestor in elem.xpath('ancestor-or-self::*'):
while ancestor.getprevious() is not None:
del ancestor.getparent()[0]
del context
progressbar.finish()
# check if sample is of required size
if pages < self.sample_size():
logging.warning('Failed to create sample of {n} pages.'.format(
n=self.sample_size()))
if articles:
logging.warning('Following articles not found:' +
'\n'.join(['- ' + title for title in articles]))
# write sample xml to file
with open(sample_path, 'w') as sample_file:
sample_file.write(etree.tostring(sample_root,
pretty_print=True,
xml_declaration=True))
logging.info('Sample of {pages} pages created at: {path}'.format(
pages=pages, path=sample_path))
