def downloadECMWF(self): result = self.connection.result() fileName = self.request.get("target") # ~> tries connecting 3 times before stopping tries = 0 while True: # ~> downloading file by blocks http = urllib2.urlopen(result["href"]) f = open(fileName, "wb") ibar = 0 pbar = ProgressBar(maxval=result["size"]).start() while True: chunk = http.read(1024 * 1024) if not chunk: break f.write(chunk) ibar += len(chunk) pbar.update(ibar) f.flush() f.close() pbar.finish() # ~> have I got everything ? if ibar == result["size"]: break if tries == 3: print " ... exhausted the number of download trials.\nYou may wish to attempt this again later." sys.exit() print " ... trying to download the data once more ..." tries += 1
def put_content(self, file_name, showbar=True): self.open_grib(file_name) print(' +> Write Selafin header') self.append_header_grib() print(' +> Write Selafin core') if showbar: pbar = ProgressBar(maxval=len(self.dataset)).start() for itime in range(len(self.dataset)): seconds = int(self.slf2d.tags['times'][itime]) date = (datetime(*self.slf2d.datetime) + timedelta(seconds=seconds)).timetuple()[0:6] print(" - {}-{}-{} {}:{}:{}".format(date[2], date[1], date[0], date[3], date[4], date[5])) self.append_core_time_grib(itime) self.append_core_vars_grib(itime) if showbar: pbar.update(itime) if showbar: pbar.finish() self.close_grib()
def get_edges_slf(ikle, meshx, meshy, showbar=True): """ Returns the list of edges of the mesh @param ikle (np.array) Connectivity table @param meshx (np.array) X coordinates of the mesh points @param meshy (np.array) Y coordinates of the mesh points @param showbar (boolean) If True display a progress bar @returns (list) The list of edges """ try: from matplotlib.tri import Triangulation edges = Triangulation(meshx, meshy, ikle).get_cpp_triangulation()\ .get_edges() except ImportError: edges = [] ibar = 0 if showbar: pbar = ProgressBar(maxval=len(ikle)).start() for elem in ikle: ibar += 1 if showbar: pbar.update(ibar) if [elem[0], elem[1]] not in edges: edges.append([elem[1], elem[0]]) if [elem[1], elem[2]] not in edges: edges.append([elem[2], elem[1]]) if [elem[2], elem[0]] not in edges: edges.append([elem[0], elem[2]]) if showbar: pbar.finish() return edges
def fit(self, x, y, steps=0, batch_size=32): batch_size = 32 if batch_size == None else batch_size num_batches = x.shape[0] // batch_size for i, p in enumerate(self.particles): local_score = p.get_score(x, y) if local_score < self.global_best_score: self.global_best_score = local_score self.global_best_weights = p.get_best_weights() print("PSO -- Initial best score {:0.4f}".format(self.global_best_score)) bar = ProgressBar(steps, updates=20) for i in range(steps): for j in range(num_batches): x_ = x[j*batch_size:(j+1)*batch_size,:] y_ = y[j*batch_size:(j+1)*batch_size] for p in self.particles: local_score = p.step(x_, y_, self.global_best_weights) if local_score < self.global_best_score: self.global_best_score = local_score self.global_best_weights = p.get_best_weights() bar.update(i) bar.done()
def big2littlePTS(self, fileName, foleName): # ~~ Openning files ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ fle = open(fileName, 'rb') fole = open(foleName, 'wb') print ' +> writing the volumes-file: ', foleName # ~~ Read/Write dimensions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ n1 = self.NPOIN3 + 1 minvol = self.minvol * np.ones(self.NPOIN3, dtype=np.float32) # ~~ Read volumes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ pbar = ProgressBar(maxval=self.HYDROIT).start() for i in range(self.HYDROIT): l, it = unpack('>ii', fle.read(4 + 4)) VOLUME = np.asarray( unpack('>' + str(self.NPOIN3) + 'f', fle.read(4 * self.NPOIN3))) VOLUME = np.maximum(VOLUME, minvol) fle.seek(4, 1) if it >= self.tfrom and it <= self.tstop: pbar.write(' ~> read iteration: ' + str(it), i) fole.write(pack('<ii', 4 * n1, it - self.HYDRO00)) fole.write(pack('<' + str(self.NPOIN3) + 'f', *(VOLUME))) fole.write(pack('<i', 4 * n1)) else: pbar.write(' ~> ignore iteration: ' + str(it), i) pbar.update(i) pbar.finish() fle.close() fole.close()
def put_file_content(fle, lines): """ put line to file @param fle (string) file @param lines (string) adding line """ if path.exists(fle): remove(fle) src_file = open(fle, 'wb') if len(lines) > 0: ibar = 0 pbar = ProgressBar(maxval=len(lines)).start() src_file.write(bytes((lines[0].rstrip()).replace('\r', '')\ .replace('\n\n', '\n'), 'utf-8')) for line in lines[1:]: pbar.update(ibar) ibar += 1 src_file.write(bytes('\n'+(line.rstrip()).replace('\r', '')\ .replace('\n\n', '\n'), 'utf-8')) pbar.finish() src_file.close() return
def big2littleARE(self, fileName, foleName): # ~~ Openning files ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ fle = open(fileName, 'rb') fole = open(foleName, 'wb') print ' +> writing the areas-file: ', foleName # ~~ Read/Write dimensions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ NSEG2 = (3 * self.geo.NELEM3 + self.conlim.NPTFR) / 2 MBND2 = np.count_nonzero(self.conlim.BOR['lih'] != 2) n3 = (NSEG2 + MBND2) #n4 = 2*( self.geo.NPLAN-1 )*self.geo.NPOIN3 # ~~ Read volumes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ pbar = ProgressBar(maxval=self.HYDROIT).start() for i in range(self.HYDROIT): l, it = unpack('>ii', fle.read(4 + 4)) AREAS = np.asarray(unpack('>' + str(n3) + 'f', fle.read(4 * n3))) fle.seek(4, 1) if it >= self.tfrom and it <= self.tstop: pbar.write(' ~> read iteration: ' + str(it), i) fole.write(pack('<ii', 4 * n3, it - self.HYDRO00)) fole.write(pack('<' + str(n3) + 'f', *(AREAS))) fole.write(pack('<i', 4 * n3)) else: pbar.write(' ~> ignore iteration: ' + str(it), i) pbar.update(i) pbar.finish() fle.close() fole.close()
def big2little_are(self, file_name, fole_name): # ~~ Openning files ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ fle = open(file_name, 'rb') fole = open(fole_name, 'wb') print(' +> writing the areas-file: ' + fole_name) # ~~ Read/Write dimensions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ nseg2 = (3 * self.geo.nelem3 + self.conlim.nptfr) / 2 mbnd2 = np.count_nonzero(self.conlim.bor['lih'] != 2) n_3 = (nseg2 + mbnd2) # ~~ Read volumes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ pbar = ProgressBar(maxval=self.hydroit).start() for i in range(self.hydroit): _, itime = unpack('>ii', fle.read(4 + 4)) areas = np.asarray(unpack('>' + str(n_3) + 'f', fle.read(4 * n_3))) fle.seek(4, 1) if itime >= self.tfrom and itime <= self.tstop: pbar.write(' ~> read iteration: ' + str(itime), i) fole.write(pack('<ii', 4 * n_3, itime - self.hydro00)) fole.write(pack('<' + str(n_3) + 'f', *(areas))) fole.write(pack('<i', 4 * n_3)) else: pbar.write(' ~> ignore iteration: ' + str(itime), i) pbar.update(i) pbar.finish() fle.close() fole.close()
def big2little_pts(self, file_name, fole_name): # ~~ Openning files ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ fle = open(file_name, 'rb') fole = open(fole_name, 'wb') print(' +> writing the volumes-file: ' + fole_name) # ~~ Read/Write dimensions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ n_1 = self.npoin3 + 1 minvol = self.minvol * np.ones(self.npoin3, dtype=np.float32) # ~~ Read volumes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ pbar = ProgressBar(maxval=self.hydroit).start() for i in range(self.hydroit): _, itime = unpack('>ii', fle.read(4 + 4)) volume = np.asarray( unpack('>' + str(self.npoin3) + 'f', fle.read(4 * self.npoin3))) volume = np.maximum(volume, minvol) fle.seek(4, 1) if itime >= self.tfrom and itime <= self.tstop: pbar.write(' ~> read iteration: ' + str(itime), i) fole.write(pack('<ii', 4 * n_1, itime - self.hydro00)) fole.write(pack('<' + str(self.npoin3) + 'f', *(volume))) fole.write(pack('<i', 4 * n_1)) else: pbar.write(' ~> ignore iteration: ' + str(itime), i) pbar.update(i) pbar.finish() fle.close() fole.close()
def removeDuplangles(self): ibar = 0 pbar = ProgressBar(maxval=self.npoin).start() ip = 0 while ip < len(self.poly): ibar += len(self.poly[ip]) lb = len(self.poly[ip]) self.poly[ip], self.type[ip] = removeDuplangles( self.poly[ip], self.type[ip]) la = len(self.poly[ip]) if la < lb: pbar.write( ' +> removed ' + str(lb - la) + ' points of ' + str(lb) + ' from polygon ' + str(ip + 1), ibar) if self.poly[ip] == []: pbar.write(' +> removed entire polygon ' + str(ip + 1), ibar) self.poly.pop(ip) self.type.pop(ip) else: ip += 1 pbar.update(ibar) pbar.finish() return self.poly, self.type
def putContent(self, fileName, showbar=True): # ~~> netcdf reader ecmwfdata = netcdf.netcdf_file(self.request.get("target"), 'r') # ~~> new SELAFIN writer self.slf2d.fole = {} self.slf2d.fole.update({'hook': open(fileName, 'wb')}) self.slf2d.fole.update({'name': fileName}) self.slf2d.fole.update({'endian': ">"}) # big endian self.slf2d.fole.update({'float': ('f', 4)}) # single precision print ' +> Write SELAFIN header' self.appendHeaderECMWF(ecmwfdata) print ' +> Write SELAFIN core' ibar = 0 if showbar: pbar = ProgressBar(maxval=len(self.slf2d.tags['times'])).start() for t in range(len(self.slf2d.tags['times'])): self.appendCoreTimeECMWF(t) self.appendCoreVarsECMWF(ecmwfdata, ibar) ibar += 1 if showbar: pbar.update(ibar) self.slf2d.fole['hook'].close() if showbar: pbar.finish()
def get_neighbours_slf(ikle, meshx, meshy, showbar=True): """ Return a list containing for each element the list of elements that are neighbours to that element @param ikle (np.array) Connectivity table @param meshx (np.array) X coordinates of the mesh points @param meshy (np.array) Y coordinates of the mesh points @param showbar (boolean) If True display a progress bar @returns (list) The list of neighbours @param """ try: from matplotlib.tri import Triangulation neighbours = Triangulation(meshx, meshy, ikle).get_cpp_triangulation()\ .get_neighbors() except ImportError: insiders = {} bounders = {} ibar = 0 if showbar: pbar = ProgressBar(maxval=(3 * len(ikle))).start() for elem, i in zip(ikle, range(len(ikle))): n_k = bounders.keys() for k in [0, 1, 2]: ibar += 1 if showbar: pbar.update(ibar) if (elem[k], elem[(k + 1) % 3]) not in n_k: bounders.update({(elem[(k + 1) % 3], elem[k]): i}) else: j = bounders[(elem[k], elem[(k + 1) % 3])] insiders.update({(elem[k], elem[(k + 1) % 3]): [i, j]}) del bounders[(elem[k], elem[(k + 1) % 3])] ibar = 0 neighbours = -np.ones((len(ikle), 3), dtype=np.int) for elem, i in zip(ikle, range(len(ikle))): for k in [0, 1, 2]: ibar += 1 if showbar: pbar.update(ibar) if (elem[k], elem[(k + 1) % 3]) in insiders: elem_a, elem_b = insiders[(elem[k], elem[(k + 1) % 3])] if elem_a == i: neighbours[i][k] = elem_b if elem_b == i: neighbours[i][k] = elem_a if (elem[(k + 1) % 3], elem[k]) in insiders: elem_a, elem_b = insiders[(elem[(k + 1) % 3], elem[k])] if elem_a == i: neighbours[i][k] = elem_b if elem_b == i: neighbours[i][k] = elem_a if showbar: pbar.finish() return neighbours
def copyFiles(pi,po): ld = listdir(pi) ibar = 0; pbar = ProgressBar(maxval=len(ld)).start() for f in ld: if path.isfile(path.join(pi,f)): shutil.copy(path.join(pi,f),po) pbar.update(ibar); ibar += 1 pbar.finish() return
def copyFiles(pi,po): ld = listdir(pi) ibar = 0; pbar = ProgressBar(maxval=len(ld)).start() for f in ld: if path.isfile(path.join(pi,f)): shutil.copy(path.join(pi,f),po) pbar.update(ibar); ibar += 1 pbar.finish() return
def addFileContent(file,lines): SrcF = open(file,'ab') ibar = 0; pbar = ProgressBar(maxval=len(lines)).start() for line in lines[0:]: ibar += 1; pbar.update(ibar) SrcF.write('\n'+(line.rstrip()).replace('\r','').replace('\n\n','\n')) pbar.finish() SrcF.close() return
def addFileContent(file,lines): SrcF = open(file,'ab') ibar = 0; pbar = ProgressBar(maxval=len(lines)).start() for line in lines[0:]: ibar += 1; pbar.update(ibar) SrcF.write('\n'+(line.rstrip()).replace('\r','').replace('\n\n','\n')) pbar.finish() SrcF.close() return
def get_series(self, nodes, vars_indexes=None, showbar=True): """ Return the value for a list of nodes on variables given in vars_indexes for each time step @param nodes (list) list of nodes for which to extract data @param vars_indexes (list) List of variables to extract data for @param showbar (boolean) If True display a showbar for the progress """ f = self.file['hook'] endian = self.file['endian'] ftype, fsize = self.file['float'] if vars_indexes is None: vars_indexes = self.varindex # ~~ Ordering the nodes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # This assumes that nodes starts at 1 onodes = np.sort(np.array(zip(range(len(nodes)), nodes), dtype=[('0', int), ('1', int)]), order='1') # ~~ Extract time profiles ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ if fsize == 4: z = np.zeros( (len(vars_indexes), len(nodes), len(self.tags['cores'])), dtype=np.float32) else: z = np.zeros( (len(vars_indexes), len(nodes), len(self.tags['cores'])), dtype=np.float64) f.seek(self.tags['cores'][0]) if showbar: pbar = ProgressBar(maxval=len(self.tags['cores'])).start() for time in range(len(self.tags['cores'])): f.seek(self.tags['cores'][time]) f.seek(4 + fsize + 4, 1) if showbar: pbar.update(time) for ivar in range(self.nvar): f.seek(4, 1) if ivar in vars_indexes: jnod = onodes[0] f.seek(fsize * (jnod[1] - 1), 1) z[vars_indexes.index(ivar), jnod[0], time] = \ unpack(endian+ftype, f.read(fsize))[0] for inod in onodes[1:]: f.seek(fsize * (inod[1] - jnod[1] - 1), 1) z[vars_indexes.index(ivar), inod[0], time] = \ unpack(endian+ftype, f.read(fsize))[0] jnod = inod f.seek(fsize * self.npoin3 - fsize * jnod[1], 1) else: f.seek(fsize * self.npoin3, 1) f.seek(4, 1) if showbar: pbar.finish() return z
def put_content(self, file_name, stream, showbar=True): # ~~> netcdf reader self.typ = stream if self.typ == 'spec': # ~~> new Selafin writer self.slf2d.fole = {} self.slf2d.fole.update({'hook': open(file_name, 'wb')}) self.slf2d.fole.update({'name': file_name}) self.slf2d.fole.update({'endian': ">"}) # big endian self.slf2d.fole.update({'float': ('f', 4)}) # single precision print(' +> Write Selafin header') self.append_header_ecmwf() print(' +> Write Selafin core') ibar = 0 if showbar: pbar = ProgressBar(maxval=len(self.slf2d.tags['times']))\ .start() for itime in range(len(self.slf2d.tags['times'])): self.append_core_time_ecmwf(itime) self.append_core_vars_ecmwf(ibar) ibar += 1 if showbar: pbar.update(ibar) self.slf2d.fole['hook'].close() if showbar: pbar.finish() else: # ~~> new Selafin writer self.slf2d.fole = {} self.slf2d.fole.update({'hook': open(file_name, 'wb')}) self.slf2d.fole.update({'name': file_name}) self.slf2d.fole.update({'endian': ">"}) # big endian self.slf2d.fole.update({'float': ('f', 4)}) # single precision print(' +> Write Selafin header') self.append_header_ecmwf() print(' +> Write Selafin core') ibar = 0 if showbar: pbar = ProgressBar(maxval=len(self.slf2d.tags['times']))\ .start() for itime in range(len(self.slf2d.tags['times'])): self.append_core_time_ecmwf(itime) self.append_core_vars_ecmwf(ibar) ibar += 1 if showbar: pbar.update(ibar) self.slf2d.fole['hook'].close() if showbar: pbar.finish()
def put_content(self, file_name, showbar=True): # ~~> Output file header self.slf.fole.update({'hook': open(file_name, 'wb')}) self.slf.fole['name'] = file_name self.slf.nbv1 = 0 self.slf.nbv2 = 0 self.slf.varnames = [] self.slf.varunits = [] for calc in self.calcs: for cname, cunit in calc['vars']: self.slf.varnames.append(cname) self.slf.varunits.append(cunit) self.slf.nbv1 += 1 self.slf.append_header_slf() # ~~> Time stepping print('\n > Input signal based on:') print(' - ' + str(self.slf.nvar / 2) + ' direction(s)') print(' - for the following periods:' + repr(self.slf.tags['times'])) # ~~> Time stepping print('\n > Going through time ('+str(len(self.history))+\ 'time steps) :') pbar = ProgressBar(maxval=len(self.history)).start() for t_i in range(len(self.history)): ttime = self.history[t_i] # ~~> Initialise vari to 0 for each time step vari = [] t_0 = self.slf.tags['times'][0] for calc, icalc in zip(self.calcs, range(len(self.calcs))): fct, args = calc['init'] vari.append(fct(vari, args, t_0, t_0)) # ~~> Sweeps through wave periods, adding up all components of # free surface into "vari" for a given time # in this particular case self.slf.tags['times'] holds # wave periods, not times for t_p in range(len(self.slf.tags['times'])): vrs = self.get_palues(t_p) for icalc, calc in enumerate(self.calcs): fct, args = calc['calc'] times = self.slf.tags['times'] vari[icalc] = fct(vrs, args, ttime, times[t_p], times[len(times) - 1], vari[icalc]) # ~~> Print time record self.slf.append_core_time_slf(float(ttime)) for icalc, calc in enumerate(self.calcs): fct = calc['stop'] self.slf.append_core_vars_slf(fct(t_0, ttime, vari[icalc])) pbar.update(t_i) pbar.finish() self.slf.fole['hook'].close()
def putFileContent(file,lines): if path.exists(file): remove(file) SrcF = open(file,'wb') if len(lines)>0: ibar = 0; pbar = ProgressBar(maxval=len(lines)).start() SrcF.write((lines[0].rstrip()).replace('\r','').replace('\n\n','\n')) for line in lines[1:]: pbar.update(ibar); ibar += 1 SrcF.write('\n'+(line.rstrip()).replace('\r','').replace('\n\n','\n')) pbar.finish() SrcF.close() return
def putFileContent(file,lines): if path.exists(file): remove(file) SrcF = open(file,'wb') if len(lines)>0: ibar = 0; pbar = ProgressBar(maxval=len(lines)).start() SrcF.write((lines[0].rstrip()).replace('\r','').replace('\n\n','\n')) for line in lines[1:]: pbar.update(ibar); ibar += 1 SrcF.write('\n'+(line.rstrip()).replace('\r','').replace('\n\n','\n')) pbar.finish() SrcF.close() return
def makeClockwise(self): ibar = 0 pbar = ProgressBar(maxval=self.npoin).start() for ip in range(len(self.poly)): ibar += len(self.poly[ip]) if self.type[ip] != 0: if not isClockwise(self.poly[ip]): pbar.write( ' +> turned clockwise polygon ' + str(ip + 1), ibar) self.poly[ip] = np.flipud(self.poly[ip]) pbar.update(ibar) pbar.finish() return self.poly
def append_core_vars_grib(self, itime): if self.typ == 'wave': pass # ~~> WAVE HEIGHT == 'swh' # ~~> SIGNIFICANT WAVE PERIOD == 'mwp' # ~~> MEAN WAVE DIRECTION == 'mwd' elif self.typ == 'oper': var2d = np.zeros((self.slf2d.nvar, self.slf2d.npoin2), dtype=np.float) grbs = pygrib.open(self.dataset[itime]) for grb in grbs: if grb.indicatorOfParameter in self.variables: jvar = self.variables.index(grb.indicatorOfParameter) var2d[jvar, :] = np.ravel(grb.values.T) grbs.close() for jvar in range(self.slf2d.nvar): self.slf2d.append_core_vars_slf([var2d[jvar, :]]) elif self.typ == 'spec': spec = np.zeros( (self.nb_direct, self.nb_freq, self.nx1d, self.ny1d), dtype=np.float) grbs = pygrib.open(self.dataset[itime]) ibar = 0 maxval = self.nb_direct * self.nb_freq pbar = ProgressBar(maxval=maxval).start() for grb in grbs: i_i = 0 data = grb.values.data data[np.where(np.absolute(data) <= 0.001)] = np.nan data[np.where(data == self.missing_value)] = np.nan data = 10.**data data[np.isnan(data)] = 0. for i_y in range(len(self.masky)): if self.masky[i_y]: spec[grb.directionNumber-1, grb.frequencyNumber-1, :, i_i] = \ data[i_y][self.maskx] i_i += 1 ibar += 1 pbar.update(ibar) pbar.finish() grbs.close() for i_x in range(self.nx1d): for i_y in range(self.ny1d): self.slf2d.append_core_vars_slf([np.ravel\ (spec[:, :, i_x, i_y].T)])
def copy_files(src, dest): """ Copy all the files within src @param src (string) source directory @param dest (string) target directory """ l_d = listdir(src) ibar = 0 pbar = ProgressBar(maxval=len(l_d)).start() for f in l_d: if path.isfile(path.join(src, f)): shutil.copy(path.join(src, f), dest) pbar.update(ibar) ibar += 1 pbar.finish()
def put_content(self, file_name): self.slf.fole.update({'hook': open(file_name, 'wb')}) self.slf.file['name'] = file_name pbar = ProgressBar(maxval=len(self.slf.tags['times'])).start() self.slf.append_header_slf() # ~~> Time stepping for itime in range(len(self.slf.tags['times'])): self.slf.append_core_time_slf(itime) vrs = self.get_palues(itime) self.slf.append_core_vars_slf(vrs) for fct, args in self.calcs: self.slf.append_core_vars_slf(fct(vrs, args)) pbar.update(itime) pbar.finish() self.slf.fole['hook'].close()
def smoothSubdivise(self, weight=0.5): ibar = 0 pbar = ProgressBar(maxval=self.npoin).start() for ip in range(len(self.poly)): ibar += len(self.poly[ip]) lb = len(self.poly[ip]) self.poly[ip], self.type[ip] = smoothSubdivise( self.poly[ip], self.type[ip], weight) la = len(self.poly[ip]) if la > lb: pbar.write( ' +> added ' + str(la - lb) + ' points to polygon ' + str(ip + 1), ibar) pbar.update(ibar) pbar.finish() return self.poly, self.type
def put_content(self, file_name, showbar=True): # ~~> Sweep through time steps, saving "vari" vari = [] initiate = True if showbar: pbar = ProgressBar(maxval=len(self.slf.tags['times'])).start() t_0 = self.slf.tags['times'][0] for itime in range(len(self.slf.tags['times'])): vrs = self.get_palues(itime) if initiate: for icalc, calc in enumerate(self.calcs): fct, args = calc['init'] vari.append( fct(vrs, args, t_0, self.slf.tags['times'][itime])) initiate = False else: for icalc, calc in enumerate(self.calcs): fct, args = calc['calc'] vari[icalc] = fct(vrs, args, t_0, self.slf.tags['times'][itime], vari[icalc]) if showbar: pbar.update(itime) if showbar: pbar.finish() # ~~> Header self.slf.fole.update({'hook': open(file_name, 'wb')}) self.slf.fole['name'] = file_name self.slf.nbv1 = 0 self.slf.nbv2 = 0 self.slf.varnames = [] self.slf.varunits = [] for calc in self.calcs: for cname, cunit in calc['vars']: self.slf.varnames.append(cname) self.slf.varunits.append(cunit) self.slf.nbv1 += 1 self.slf.append_header_slf() # ~~> Core # TODO: writing only for first timestep self.slf.append_core_time_slf(0) for icalc, calc in enumerate(self.calcs): fct = calc['stop'] itime = t_0 self.slf.append_core_vars_slf(fct(t_0, itime, vari[icalc])) self.slf.fole['hook'].close()
def add_file_content(fle, lines): """ Add line to file @param fle (string) file @param lines (string) adding line """ src_file = open(fle, 'ab') ibar = 0 pbar = ProgressBar(maxval=len(lines)).start() for line in lines[0:]: ibar += 1 pbar.update(ibar) src_file.write(bytes('\n'+(line.rstrip()).replace('\r', '')\ .replace('\n\n', '\n'), 'utf-8')) pbar.finish() src_file.close() return
def removeDuplilines(self): ibar = 0 pbar = ProgressBar(maxval=self.npoin).start() ip = 0 while ip < len(self.poly): ibar += len(self.poly[ip]) p, t = removeDuplilines(self.poly[ip], self.type[ip]) pbar.trace() # /!\ the call requires sub-progress bar if len(p) > 1: pbar.maxval -= len(self.poly[ip]) ibar -= len(self.poly[ip]) self.poly.pop(ip) self.type.pop(ip) for po, to in zip(p, t): pbar.maxval += len(po) self.poly.append(po) self.type.append(to) else: ip += 1 pbar.update(ibar) pbar.finish() return self.poly, self.type
def put_content(self, file_name, showbar=True): """ Write global content into file_name @param file_name (string) Name of the output file @param showbar (boolean) display a showbar (default=True) """ self.slf.fole.update({'hook':open(file_name, 'wb')}) ibar = 0 if showbar: pbar = ProgressBar(maxval=len(self.slf.tags['times'])).start() self.slf.append_header_slf() # ~~> Time stepping for itime in range(len(self.slf.tags['times'])): ibar += 1 self.slf.append_core_time_slf(itime) # Time stamps self.slf.append_core_vars_slf(self.get_palues(itime)) if showbar: pbar.update(ibar) self.slf.fole['hook'].close() if showbar: pbar.finish()
def put_geometry(self, file_name): print(' +> writing up the geometry file') self.slf2d.fole = {} self.slf2d.fole.update({'hook':open(file_name, 'wb')}) self.slf2d.fole.update({'name':file_name}) self.slf2d.fole.update({'endian':">"}) # big endian self.slf2d.fole.update({'float':('f', 4)}) # single precision self.slf2d.varnames = ['RANGE '] self.slf2d.varunits = ['UI '] self.slf2d.nbv1 = len(self.slf2d.varnames) self.slf2d.nvar = self.slf2d.nbv1 self.slf2d.varindex = range(self.slf2d.nvar) print(' - Write Selafin geometry') self.slf2d.append_header_slf() print(' - Write Selafin core') varof = self.ecmwfdata.variables['d2fd'].add_offset varsf = self.ecmwfdata.variables['d2fd'].scale_factor var2d = np.zeros((self.nx1d, self.ny1d), dtype=np.float) ibar = 0 pbar = ProgressBar(maxval=len(self.slf2d.tags['times'])).start() for itime in range(len(self.slf2d.tags['times'])): self.slf2d.append_core_time_slf(itime) var = self.ecmwfdata.variables['d2fd'][itime] z = 10 ** (varsf * np.swapaxes(np.swapaxes(var, 1, 3), 0, 2) + \ varof) for j in range(self.ny1d): for i in range(self.nx1d): var2d[i, j] = max(z[j][i].ravel()) self.slf2d.append_core_vars_slf([var2d.ravel()]) ibar += 1 pbar.update(ibar) pbar.finish() self.slf2d.fole['hook'].close()
def put_content(self, file_name, showbar=True): """ Write content of the object into a Serafin file @param file_name (string) Name of the serafin file @param showbar (boolean) If True displays a showbar """ self.fole.update({'name': file_name}) self.fole.update({'hook': open(file_name, 'wb')}) ibar = 0 if showbar: pbar = ProgressBar(maxval=len(self.tags['times'])).start() self.append_header_slf() for time in range(len(self.tags['times'])): ibar += 1 self.append_core_time_slf(time) self.append_core_vars_slf(self.get_values(time)) if showbar: pbar.update(ibar) self.fole['hook'].close() if showbar: pbar.finish()
def __init__(self, fname, vals=(None, None)): """ @brief Main class that fetches the GEBCO bathymetry data and puts it into the SELAFIN data format @param fname (string): the name of the GEBCO file """ # ~~> empty SELAFIN Selafin.__init__(self, '') self.datetime = [] # ~~> variables self.title = '' self.nbv1 = 1 # bathymetry only self.nvar = self.nbv1 self.varindex = range(self.nvar) self.varnames = ['BOTTOM '] self.varunits = ['M '] print(' +> header') # ~~> load header (ASC type) gebcofile = open(fname, 'r') # ~~ gline = [] gline.append(gebcofile.readline().split()) if gline[-1][0].lower() == "ncols": nx1d = int(gline[-1][1]) else: raise TelemacException(\ '.. Could not read this file format. ' 'Key ncols expected here.') gline.append(gebcofile.readline().split()) if gline[-1][0].lower() == "nrows": ny1d = int(gline[-1][1]) else: raise TelemacException(\ '.. Could not read this file format. ' 'Key nrows expected here.') gline.append(gebcofile.readline().split()) if gline[-1][0].lower() == "xllcorner": xllcorner = np.float(gline[-1][1]) else: raise TelemacException(\ '.. Could not read this file format. ' 'Key xllcorner expected here.') gline.append(gebcofile.readline().split()) if gline[-1][0].lower() == "yllcorner": yllcorner = np.float(gline[-1][1]) else: raise TelemacException(\ '.. Could not read this file format. ' 'Key yllcorner expected here.') gline.append(gebcofile.readline().split()) if gline[-1][0].lower() == "cellsize": xdim = np.float(gline[-1][1]) ydim = xdim elif gline[-1][0].lower() in ["xdim", "dx"]: xdim = np.float(gline[-1][1]) gline.append(gebcofile.readline().split()) if gline[-1][0].lower() in ["ydim", "dy"]: ydim = np.float(gline[-1][1]) else: raise TelemacException(\ '.. Could not read this file format.' ' Key ydim expected here.') else: raise TelemacException(\ '.. Could not read this file format. ' 'Key cellsize or xdim expected here.') gline.append(gebcofile.readline().split()) if gline[-1][0].lower() == "nodata_value": nodata_value = int(gline[-1][1]) else: raise TelemacException(\ '.. Could not read this file format. ' 'Key NODATA_value expected here.') # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ gebcofile.close() print(' +> bathymetry') # ~~> load ASCII content, ignoring the header lines z = np.loadtxt(fname, skiprows=len(gline)).T.ravel() print(' +> filtered connectivity') # ~~> temporary ikle aval = min(z) - 1 if vals[0] != None: aval = float(vals[0]) bval = max(z) + 1 if vals[1] != None: bval = float(vals[1]) ielem = 0 pbar = ProgressBar(maxval=2*(nx1d-1)*(ny1d-1)).start() ikle3 = - np.ones((2*(nx1d-1)*(ny1d-1), 3), dtype=np.int) for i in range(1, nx1d): for j in range(1, ny1d): ipoin = (i-1)*ny1d + j - 1 # ~~> first triangle if (aval < z[ipoin] < bval) and \ (aval < z[ipoin + ny1d] < bval) and \ (aval < z[ipoin + 1] < bval): ikle3[ielem] = [ipoin, ipoin + 1, ipoin + ny1d] ielem = ielem + 1 pbar.update(ielem) # ~~> second triangle if (aval < z[ipoin + ny1d] < bval) and \ (aval < z[ipoin + ny1d + 1] < bval) and \ (aval < z[ipoin + 1] < bval): ikle3[ielem] = [ipoin + ny1d, ipoin + 1, ipoin + ny1d + 1] ielem = ielem + 1 pbar.update(ielem) pbar.finish() print(' +> renumbered connectivity') # ~~> intermediate connectivity gikle = ikle3[np.not_equal(*(np.sort(ikle3).T[0::2]))] knolg = np.unique(np.ravel(gikle)) knogl = dict(zip(knolg, range(len(knolg)))) # ~~> final connectivity self.ikle3 = - np.ones_like(gikle, dtype=np.int) pbar = ProgressBar(maxval=len(gikle)).start() for k in range(len(gikle)): self.ikle3[k] = [knogl[gikle[k][0]], \ knogl[gikle[k][1]], \ knogl[gikle[k][2]]] pbar.update(k) pbar.finish() print(' +> mesh x,y,z') # ~~> defines grid x = xllcorner + xdim * np.arange(nx1d, dtype=np.float) - xdim/2. y = yllcorner - ydim * np.arange(ny1d, dtype=np.float) + \ ydim * ny1d - ydim/2. self.meshx = np.tile(x, ny1d).reshape(ny1d, nx1d).T.ravel()[knolg] self.meshy = np.tile(y, nx1d)[knolg] self.z = z[knolg] print(' +> sizes') # ~~> sizes self.nplan = 1 self.ndp2 = 3 self.ndp3 = self.ndp2 self.npoin2 = len(self.meshx) self.npoin3 = self.npoin2 self.nelem2 = len(self.ikle3) self.nelem3 = self.nelem2 self.iparam = [0, 0, 0, 0, 0, 0, 1, 0, 0, 0] print(' +> boundaries') # ~~> establish neighborhood neighbours = Triangulation(self.meshx, self.meshy, self.ikle3)\ .get_cpp_triangulation().get_neighbors() # ~~> build the enssemble of boundary segments ebounds = [] print(' - identify') pbar = ProgressBar(maxval=self.nelem3).start() for i in range(self.nelem3): if neighbours[i, 0] < 0: ebounds.append([self.ikle3[i][0], self.ikle3[i][1]]) if neighbours[i, 1] < 0: ebounds.append([self.ikle3[i][1], self.ikle3[i][2]]) if neighbours[i, 2] < 0: ebounds.append([self.ikle3[i][2], self.ikle3[i][0]]) pbar.update(i) pbar.finish() # ~~> assemble the enssemble of boundary segments print(' - assemble') pbounds = polygons.join_segments(ebounds) # ~~> define ipobO from an arbitrary start point print(' - set') self.ipob3 = np.zeros(self.npoin3, dtype=np.int) iptfr = 0 for bound in pbounds: for n in bound[1:]: iptfr += 1 self.ipob3[n] = iptfr self.ipob2 = self.ipob3
def create_sample_dump(self, articles=None): """ Creates smaller sample dump from large dump of given language :articles: list/set of unicodes [optional] Titles of articles you want to appear in sample. Number of titles is arbitrary, if there are too many of them, some will be ommited, if there are too few, smaller dump will be created and message will be displayed. """ # TODO: check that all items of articles are unicodes, not just str # find parent dump parent = self.get_parent_corpus() # select sample path sample_path = self.get_dump_path() # find the namespace namespace = parent.get_namespace() # articles specified if articles: specific_sample = True articles = set(articles) else: specific_sample = False # create qualified names (= names with namespaces) for tags we need TEXT_TAG = qualified_name('text', namespace) TITLE_TAG = qualified_name('title', namespace) REDIRECT_TAG = qualified_name('redirect', namespace) NS_TAG = qualified_name('ns', namespace) # iterate through xml and build a sample file with parent._open_dump() as dump_file: context = etree.iterparse(dump_file, events=('end',)) # create root under which we will add sample articles sample_root = etree.Element('mediawiki', nsmap={None: namespace}) skip = True pages = 0 last_title = None if specific_sample: # in case of specific sample, it's easily possible # that we will need to go through the whole dump # -> meassure progress as a ratio of processed part progressbar = ProgressBar(parent.get_dump_length()) else: progressbar = ProgressBar(self.sample_size()) for event, elem in context: if elem.tag == REDIRECT_TAG: # ignore redirect pages skip = True elif elem.tag == NS_TAG: last_ns = elem.text if elem.text != WikiCorpus.ARTICLE_NS: skip = True elif elem.tag == TITLE_TAG: # remember the title last_title = elem.text elif elem.tag == TEXT_TAG: if skip: skip = False continue # if articles are not specified, take any article, # if they are specified, check if this is wanted article if not specific_sample or last_title in articles: # build page node with title and text subelements page_node = etree.Element('page') title_node = etree.SubElement(page_node, 'title') title_node.text = last_title ns_node = etree.SubElement(page_node, 'ns') ns_node.text = last_ns page_node.append(deepcopy(elem)) # text # append this node to sample articles sample_root.append(page_node) pages += 1 if specific_sample: articles.remove(last_title) if pages == self.sample_size(): break # progress update if specific_sample: progressbar.update(dump_file.tell()) else: progressbar.update(pages) # cleanup elem.clear() #while elem.getprevious() is not None: # del elem.getparent()[0] for ancestor in elem.xpath('ancestor-or-self::*'): while ancestor.getprevious() is not None: del ancestor.getparent()[0] del context progressbar.finish() # check if sample is of required size if pages < self.sample_size(): logging.warning('Failed to create sample of {n} pages.'.format( n=self.sample_size())) if articles: logging.warning('Following articles not found:' + '\n'.join(['- ' + title for title in articles])) # write sample xml to file with open(sample_path, 'w') as sample_file: sample_file.write(etree.tostring(sample_root, pretty_print=True, xml_declaration=True)) logging.info('Sample of {pages} pages created at: {path}'.format( pages=pages, path=sample_path))