예제 #1
0
    def big2littlePTS(self, fileName, foleName):

        # ~~ Openning files ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        fle = open(fileName, 'rb')
        fole = open(foleName, 'wb')
        print '           +> writing the volumes-file: ', foleName

        # ~~ Read/Write dimensions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        n1 = self.NPOIN3 + 1
        minvol = self.minvol * np.ones(self.NPOIN3, dtype=np.float32)

        # ~~ Read volumes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        pbar = ProgressBar(maxval=self.HYDROIT).start()
        for i in range(self.HYDROIT):
            l, it = unpack('>ii', fle.read(4 + 4))
            VOLUME = np.asarray(
                unpack('>' + str(self.NPOIN3) + 'f',
                       fle.read(4 * self.NPOIN3)))
            VOLUME = np.maximum(VOLUME, minvol)
            fle.seek(4, 1)
            if it >= self.tfrom and it <= self.tstop:
                pbar.write('            ~> read iteration: ' + str(it), i)
                fole.write(pack('<ii', 4 * n1, it - self.HYDRO00))
                fole.write(pack('<' + str(self.NPOIN3) + 'f', *(VOLUME)))
                fole.write(pack('<i', 4 * n1))
            else:
                pbar.write('            ~> ignore iteration: ' + str(it), i)
            pbar.update(i)
        pbar.finish()

        fle.close()
        fole.close()
예제 #2
0
    def removeDuplangles(self):
        ibar = 0
        pbar = ProgressBar(maxval=self.npoin).start()
        ip = 0
        while ip < len(self.poly):
            ibar += len(self.poly[ip])
            lb = len(self.poly[ip])
            self.poly[ip], self.type[ip] = removeDuplangles(
                self.poly[ip], self.type[ip])
            la = len(self.poly[ip])
            if la < lb:
                pbar.write(
                    '    +> removed ' + str(lb - la) + ' points of ' +
                    str(lb) + ' from polygon ' + str(ip + 1), ibar)
            if self.poly[ip] == []:
                pbar.write('    +> removed entire polygon ' + str(ip + 1),
                           ibar)
                self.poly.pop(ip)
                self.type.pop(ip)
            else:
                ip += 1
            pbar.update(ibar)

        pbar.finish()
        return self.poly, self.type
예제 #3
0
    def big2littleARE(self, fileName, foleName):

        # ~~ Openning files ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        fle = open(fileName, 'rb')
        fole = open(foleName, 'wb')
        print '           +> writing the areas-file: ', foleName

        # ~~ Read/Write dimensions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        NSEG2 = (3 * self.geo.NELEM3 + self.conlim.NPTFR) / 2
        MBND2 = np.count_nonzero(self.conlim.BOR['lih'] != 2)
        n3 = (NSEG2 + MBND2)
        #n4 = 2*( self.geo.NPLAN-1 )*self.geo.NPOIN3

        # ~~ Read volumes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        pbar = ProgressBar(maxval=self.HYDROIT).start()
        for i in range(self.HYDROIT):
            l, it = unpack('>ii', fle.read(4 + 4))
            AREAS = np.asarray(unpack('>' + str(n3) + 'f', fle.read(4 * n3)))
            fle.seek(4, 1)
            if it >= self.tfrom and it <= self.tstop:
                pbar.write('            ~> read iteration: ' + str(it), i)
                fole.write(pack('<ii', 4 * n3, it - self.HYDRO00))
                fole.write(pack('<' + str(n3) + 'f', *(AREAS)))
                fole.write(pack('<i', 4 * n3))
            else:
                pbar.write('            ~> ignore iteration: ' + str(it), i)
            pbar.update(i)
        pbar.finish()

        fle.close()
        fole.close()
    def big2little_pts(self, file_name, fole_name):

        # ~~ Openning files ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        fle = open(file_name, 'rb')
        fole = open(fole_name, 'wb')
        print('           +> writing the volumes-file: ' + fole_name)

        # ~~ Read/Write dimensions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        n_1 = self.npoin3 + 1
        minvol = self.minvol * np.ones(self.npoin3, dtype=np.float32)

        # ~~ Read volumes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        pbar = ProgressBar(maxval=self.hydroit).start()
        for i in range(self.hydroit):
            _, itime = unpack('>ii', fle.read(4 + 4))
            volume = np.asarray(
                unpack('>' + str(self.npoin3) + 'f',
                       fle.read(4 * self.npoin3)))
            volume = np.maximum(volume, minvol)
            fle.seek(4, 1)
            if itime >= self.tfrom and itime <= self.tstop:
                pbar.write('            ~> read iteration: ' + str(itime), i)
                fole.write(pack('<ii', 4 * n_1, itime - self.hydro00))
                fole.write(pack('<' + str(self.npoin3) + 'f', *(volume)))
                fole.write(pack('<i', 4 * n_1))
            else:
                pbar.write('            ~> ignore iteration: ' + str(itime), i)
            pbar.update(i)
        pbar.finish()

        fle.close()
        fole.close()
    def big2little_are(self, file_name, fole_name):

        # ~~ Openning files ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        fle = open(file_name, 'rb')
        fole = open(fole_name, 'wb')
        print('           +> writing the areas-file: ' + fole_name)

        # ~~ Read/Write dimensions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        nseg2 = (3 * self.geo.nelem3 + self.conlim.nptfr) / 2
        mbnd2 = np.count_nonzero(self.conlim.bor['lih'] != 2)
        n_3 = (nseg2 + mbnd2)

        # ~~ Read volumes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        pbar = ProgressBar(maxval=self.hydroit).start()
        for i in range(self.hydroit):
            _, itime = unpack('>ii', fle.read(4 + 4))
            areas = np.asarray(unpack('>' + str(n_3) + 'f', fle.read(4 * n_3)))
            fle.seek(4, 1)
            if itime >= self.tfrom and itime <= self.tstop:
                pbar.write('            ~> read iteration: ' + str(itime), i)
                fole.write(pack('<ii', 4 * n_3, itime - self.hydro00))
                fole.write(pack('<' + str(n_3) + 'f', *(areas)))
                fole.write(pack('<i', 4 * n_3))
            else:
                pbar.write('            ~> ignore iteration: ' + str(itime), i)
            pbar.update(i)
        pbar.finish()

        fle.close()
        fole.close()
예제 #6
0
def copyFiles(pi,po):
   ld = listdir(pi)
   ibar = 0; pbar = ProgressBar(maxval=len(ld)).start()
   for f in ld:
      if path.isfile(path.join(pi,f)): shutil.copy(path.join(pi,f),po)
      pbar.update(ibar); ibar += 1
   pbar.finish()
   return
예제 #7
0
def addFileContent(file,lines):
   SrcF = open(file,'ab')
   ibar = 0; pbar = ProgressBar(maxval=len(lines)).start()
   for line in lines[0:]:
      ibar += 1; pbar.update(ibar)
      SrcF.write('\n'+(line.rstrip()).replace('\r','').replace('\n\n','\n'))
   pbar.finish()
   SrcF.close()
   return
예제 #8
0
def putFileContent(file,lines):
   if path.exists(file): remove(file)
   SrcF = open(file,'wb')
   if len(lines)>0:
      ibar = 0; pbar = ProgressBar(maxval=len(lines)).start()
      SrcF.write((lines[0].rstrip()).replace('\r','').replace('\n\n','\n'))
      for line in lines[1:]:
         pbar.update(ibar); ibar += 1
         SrcF.write('\n'+(line.rstrip()).replace('\r','').replace('\n\n','\n'))
      pbar.finish()
   SrcF.close()
   return
예제 #9
0
    def _generation(self):
        clones = list(self.all_clones.clones())
        num_clones = len(clones)
        pb = ProgressBar(ncr(num_clones, 4) + ncr(num_clones, 3))
        sys.stderr.write("Starting generation {} with {} clones\n".format(self.current_generation, num_clones))
        for i in range(num_clones):
            for j in range(i+1, num_clones):
                for k in range(j+1, num_clones):
                    for l in range(k+1, num_clones):
                        box = Box([clones[i], clones[j], clones[k], clones[l]], self.current_generation)
                        self._process(box)
                        if self.done:
                            return
                        pb.increment()

        for i in range(num_clones):
            for j in range(i+1, num_clones):
                for k in range(j+1, num_clones):
                    clone1 = clones[i]
                    clone2 = clones[j]
                    clone3 = clones[k]
                    box = Box([clones[i], clones[j], clones[k]], self.current_generation)
                    self._process(box)
                    if self.done:
                        return
                    pb.increment()
        pb.clear()
예제 #10
0
    def smoothSubsampleAngle(self, angle):
        ibar = 0
        pbar = ProgressBar(maxval=self.npoin).start()
        # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~/^\~~~~~~~~
        #                            subsampling by anlgle \_/
        ip = 0
        while ip < len(self.poly):
            ibar += len(self.poly[ip])
            lb = len(self.poly[ip])
            self.poly[ip], self.type[ip] = subsampleAngle(
                self.poly[ip], self.type[ip], angle)
            la = len(self.poly[ip])
            if la < lb:
                pbar.write(
                    '    +> removed ' + str(lb - la) + ' points of ' +
                    str(lb) + ' from polygon ' + str(ip + 1), ibar)
            if self.poly[ip] == []:
                pbar.write('    +> removed entire polygon ' + str(ip + 1),
                           ibar)
                self.poly.pop(ip)
                self.type.pop(ip)
            else:
                ip += 1
            #pbar.update(ibar)

        pbar.finish()
        return self.poly, self.type
예제 #11
0
def get_neighbours_slf(ikle, meshx, meshy, showbar=True):
    """
    Return a list containing for each element the list of elements that are
    neighbours to that element

    @param ikle (np.array) Connectivity table
    @param meshx (np.array) X coordinates of the mesh points
    @param meshy (np.array) Y coordinates of the mesh points
    @param showbar (boolean) If True display a progress bar

    @returns (list) The list of neighbours
    @param
    """

    try:
        from matplotlib.tri import Triangulation
        neighbours = Triangulation(meshx, meshy, ikle).get_cpp_triangulation()\
                                                      .get_neighbors()
    except ImportError:
        insiders = {}
        bounders = {}
        ibar = 0
        if showbar:
            pbar = ProgressBar(maxval=(3 * len(ikle))).start()
        for elem, i in zip(ikle, range(len(ikle))):
            n_k = bounders.keys()
            for k in [0, 1, 2]:
                ibar += 1
                if showbar:
                    pbar.update(ibar)
                if (elem[k], elem[(k + 1) % 3]) not in n_k:
                    bounders.update({(elem[(k + 1) % 3], elem[k]): i})
                else:
                    j = bounders[(elem[k], elem[(k + 1) % 3])]
                    insiders.update({(elem[k], elem[(k + 1) % 3]): [i, j]})
                    del bounders[(elem[k], elem[(k + 1) % 3])]
        ibar = 0
        neighbours = -np.ones((len(ikle), 3), dtype=np.int)
        for elem, i in zip(ikle, range(len(ikle))):
            for k in [0, 1, 2]:
                ibar += 1
                if showbar:
                    pbar.update(ibar)
                if (elem[k], elem[(k + 1) % 3]) in insiders:
                    elem_a, elem_b = insiders[(elem[k], elem[(k + 1) % 3])]
                    if elem_a == i:
                        neighbours[i][k] = elem_b
                    if elem_b == i:
                        neighbours[i][k] = elem_a
                if (elem[(k + 1) % 3], elem[k]) in insiders:
                    elem_a, elem_b = insiders[(elem[(k + 1) % 3], elem[k])]
                    if elem_a == i:
                        neighbours[i][k] = elem_b
                    if elem_b == i:
                        neighbours[i][k] = elem_a
        if showbar:
            pbar.finish()

    return neighbours
예제 #12
0
    def valid_epoch(self, data_loader):
        pbar = ProgressBar(n_total=len(data_loader), desc='Evaluating')
        self.entity_score.reset()
        valid_loss = AverageMeter()
        #output_file = jsonlines.open("data/case_out.jsonl","w")
        for step, batch in enumerate(data_loader):

            batch = tuple(t.to(self.device) for t in batch)

            input_ids, input_mask, trigger_mask, segment_ids, label_ids, input_lens, one_hot_labels = batch

            if not self.trigger:
                trigger_mask = None
            if not self.partial:
                one_hot_labels = None
            input_lens = input_lens.cpu().detach().numpy().tolist()
            self.model.eval()
            with torch.no_grad():
                features, loss = self.model(input_ids,
                                            segment_ids,
                                            input_mask,
                                            trigger_mask,
                                            label_ids,
                                            input_lens,
                                            one_hot_labels=one_hot_labels)
                tags, _ = self.model.crf._obtain_labels(
                    features, self.id2label, input_lens)
            valid_loss.update(val=loss.item(), n=input_ids.size(0))
            pbar(step=step, info={"loss": loss.item()})
            label_ids = label_ids.to('cpu').numpy().tolist()

            for i, label in enumerate(label_ids):
                temp_1 = []
                temp_2 = []
                for j, m in enumerate(label):
                    if j == 0:
                        continue
                    elif j == input_lens[i] - 1:  # 控制结束的位置
                        r = self.entity_score.update(pred_paths=[temp_2],
                                                     label_paths=[temp_1])
                        r["input_ids"] = input_ids[i, :].to(
                            "cpu").numpy().tolist()
                        #output_file.write(r)
                        break
                    else:
                        temp_1.append(self.id2label[label_ids[i][j]])
                        try:
                            temp_2.append(tags[i][j])
                        except Exception as e:
                            print(i, j)

        valid_info, ex_valid_info, class_info = self.entity_score.result()
        ex_info = {f'{key}': value for key, value in ex_valid_info.items()}
        info = {f'{key}': value for key, value in valid_info.items()}
        info['valid_loss'] = valid_loss.avg
        if 'cuda' in str(self.device):
            torch.cuda.empty_cache()
        return info, ex_info, class_info
예제 #13
0
 def makeClockwise(self):
     ibar = 0
     pbar = ProgressBar(maxval=self.npoin).start()
     for ip in range(len(self.poly)):
         ibar += len(self.poly[ip])
         if self.type[ip] != 0:
             if not isClockwise(self.poly[ip]):
                 pbar.write(
                     '    +> turned clockwise polygon ' + str(ip + 1), ibar)
                 self.poly[ip] = np.flipud(self.poly[ip])
         pbar.update(ibar)
     pbar.finish()
     return self.poly
예제 #14
0
    def downloadECMWF(self):

        result = self.connection.result()
        fileName = self.request.get("target")

        # ~> tries connecting 3 times before stopping
        tries = 0
        while True:

            # ~> downloading file by blocks
            http = urllib2.urlopen(result["href"])
            f = open(fileName, "wb")
            ibar = 0
            pbar = ProgressBar(maxval=result["size"]).start()
            while True:
                chunk = http.read(1024 * 1024)
                if not chunk: break
                f.write(chunk)
                ibar += len(chunk)
                pbar.update(ibar)
            f.flush()
            f.close()
            pbar.finish()
            # ~> have I got everything ?
            if ibar == result["size"]: break
            if tries == 3:
                print "    ... exhausted the number of download trials.\nYou may wish to attempt this again later."
                sys.exit()
            print "    ... trying to download the data once more ..."
            tries += 1
예제 #15
0
def get_edges_slf(ikle, meshx, meshy, showbar=True):
    """
    Returns the list of edges of the mesh

    @param ikle (np.array) Connectivity table
    @param meshx (np.array) X coordinates of the mesh points
    @param meshy (np.array) Y coordinates of the mesh points
    @param showbar (boolean) If True display a progress bar

    @returns (list) The list of edges
    """

    try:
        from matplotlib.tri import Triangulation
        edges = Triangulation(meshx, meshy, ikle).get_cpp_triangulation()\
                                                              .get_edges()
    except ImportError:
        edges = []
        ibar = 0
        if showbar:
            pbar = ProgressBar(maxval=len(ikle)).start()
        for elem in ikle:
            ibar += 1
            if showbar:
                pbar.update(ibar)
            if [elem[0], elem[1]] not in edges:
                edges.append([elem[1], elem[0]])
            if [elem[1], elem[2]] not in edges:
                edges.append([elem[2], elem[1]])
            if [elem[2], elem[0]] not in edges:
                edges.append([elem[0], elem[2]])
        if showbar:
            pbar.finish()

    return edges
예제 #16
0
    def putContent(self, fileName, showbar=True):

        # ~~> netcdf reader
        ecmwfdata = netcdf.netcdf_file(self.request.get("target"), 'r')

        # ~~> new SELAFIN writer
        self.slf2d.fole = {}
        self.slf2d.fole.update({'hook': open(fileName, 'wb')})
        self.slf2d.fole.update({'name': fileName})
        self.slf2d.fole.update({'endian': ">"})  # big endian
        self.slf2d.fole.update({'float': ('f', 4)})  # single precision

        print '     +> Write SELAFIN header'
        self.appendHeaderECMWF(ecmwfdata)

        print '     +> Write SELAFIN core'
        ibar = 0
        if showbar:
            pbar = ProgressBar(maxval=len(self.slf2d.tags['times'])).start()
        for t in range(len(self.slf2d.tags['times'])):
            self.appendCoreTimeECMWF(t)
            self.appendCoreVarsECMWF(ecmwfdata, ibar)
            ibar += 1
            if showbar: pbar.update(ibar)
        self.slf2d.fole['hook'].close()
        if showbar: pbar.finish()
예제 #17
0
def put_file_content(fle, lines):
    """
    put line to file

    @param fle (string) file
    @param lines (string) adding line
    """
    if path.exists(fle):
        remove(fle)
    src_file = open(fle, 'wb')
    if len(lines) > 0:
        ibar = 0
        pbar = ProgressBar(maxval=len(lines)).start()
        src_file.write(bytes((lines[0].rstrip()).replace('\r', '')\
                                                .replace('\n\n', '\n'),
                             'utf-8'))
        for line in lines[1:]:
            pbar.update(ibar)
            ibar += 1
            src_file.write(bytes('\n'+(line.rstrip()).replace('\r', '')\
                                                     .replace('\n\n', '\n'),
                                 'utf-8'))
        pbar.finish()
    src_file.close()
    return
예제 #18
0
    def fit(self, x, y, steps=0, batch_size=32):
        batch_size = 32 if batch_size == None else batch_size
        num_batches = x.shape[0] // batch_size

        for i, p in enumerate(self.particles):
            local_score = p.get_score(x, y)

            if local_score < self.global_best_score:
                self.global_best_score = local_score
                self.global_best_weights = p.get_best_weights()

        print("PSO -- Initial best score {:0.4f}".format(self.global_best_score))

        bar = ProgressBar(steps, updates=20)

        for i in range(steps):
            for j in range(num_batches):
                x_ = x[j*batch_size:(j+1)*batch_size,:]
                y_ = y[j*batch_size:(j+1)*batch_size]

                for p in self.particles:
                    local_score = p.step(x_, y_, self.global_best_weights)

                    if local_score < self.global_best_score:
                        self.global_best_score = local_score
                        self.global_best_weights = p.get_best_weights()

            bar.update(i)

        bar.done()
예제 #19
0
    def put_content(self, file_name, showbar=True):

        self.open_grib(file_name)

        print('     +> Write Selafin header')
        self.append_header_grib()

        print('     +> Write Selafin core')
        if showbar:
            pbar = ProgressBar(maxval=len(self.dataset)).start()
        for itime in range(len(self.dataset)):
            seconds = int(self.slf2d.tags['times'][itime])
            date = (datetime(*self.slf2d.datetime) +
                    timedelta(seconds=seconds)).timetuple()[0:6]
            print("        - {}-{}-{} {}:{}:{}".format(date[2], date[1],
                                                       date[0], date[3],
                                                       date[4], date[5]))
            self.append_core_time_grib(itime)
            self.append_core_vars_grib(itime)
            if showbar:
                pbar.update(itime)
        if showbar:
            pbar.finish()

        self.close_grib()
예제 #20
0
파일: files.py 프로젝트: ogoe/OpenTelemac
def copyFiles(pi,po):
   ld = listdir(pi)
   ibar = 0; pbar = ProgressBar(maxval=len(ld)).start()
   for f in ld:
      if path.isfile(path.join(pi,f)): shutil.copy(path.join(pi,f),po)
      pbar.update(ibar); ibar += 1
   pbar.finish()
   return
예제 #21
0
    def measure(self, cls, name, timeout=None, tries=None, processes=None):
        timeout = timeout if timeout is not None else self.timeout
        tries = tries if tries is not None else self.tries
        processes = processes if processes is not None else self.processes

        pb = ProgressBar(maximum=tries, width=30, prefix="{self.name:25}",
                         suffix=" {self.last_progress}/{self.maximum}")

        measure_result = list()
        for no_cpu in processes:
            pb.name = "{:s} {:d} {:s}".format(name, no_cpu, 'core' if no_cpu == 1 else 'cores')
            results = list()
            for i in range(0, tries):
                if print_output:
                    pb.progress(i)

                targets = [cls() for j in range(0, no_cpu)]

                with timer.measured("{:s} {:d}".format(name, i), False):
                    # start processes
                    for target in targets:
                        target.start()

                    # wait for timeout
                    time.sleep(timeout)

                    # send exit status
                    for target in targets:
                        target.shutdown()

                    # join threads
                    for target in targets:
                        target.join()

                tmp = dict()
                tmp['duration'] = timer.time()
                tmp['value'] = sum(pluck(targets, 'result.value'))
                tmp['exit'] = not max(pluck(targets, 'terminated'))
                results.append(tmp)

            if print_output:
                pb.end()

            result = dict()
            result['processes'] = no_cpu
            result['exit'] = min(pluck(results, 'exit'))
            result['duration'] = sum(pluck(results, 'duration')) / float(tries)
            result['value'] = sum(pluck(results, 'value')) / float(tries)
            result['performance'] = result['value'] / result['duration']


            if human_format:
                result['value'] = human_readable(result['value'])
                result['performance'] = human_readable(result['performance'])

            measure_result.append(result)

        return measure_result
예제 #22
0
 def smoothSubdivise(self, weight=0.5):
     ibar = 0
     pbar = ProgressBar(maxval=self.npoin).start()
     for ip in range(len(self.poly)):
         ibar += len(self.poly[ip])
         lb = len(self.poly[ip])
         self.poly[ip], self.type[ip] = smoothSubdivise(
             self.poly[ip], self.type[ip], weight)
         la = len(self.poly[ip])
         if la > lb:
             pbar.write(
                 '    +> added ' + str(la - lb) + ' points to polygon ' +
                 str(ip + 1), ibar)
         pbar.update(ibar)
     pbar.finish()
     return self.poly, self.type
예제 #23
0
파일: files.py 프로젝트: ogoe/OpenTelemac
def addFileContent(file,lines):
   SrcF = open(file,'ab')
   ibar = 0; pbar = ProgressBar(maxval=len(lines)).start()
   for line in lines[0:]:
      ibar += 1; pbar.update(ibar)
      SrcF.write('\n'+(line.rstrip()).replace('\r','').replace('\n\n','\n'))
   pbar.finish()
   SrcF.close()
   return
예제 #24
0
    def get_series(self, nodes, vars_indexes=None, showbar=True):
        """
        Return the value for a list of nodes on variables given in vars_indexes
        for each time step

        @param nodes (list) list of nodes for which to extract data
        @param vars_indexes (list) List of variables to extract data for
        @param showbar (boolean) If True display a showbar for the progress

        """
        f = self.file['hook']
        endian = self.file['endian']
        ftype, fsize = self.file['float']
        if vars_indexes is None:
            vars_indexes = self.varindex
        # ~~ Ordering the nodes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        # This assumes that nodes starts at 1
        onodes = np.sort(np.array(zip(range(len(nodes)), nodes),
                                  dtype=[('0', int), ('1', int)]),
                         order='1')
        # ~~ Extract time profiles ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        if fsize == 4:
            z = np.zeros(
                (len(vars_indexes), len(nodes), len(self.tags['cores'])),
                dtype=np.float32)
        else:
            z = np.zeros(
                (len(vars_indexes), len(nodes), len(self.tags['cores'])),
                dtype=np.float64)
        f.seek(self.tags['cores'][0])
        if showbar:
            pbar = ProgressBar(maxval=len(self.tags['cores'])).start()
        for time in range(len(self.tags['cores'])):
            f.seek(self.tags['cores'][time])
            f.seek(4 + fsize + 4, 1)
            if showbar:
                pbar.update(time)
            for ivar in range(self.nvar):
                f.seek(4, 1)
                if ivar in vars_indexes:
                    jnod = onodes[0]
                    f.seek(fsize * (jnod[1] - 1), 1)
                    z[vars_indexes.index(ivar), jnod[0], time] = \
                        unpack(endian+ftype, f.read(fsize))[0]
                    for inod in onodes[1:]:
                        f.seek(fsize * (inod[1] - jnod[1] - 1), 1)
                        z[vars_indexes.index(ivar), inod[0], time] = \
                            unpack(endian+ftype, f.read(fsize))[0]
                        jnod = inod
                    f.seek(fsize * self.npoin3 - fsize * jnod[1], 1)
                else:
                    f.seek(fsize * self.npoin3, 1)
                f.seek(4, 1)
        if showbar:
            pbar.finish()
        return z
예제 #25
0
    def put_content(self, file_name, showbar=True):
        # ~~> Output file header
        self.slf.fole.update({'hook': open(file_name, 'wb')})
        self.slf.fole['name'] = file_name
        self.slf.nbv1 = 0
        self.slf.nbv2 = 0
        self.slf.varnames = []
        self.slf.varunits = []
        for calc in self.calcs:
            for cname, cunit in calc['vars']:
                self.slf.varnames.append(cname)
                self.slf.varunits.append(cunit)
                self.slf.nbv1 += 1
        self.slf.append_header_slf()

        # ~~> Time stepping
        print('\n      > Input signal based on:')
        print('      - ' + str(self.slf.nvar / 2) + ' direction(s)')
        print('      - for the following periods:' +
              repr(self.slf.tags['times']))
        # ~~> Time stepping
        print('\n      > Going through time ('+str(len(self.history))+\
                'time steps) :')
        pbar = ProgressBar(maxval=len(self.history)).start()

        for t_i in range(len(self.history)):
            ttime = self.history[t_i]
            # ~~> Initialise vari to 0 for each time step
            vari = []
            t_0 = self.slf.tags['times'][0]
            for calc, icalc in zip(self.calcs, range(len(self.calcs))):
                fct, args = calc['init']
                vari.append(fct(vari, args, t_0, t_0))
            # ~~> Sweeps through wave periods, adding up all components of
            # free surface into "vari" for a given time
            #     in this particular case self.slf.tags['times'] holds
            #     wave periods, not times
            for t_p in range(len(self.slf.tags['times'])):
                vrs = self.get_palues(t_p)
                for icalc, calc in enumerate(self.calcs):
                    fct, args = calc['calc']
                    times = self.slf.tags['times']
                    vari[icalc] = fct(vrs, args, ttime, times[t_p],
                                      times[len(times) - 1], vari[icalc])
            # ~~> Print time record
            self.slf.append_core_time_slf(float(ttime))
            for icalc, calc in enumerate(self.calcs):
                fct = calc['stop']
                self.slf.append_core_vars_slf(fct(t_0, ttime, vari[icalc]))
            pbar.update(t_i)

        pbar.finish()
        self.slf.fole['hook'].close()
예제 #26
0
파일: files.py 프로젝트: ogoe/OpenTelemac
def putFileContent(file,lines):
   if path.exists(file): remove(file)
   SrcF = open(file,'wb')
   if len(lines)>0:
      ibar = 0; pbar = ProgressBar(maxval=len(lines)).start()
      SrcF.write((lines[0].rstrip()).replace('\r','').replace('\n\n','\n'))
      for line in lines[1:]:
         pbar.update(ibar); ibar += 1
         SrcF.write('\n'+(line.rstrip()).replace('\r','').replace('\n\n','\n'))
      pbar.finish()
   SrcF.close()
   return
예제 #27
0
    def append_core_vars_grib(self, itime):

        if self.typ == 'wave':
            pass
            # ~~> WAVE HEIGHT == 'swh'
            # ~~> SIGNIFICANT WAVE PERIOD == 'mwp'
            # ~~> MEAN WAVE DIRECTION == 'mwd'

        elif self.typ == 'oper':
            var2d = np.zeros((self.slf2d.nvar, self.slf2d.npoin2),
                             dtype=np.float)
            grbs = pygrib.open(self.dataset[itime])
            for grb in grbs:
                if grb.indicatorOfParameter in self.variables:
                    jvar = self.variables.index(grb.indicatorOfParameter)
                    var2d[jvar, :] = np.ravel(grb.values.T)
            grbs.close()
            for jvar in range(self.slf2d.nvar):
                self.slf2d.append_core_vars_slf([var2d[jvar, :]])

        elif self.typ == 'spec':

            spec = np.zeros(
                (self.nb_direct, self.nb_freq, self.nx1d, self.ny1d),
                dtype=np.float)
            grbs = pygrib.open(self.dataset[itime])
            ibar = 0
            maxval = self.nb_direct * self.nb_freq
            pbar = ProgressBar(maxval=maxval).start()
            for grb in grbs:
                i_i = 0
                data = grb.values.data
                data[np.where(np.absolute(data) <= 0.001)] = np.nan
                data[np.where(data == self.missing_value)] = np.nan
                data = 10.**data
                data[np.isnan(data)] = 0.
                for i_y in range(len(self.masky)):
                    if self.masky[i_y]:
                        spec[grb.directionNumber-1, grb.frequencyNumber-1, :,
                             i_i] = \
                                  data[i_y][self.maskx]
                        i_i += 1
                ibar += 1
                pbar.update(ibar)
            pbar.finish()
            grbs.close()

            for i_x in range(self.nx1d):
                for i_y in range(self.ny1d):
                    self.slf2d.append_core_vars_slf([np.ravel\
                            (spec[:, :, i_x, i_y].T)])
예제 #28
0
    def train_epoch(self, data_loader):

        pbar = ProgressBar(n_total=len(data_loader), desc='Training')
        tr_loss = AverageMeter()

        for step, batch in enumerate(data_loader):
            self.model.train()

            batch = tuple(t.to(self.device) for t in batch)

            input_ids, input_mask, trigger_mask, segment_ids, label_ids, input_lens, one_hot_labels = batch
            if not self.partial:
                one_hot_labels = None
            if not self.trigger:
                trigger_mask = None
            input_lens = input_lens.cpu().detach().numpy().tolist()
            _, loss, = self.model(input_ids, segment_ids, input_mask,
                                  trigger_mask, label_ids, input_lens,
                                  one_hot_labels)

            if len(self.n_gpu.split(",")) >= 2:
                loss = loss.mean()
            if self.gradient_accumulation_steps > 1:
                loss = loss / self.gradient_accumulation_steps

            loss.backward()
            clip_grad_norm_(self.model.parameters(), self.grad_clip)
            if (step + 1) % self.gradient_accumulation_steps == 0:
                self.optimizer.step()
                self.optimizer.zero_grad()
                self.global_step += 1
            tr_loss.update(loss.item(), n=1)
            self.tb_logger.scalar_summary("loss", tr_loss.avg,
                                          self.global_step)
            pbar(step=step, info={'loss': loss.item()})
            # if step%5==0:
            # 	self.logger.info("step:{},loss={:.4f}".format(self.global_step,loss.item()))

        info = {'loss': tr_loss.avg}
        if "cuda" in str(self.device):
            torch.cuda.empty_cache()
        return info
예제 #29
0
 def removeDuplilines(self):
     ibar = 0
     pbar = ProgressBar(maxval=self.npoin).start()
     ip = 0
     while ip < len(self.poly):
         ibar += len(self.poly[ip])
         p, t = removeDuplilines(self.poly[ip], self.type[ip])
         pbar.trace()  # /!\ the call requires sub-progress bar
         if len(p) > 1:
             pbar.maxval -= len(self.poly[ip])
             ibar -= len(self.poly[ip])
             self.poly.pop(ip)
             self.type.pop(ip)
             for po, to in zip(p, t):
                 pbar.maxval += len(po)
                 self.poly.append(po)
                 self.type.append(to)
         else:
             ip += 1
         pbar.update(ibar)
     pbar.finish()
     return self.poly, self.type
예제 #30
0
def copy_files(src, dest):
    """
    Copy all the files within src
    @param src (string) source directory
    @param dest (string) target directory
    """
    l_d = listdir(src)
    ibar = 0
    pbar = ProgressBar(maxval=len(l_d)).start()
    for f in l_d:
        if path.isfile(path.join(src, f)):
            shutil.copy(path.join(src, f), dest)
        pbar.update(ibar)
        ibar += 1
    pbar.finish()
예제 #31
0
 def put_content(self, file_name):
     self.slf.fole.update({'hook': open(file_name, 'wb')})
     self.slf.file['name'] = file_name
     pbar = ProgressBar(maxval=len(self.slf.tags['times'])).start()
     self.slf.append_header_slf()
     # ~~> Time stepping
     for itime in range(len(self.slf.tags['times'])):
         self.slf.append_core_time_slf(itime)
         vrs = self.get_palues(itime)
         self.slf.append_core_vars_slf(vrs)
         for fct, args in self.calcs:
             self.slf.append_core_vars_slf(fct(vrs, args))
         pbar.update(itime)
     pbar.finish()
     self.slf.fole['hook'].close()
예제 #32
0
 def put_content(self, file_name, showbar=True):
     # ~~> Sweep through time steps, saving "vari"
     vari = []
     initiate = True
     if showbar:
         pbar = ProgressBar(maxval=len(self.slf.tags['times'])).start()
     t_0 = self.slf.tags['times'][0]
     for itime in range(len(self.slf.tags['times'])):
         vrs = self.get_palues(itime)
         if initiate:
             for icalc, calc in enumerate(self.calcs):
                 fct, args = calc['init']
                 vari.append(
                     fct(vrs, args, t_0, self.slf.tags['times'][itime]))
             initiate = False
         else:
             for icalc, calc in enumerate(self.calcs):
                 fct, args = calc['calc']
                 vari[icalc] = fct(vrs, args, t_0,
                                   self.slf.tags['times'][itime],
                                   vari[icalc])
         if showbar:
             pbar.update(itime)
     if showbar:
         pbar.finish()
     # ~~> Header
     self.slf.fole.update({'hook': open(file_name, 'wb')})
     self.slf.fole['name'] = file_name
     self.slf.nbv1 = 0
     self.slf.nbv2 = 0
     self.slf.varnames = []
     self.slf.varunits = []
     for calc in self.calcs:
         for cname, cunit in calc['vars']:
             self.slf.varnames.append(cname)
             self.slf.varunits.append(cunit)
             self.slf.nbv1 += 1
     self.slf.append_header_slf()
     # ~~> Core
     # TODO: writing only for first timestep
     self.slf.append_core_time_slf(0)
     for icalc, calc in enumerate(self.calcs):
         fct = calc['stop']
         itime = t_0
         self.slf.append_core_vars_slf(fct(t_0, itime, vari[icalc]))
     self.slf.fole['hook'].close()
예제 #33
0
def add_file_content(fle, lines):
    """
    Add line to file

    @param fle (string) file
    @param lines (string) adding line
    """
    src_file = open(fle, 'ab')
    ibar = 0
    pbar = ProgressBar(maxval=len(lines)).start()
    for line in lines[0:]:
        ibar += 1
        pbar.update(ibar)
        src_file.write(bytes('\n'+(line.rstrip()).replace('\r', '')\
                                                 .replace('\n\n', '\n'),
                             'utf-8'))
    pbar.finish()
    src_file.close()
    return
예제 #34
0
    def put_content(self, file_name, showbar=True):
        """
        Write content of the object into a Serafin file

        @param file_name (string) Name of the serafin file
        @param showbar (boolean) If True displays a showbar
        """
        self.fole.update({'name': file_name})
        self.fole.update({'hook': open(file_name, 'wb')})
        ibar = 0
        if showbar:
            pbar = ProgressBar(maxval=len(self.tags['times'])).start()
        self.append_header_slf()
        for time in range(len(self.tags['times'])):
            ibar += 1
            self.append_core_time_slf(time)
            self.append_core_vars_slf(self.get_values(time))
            if showbar:
                pbar.update(ibar)
        self.fole['hook'].close()
        if showbar:
            pbar.finish()
예제 #35
0
    def create_sample_dump(self, articles=None):
        """ Creates smaller sample dump from large dump of given language

        :articles: list/set of unicodes [optional]
            Titles of articles you want to appear in sample. Number of titles
            is arbitrary, if there are too many of them, some will be ommited,
            if there are too few, smaller dump will be created and message
            will be displayed.
        """
        # TODO: check that all items of articles are unicodes, not just str

        # find parent dump
        parent = self.get_parent_corpus()

        # select sample path
        sample_path = self.get_dump_path()

        # find the namespace
        namespace = parent.get_namespace()

        # articles specified
        if articles:
            specific_sample = True
            articles = set(articles)
        else:
            specific_sample = False

        # create qualified names (= names with namespaces) for tags we need
        TEXT_TAG = qualified_name('text', namespace)
        TITLE_TAG = qualified_name('title', namespace)
        REDIRECT_TAG = qualified_name('redirect', namespace)
        NS_TAG = qualified_name('ns', namespace)

        # iterate through xml and build a sample file
        with parent._open_dump() as dump_file:
            context = etree.iterparse(dump_file, events=('end',))
            # create root under which we will add sample articles
            sample_root = etree.Element('mediawiki', nsmap={None: namespace})
            skip = True
            pages = 0
            last_title = None
            if specific_sample:
                # in case of specific sample, it's easily possible
                # that we will need to go through the whole dump
                # -> meassure progress as a ratio of processed part
                progressbar = ProgressBar(parent.get_dump_length())
            else:
                progressbar = ProgressBar(self.sample_size())
            for event, elem in context:
                if elem.tag == REDIRECT_TAG:
                    # ignore redirect pages
                    skip = True
                elif elem.tag == NS_TAG:
                    last_ns = elem.text
                    if elem.text != WikiCorpus.ARTICLE_NS:
                        skip = True
                elif elem.tag == TITLE_TAG:
                    # remember the title
                    last_title = elem.text
                elif elem.tag == TEXT_TAG:
                    if skip:
                        skip = False
                        continue
                    # if articles are not specified, take any article,
                    # if they are specified, check if this is wanted article
                    if not specific_sample or last_title in articles:
                        # build page node with title and text subelements
                        page_node = etree.Element('page')
                        title_node = etree.SubElement(page_node, 'title')
                        title_node.text = last_title
                        ns_node = etree.SubElement(page_node, 'ns')
                        ns_node.text = last_ns
                        page_node.append(deepcopy(elem))  # text
                        # append this node to sample articles
                        sample_root.append(page_node)
                        pages += 1
                        if specific_sample:
                            articles.remove(last_title)
                        if pages == self.sample_size():
                            break
                    # progress update
                    if specific_sample:
                        progressbar.update(dump_file.tell())
                    else:
                        progressbar.update(pages)
                # cleanup
                elem.clear()
                #while elem.getprevious() is not None:
                #    del elem.getparent()[0]
                for ancestor in elem.xpath('ancestor-or-self::*'):
                    while ancestor.getprevious() is not None:
                        del ancestor.getparent()[0]
            del context
            progressbar.finish()

        # check if sample is of required size
        if pages < self.sample_size():
            logging.warning('Failed to create sample of {n} pages.'.format(
                n=self.sample_size()))
            if articles:
                logging.warning('Following articles not found:' +
                    '\n'.join(['- ' + title for title in articles]))

        # write sample xml to file
        with open(sample_path, 'w') as sample_file:
            sample_file.write(etree.tostring(sample_root,
                pretty_print=True,
                xml_declaration=True))

        logging.info('Sample of {pages} pages created at: {path}'.format(
            pages=pages, path=sample_path))