def check_files(self): """ from the list of requested subs and the location of the files, find out how complete the data list is, and find one file that can be representative w.r.t. attributes and such """ count = list() sub_data_list = list() self.data_list = glob.glob(self.data_list) for sub in self.subs: if sub in usefor.keys(): csub = usefor[sub] # csub is name of sub in CMEMS nomenclature query = [file for file in self.data_list if csub['substance'][0] in file] print('files for %s: %i' % (csub['substance'][0], len(query))) sub_data_list.append(query) count.append(len(query)) else: count.append(0) count = np.array(count) if sum(count) == 0: print('ERROR: cannot continue with boundary creation, no data files found for ANY variable') print('usefor.keys() = ') print(list(usefor.keys())) print('self.subs = ' ) print(self.subs) print('checked %s' % self.data_list) raise FileNotFoundError elif sum(count >0) > 0 and sum(count > 0) != len(self.subs): print('WARNING: no data files available for some variables, will try to continue') print('suspected missing subs:') for sind, ss in enumerate(self.subs): if count[sind] == 0: print(ss) else: print('Data files found for all variables') print('subs:') print(self.subs) print('# of files available:') print(count) if len(set(count)) > 1: print('WARNING: SOME VARIABLES COVER DIFFERENT TIME SPANS THAN OTHERS, RESULTING BOUNDARIES WILL NOT BE VALID. SEE INCONGRUENCY BETWEEN NUMBER OF FILES FOUND IN ARRAY ABOVE') # find the first substance that has a data file so the geometry can be extracted a single time ind = count > 0 for i, bol in enumerate(ind): if bol: match = i break return sub_data_list[match][0]
def write_new_ext_file(self): """ WRITE NEW EXT FILE CONTAINING THE CONSTITUENT BOUNDARIES """ boundaries = self.boundaries with open(self.ext, 'r') as new_template: for _, bnd in enumerate(boundaries.keys()): with open(os.path.join(self.dir, 'DFMWAQ_' + bnd + '_tmp.ext'), 'w') as new_ext: # copy old boundaries, should only be waterlevel for line in new_template.readlines(): new_ext.write(line) if 'waterlevelbnd' in boundaries[bnd]['type']: new_ext.write('\n') # if it is waterlevel then it was involved in the previous steps for sub in self.subs: if sub in usefor.keys(): if sub == 'steric' and not self.steric: pass else: new_ext.write('[boundary]\n') if sub in constituent_boundary_type.keys(): if sub != 'uxuy': new_ext.write( 'quantity=%s\n' % ','.join( constituent_boundary_type[ sub]['type']).replace( ',', '')) else: # advection, inconsistent naming new_ext.write( 'quantity=uxuyadvectionvelocitybnd\n' ) else: new_ext.write( 'quantity=tracerbnd%s\n' % sub) new_ext.write('locationfile=%s.pli\n' % (bnd)) new_ext.write('forcingfile=%s_%s.bc\n' % (sub, bnd)) new_ext.write('\n') if os.path.exists(os.path.join(self.dir, '..', 'ext.zip')): mode = 'a' else: mode = 'w' with zipfile.ZipFile(os.path.join(self.dir, '..', 'ext.zip'), mode) as zf: fullpath = os.path.join(self.dir, 'DFMWAQ_' + bnd + '_tmp.ext') zf.write(fullpath, os.path.basename(fullpath))
def process(): name = request.args.get('name') # print(name) out = os.path.join(current_app.root_path, 'static', 'out', name) if not os.path.exists(out): os.mkdir(out) tref = request.args.get('tref') dataset = request.args.get('dataset') pli_file = request.args.get('pli_file') credentials = json.loads(request.args.get('credentials')) time_vect = json.loads(request.args.get('time_vect')) coords = request.args.get('coords').replace('[', '').replace(']', '').split(',') coords = np.array(coords, dtype=np.float64) mod_query = Query(time_vect, dataset, coords, credentials, out) mod_query.build_query() fes_path = os.path.abspath( os.path.join(current_app.root_path, 'static', 'FES')) # file_service = FileService(account_name='coastservstorage', # account_key='HOBOcLV1nEDevDKfHMVq9N0TsgNrh4AXI2Qyxt4QAmvDi+oSUD9R6xnI8UEz7AVduzMill3+ymlNlj11jC58vw==') # generator = file_service.list_directories_and_files('static/FES') # print('****************8testprinting848484848********************') # for file_or_dir in generator: # if file_or_dir.name == 'FES': # # fes_path3=file_or_dir.name # print(file_or_dir, file_or_dir.name) # # file_service.get_file_to_path('static/FES', None, file_or_dir.name, 'out.nc') mod_tide = Tide(fes_path, coords, pli_file, out) ext = mod_tide.ext data_list = os.path.join(out, 'data', '*.nc') # convert from CMEMS parameter names to DFM names CMEMS_subs = mod_query.all_subs[dataset] sub_list = [] for sub in usefor.keys(): if usefor[sub]['substance'][0] in CMEMS_subs: sub_list.append(sub) tref = pd.Timestamp(tref).to_pydatetime() model_dir = mod_tide.out mod_bound = Boundary(ext, data_list, sub_list, tref, model_dir) return render_template('process.html', query=mod_query, tide=mod_tide, model=mod_bound, legend='Processing Query')
def process_bc(self): """ wrapper for writing a bc file """ boundaries = self.boundaries for bnd in boundaries.keys(): if boundaries[bnd]['type'] == 'waterlevelbnd': # waterlevel because an ocean boundary should be waterlevel for sub in self.subs: if sub in usefor.keys(): self.write_bc_file(sub, bnd) else: print('WARNING: requested sub %s not in CMEMS dict, no boundary made!' % sub)
def process(): name = request.args.get('name') print(name) out = os.path.join(current_app.root_path, 'static', 'out', name) if not os.path.exists(out): os.mkdir(out) tref = request.args.get('tref') dataset = request.args.get('dataset') frequency = request.args.get('frequency') pli_file = request.args.get('pli_file') credentials = json.loads(request.args.get('credentials')) time_vect = json.loads(request.args.get('time_vect')) coords = request.args.get('coords').replace('[', '').replace(']', '').split(',') coords = np.array(coords, dtype=np.float64) mod_query = Query(time_vect, dataset, frequency, coords, credentials, out) mod_query.build_query() fes_path = os.path.abspath( os.path.join(current_app.root_path, 'static', 'FES')) mod_tide = Tide(fes_path, coords, pli_file, out) ext = mod_tide.ext data_list = os.path.join(out, 'data', '*.nc') # convert from CMEMS parameter names to DFM names CMEMS_subs = mod_query.all_subs[dataset] sub_list = [] for sub in usefor.keys(): if usefor[sub]['substance'][0] in CMEMS_subs: sub_list.append(sub) tref = pd.Timestamp(tref).to_pydatetime() model_dir = mod_tide.out mod_bound = Boundary(ext, data_list, sub_list, tref, model_dir) return render_template('process.html', query=mod_query, tide=mod_tide, model=mod_bound, legend='Processing Query')