def workflow(cfg, maindb, climdb): """ This function mainly to import measurement data to MongoDB data type may include Q (discharge, m3/s), SED (mg/L), TN (mg/L), TP (mg/L), etc. the required parameters that defined in configuration file (*.ini) """ if not cfg.use_observed: return False c_list = climdb.collection_names() if not StringClass.string_in_list(DBTableNames.observes, c_list): climdb.create_collection(DBTableNames.observes) else: climdb.drop_collection(DBTableNames.observes) if not StringClass.string_in_list(DBTableNames.sites, c_list): climdb.create_collection(DBTableNames.sites) if not StringClass.string_in_list(DBTableNames.var_desc, c_list): climdb.create_collection(DBTableNames.var_desc) file_list = FileClass.get_full_filename_by_suffixes( cfg.observe_dir, ['.txt', '.csv']) meas_file_list = list() site_loc = list() for fl in file_list: if StringClass.is_substring('observed_', fl): meas_file_list.append(fl) else: site_loc.append(fl) ImportObservedData.data_from_txt(maindb, climdb, meas_file_list, site_loc, cfg.spatials.subbsn) return True
def main(): from preprocess.config import parse_ini_configuration seims_cfg = parse_ini_configuration() client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port) conn = client.get_conn() db_model = conn[seims_cfg.spatial_db] spatial_gfs = GridFS(db_model, DBTableNames.gridfs_spatial) csv_path = r'C:\z_data\zhongTianShe\model_data_seims\field_scale_params' csv_files = FileClass.get_full_filename_by_suffixes(csv_path, ['.csv']) field_count = 7419 prefix = 9999 # Create mask file mask_name = '%d_MASK' % prefix mask_array = [[1] * field_count] import_array_to_mongodb(spatial_gfs, mask_array, mask_name) # Create spatial parameters for csv_file in csv_files: print('Import %s...' % csv_file) param_arrays = read_field_arrays_from_csv(csv_file) for key, value in list(param_arrays.items()): import_array_to_mongodb(spatial_gfs, value, '%d_%s' % (prefix, key))
def workflow(cfg, maindb, climdb): """ This function mainly to import measurement data to MongoDB data type may include Q (discharge, m3/s), SED (mg/L), tn (mg/L), tp (mg/L), etc. the required parameters that defined in configuration file (*.ini) """ if not cfg.use_observed: return False c_list = climdb.collection_names() if not StringClass.string_in_list(DBTableNames.observes, c_list): climdb.create_collection(DBTableNames.observes) else: climdb.drop_collection(DBTableNames.observes) if not StringClass.string_in_list(DBTableNames.sites, c_list): climdb.create_collection(DBTableNames.sites) if not StringClass.string_in_list(DBTableNames.var_desc, c_list): climdb.create_collection(DBTableNames.var_desc) file_list = FileClass.get_full_filename_by_suffixes(cfg.observe_dir, ['.txt']) meas_file_list = [] site_loc = [] for fl in file_list: if StringClass.is_substring('observed_', fl): meas_file_list.append(fl) else: site_loc.append(fl) ImportObservedData.data_from_txt(maindb, climdb, meas_file_list, site_loc, cfg.spatials.subbsn) return True