def GetUnmatchingScenes(source_scenes, s3_scenes, pms = False): unmatched = OrderedDict() matched = OrderedDict() for id in source_scenes: for s3id in s3_scenes: source_id, meta_path = s3_scenes[s3id] if id==source_id: scene_dict = source_scenes[id] scene_fullpath = GetSceneFullpath(scene_dict) if scene_fullpath: source_process = process().input(scene_fullpath) temp_json = tempname('json') source_process.GetCoverJSON(temp_json) source_ds, source_lyr = get_lyr_by_path(temp_json) s3_ds, s3_lyr = get_lyr_by_path(meta_path) if source_lyr and s3_lyr: source_feat = source_lyr.GetNextFeature() s3_feat = s3_lyr.GetNextFeature() match = CoverMatch(source_feat, s3_feat, pms = pms) if match: print('SCENE ALREADY EXIST: %s' % s3id) matched[id] = source_scenes[id] continue else: print('COVER ERROR: %s' % s3id) else: print('CANNOT OPEN SCENE: %s' % id) unmatched[id] = source_scenes[id] return matched, unmatched
def TotalCover(pout, files, srs = None): driver = ogr.GetDriverByName('GeoJSON') ds_out = driver.CreateDataSource(pout) if srs is None: srs = osr.SpatialReference() srs.ImportFromEPSG(4326) lyr_out = ds_out.CreateLayer('', srs, ogr.wkbMultiPolygon) lyr_out.CreateField(ogr.FieldDefn('id', 4)) lyr_out.CreateField(ogr.FieldDefn('satid', 4)) feat_defn = lyr_out.GetLayerDefn() for file in files: ds_ = gdal.Open(file) if ds_: f,id,e = split3(file) feat = ogr.Feature(feat_defn) feat.SetField('id', id) geom = RasterGeometry(ds_, reference=srs) feat.SetGeometry(geom) proc = process().input(f) ascene = proc.scenes[0] if ascene: satid = ascene.meta.sat feat.SetField('satid', satid) lyr_out.CreateFeature(feat) ds_out = None
def FindScenes(path_in, imsys_list = None, skip_duplicates = False, v_cover = None): scenes = OrderedDict() for folder_in in obj2list(path_in): proc = process().input(folder_in, imsys_list=imsys_list, skip_duplicates = skip_duplicates) for ascene in proc.scenes: id = ascene.meta.id if '.PAN' in id: base_id = id.replace('.PAN','.PMS') type = 'PAN' elif '.MS' in id: base_id = id.replace('.MS','.PMS') type = 'MS' elif '.PMS' in id: base_id = id type = 'PMS' else: print('Unknown type: %s' % id) continue if base_id in scenes: if type in scenes[base_id]: print('Scene already exists: %s' % id) continue else: scenes[base_id][type] = ascene.fullpath else: scenes[base_id] = {'id': base_id, type: ascene.fullpath} # !!! Correct for multiple source folders! if not v_cover: v_cover = fullpath(folder_in, 'v_cover.json') if not os.path.exists(v_cover): # scroll(v_cover) proc.GetCoverJSON(v_cover) return scenes, v_cover
def GetMetaDG(id): files = globals()['dg_files'] if files: names = flist(files, lambda x: split3(x)[1]) if id in names: file = files[names.index(id)] dg_proc = process().input(os.path.dirname(file)) # scroll(dg_proc.get_ids(), header=id) dg_scene = dg_proc.scenes[0] satid = dg_scene.meta.name('[sat]') date = dg_scene.meta.name('[date]') time = dg_scene.meta.name('[time]') loc = dg_scene.meta.name('[location]') lvl = dg_scene.meta.name('[lvl]') # print(satid, date, time, loc, lvl) return satid, date, time, loc, lvl else: return parse_dg(id)
def TotalCover(pout, files, srs=None): driver = ogr.GetDriverByName('GeoJSON') ds_out = driver.CreateDataSource(pout) if srs is None: srs = osr.SpatialReference() srs.ImportFromEPSG(4326) lyr_out = ds_out.CreateLayer('', srs, ogr.wkbMultiPolygon) lyr_out.CreateField(ogr.FieldDefn('id', 4)) lyr_out.CreateField(ogr.FieldDefn('path', 4)) lyr_out.CreateField(ogr.FieldDefn('satid', 4)) lyr_out.CreateField(ogr.FieldDefn('geomerror', 8)) feat_defn = lyr_out.GetLayerDefn() i = 0 for file in files: try: ds_ = gdal.Open(file) except: ds_ = None if ds_: i += 1 f, id, e = split3(file) feat = ogr.Feature(feat_defn) feat.SetField('id', id) feat.SetField('path', file) try: proc = process().input(f) if len(proc) == 1: ascene = proc.scenes[0] satid = ascene.meta.sat feat.SetField('satid', satid) except: feat.SetField('satid', None) try: geom = RasterGeometry(ds_, reference=srs) feat.SetGeometry(geom) feat.SetField('geomerror', False) except: feat.SetField('geomerror', True) lyr_out.CreateFeature(feat) ds_out = None return i
if raster_geometry: break else: raster_geometry = None if raster_geometry is None: raster_geometry = raster_geom(gdal.Open(file), reference=srs) if raster_geometry is None: print('UNABLE TO EXTRACT GEOMETRY FROM: %s' % n) continue for feat in lyr_cover: vector_geometry = feat.GetGeometryRef() if raster_geometry.Intersects(vector_geometry): print('MATCH: %s' % n) if copy_: copy_file(file, fullpath(pout, n, e), id=n) else: copy_paths.append(file) if not f in scene_folders: scene_folders.append(f) break if target=='LIST COPY PATHS': if len(copy_paths) > 0: with open(fullpath(pout, 'copy_paths.txt'), 'w') as txt: txt.write('\n'.join(copy_paths)) try: proc = process().input(scene_folders, imsys_list=['RSP']) proc.GetCoverJSON(fullpath(pout, 'vector_cover.json')) except: print('ERROR CREATING VECTOR COVER')
# -*- coding: utf-8 -*- from image_processor import process from geodata import * import sys din = r'e:\tos3' dout = r'e:\rks\s3' tmpt_snt = '^S2[AB]_MSI.+\d$' tmpt_pln = '.*AnalyticMS(_SR)?$' tmpt_pln_neuro = 'IM4-PLN.*' source = r'e:\rks\source' tcover = r'e:\rks\s3\cover.json' process().input(source).GetCoverJSON(tcover) def cleansplit(str_, sep, join=''): ids = str_.split(sep) for i, id in enumerate(ids): if i: ids[i] = join+id return ids def get_export_list(din): export = OrderedDict() for path in folder_paths(din,1,'tif'): f, n, e = split3(path) if re.search(tmpt_snt, n): ids = cleansplit(n, '_S2', 'S2') for i, id in enumerate(ids):
# r'd:\digital_earth\resurs-p_new\krym', # None ] for path_in, path_out, path_data in zip(path_in_list, path_out_list, path_data_list): ds_in, lyr_in = get_lyr_by_path(path_in) feat_in_list = lyr_in json_fields(path_out, ogr.wkbMultiPolygon, 4326, fields_dict, feat_in_list, translator) if path_data is not None: proc = image_processor.process().input(path_data) ds_out, lyr_out = get_lyr_by_path(path_out) for feat in lyr_out: ''' For Planet ''' ascene = proc.get_scene(feat.GetField('id') + '_3B_AnalyticMS') if ascene is not None: metadata = ascene.meta.container.get('xmltree') feat.SetField('id', ascene.meta.id) feat.SetField('id_s', ascene.meta.name('[location]')) feat.SetField( 'id_neuro', ascene.meta.name('[fullsat]-[date]-[location]-[lvl]')) feat.SetField('datetime', get_from_tree(metadata, 'acquisitionDateTime')) # feat.SetField('clouds', get_from_tree(metadata, 'acquisitionDateTime'))
# path_in_list = folder_paths(path_in,1,'tif') rgblist = [] files_list = [] codes_list = [] def get_code(file_name): vals = file_name.split('_') if len(vals) == 6: return vals[2] return vals[3] codes_dict = {} for ascene in process().input(path_in, imsys_list=['PLN']).scenes: file = ascene.get_raster_path('Analytic') f, n, e = split3(file) code = get_code(n) if re.search(r'^\d{8}_.+_3B_AnalyticMS_clip$', n): files_list.append(file) codes_list.append(code) path_rgb = fullpath(folder_out, n + '.RGB', e) if code not in codes_dict.keys(): codes_dict[code] = [file] else: codes_dict[code].append(file) rgblist.append(path_rgb) print('appended: %s' % n) else:
out_folder = r'd:\rks\kanopus_cloud_test\pan' out_pms = r'd:\rks\kanopus_cloud_test\pms' suredir(out_folder) suredir(out_pms) def GetMSids(source_folder): ids = [] for name in folder_paths(source_folder,1,'tif'): if re.search('MS.+\.tif$', name.lower()): ids.append(name[:-4]) return ids ms_paths = folder_paths(ms_folder,1,'tif') ms_ids = flist(ms_paths, lambda x: split3(x)[1]) proc = process().input(source_folder, skip_duplicates=False) pan_paths = [] pan_ids = [] for ascene in proc.scenes: if ascene.meta.type=='PAN': # path = ascene.get_raster_path('pan') path = folder_paths(ascene.path,1,'tif')[0] or None if path: pan_paths.append(path) pan_ids.append(ascene.meta.id.strip()) else: print('PATH NOT FOUND: %s' % ascene.meta.id) # scroll(pan_ids) # scroll(ms_ids)
if dir_out is None: dir_out = dir_in # dir_in = r'\\172.21.195.2\FTP-Share\ftp\planet_imgs\LES\archive\Ostashkovskoe_lesnichestvo\Case1\ostashkovskoe_2020may_cut1_PSScene4band_b21bb6c5-2af9-439c-912b-cc9a8492c0b2' # txt_ids = None#r'd:\digital_earth\KV_Tatarstan\kv_ids.txt' # dir_out = r'd:\rks\PL_new_201023\ostashkovskoe_2020may_cut1' preserve_original = True make_rgb = False vector_granule_path = r'\\172.21.195.2\FTP-Share\ftp\images\granules_grid.shp' ms2pms = False invert_red_blue = False if json_cover is None: json_cover = tempname('json') process().input(dir_in, skip_duplicates=False).GetCoverJSON(json_cover) pass names_tmpt = { 'sentinel': ('^S2[AB]_MSI.+\d$', [3, 2, 1]), 'sentinel_rgbn': ('^S2[AB].+RGBN$', [1, 2, 3]), 'planet': ('.*Analytic(MS)?(_SR)?$', [3, 2, 1]), 'planet_neuro': ('IM4-PLN.*', [1, 2, 3]), 'kanopus': ('KV.+MS.+L2', [1, 2, 3]), # 'resursp': ('RP.+L2$', [1,2,3]), 'resursp-grn': ('RP.+PMS\.L2\.GRN\d+$', [1, 2, 3]), 'resursp-grn_new': ('\d_+RP.+PMS\.L2$', [1, 2, 3]), 'kanopus_new_ref': ('^fr.+KV.+$', [1, 2, 3]), 'digital-globe': (r'\d+[A-Z]+\d+-.*P\d+$', [3, 2, 1]), }
def GetRasterPercentileUInteger(files, min_p=0.02, max_p=0.98, bands=[1, 2, 3], nodata=0, max=65536, manage_clouds=False): max = int(max) count = np.zeros((len(bands), max), np.uint64) sums = np.zeros((len(bands)), int) for file in files: raster = gdal.Open(file) if raster: if manage_clouds: f, n, e = split3(file) proc = process().input(f) for ascene in proc.scenes: if file == ascene.get_raster_path('Analytic'): clouds = float( get_from_tree(ascene.meta.container.get('xmltree'), 'cloudCoverPercentage')[0]) / 100.0 for i, band in enumerate(bands): values, numbers = np.unique( raster.GetRasterBand(band).ReadAsArray(), return_counts=True) nodatamatch = np.where(values == nodata) if len(nodatamatch) > 0: for val in nodatamatch: numbers[val] = 0 sum = np.sum(numbers) if manage_clouds and (clouds > 0): nocloud_sum = int(sum * (1 - clouds)) sum_ = 0 sums[i] += nocloud_sum else: clouds = False sums[i] += sum for val, num in zip(values, numbers): if clouds: sum_ += num if sum_ >= nocloud_sum: count[i, val] += num - sum_ + nocloud_sum break count[i, val] += num del values del numbers result = [] for sum, hystogram in zip(sums, count): min_num = sum * min_p max_num = sum * max_p cur_min_sum = 0 for i, num in enumerate(hystogram): cur_min_sum += num if cur_min_sum < min_num: continue else: min_val = i break cur_max_sum = sum for i, num in enumerate(hystogram[::-1]): cur_max_sum -= num if cur_max_sum > max_num: continue else: max_val = max - i break result.append((min_val, max_val)) return result
from geodata import * from image_processor import process, scene proc_path = r'\\172.21.195.215\thematic\source\ntzomz\102_2020_1913' xls_dict = xls_to_dict( r'\\172.21.195.215\thematic\source\ntzomz\102_2020_1913\cloud_rsp.xls') path_out = r'\\172.21.195.2\FTP-Share\ftp\image_exchange\rsp_cloud' imsys = 'RSP' save_pms = True proc = process().input(proc_path, imsys_list=[imsys], skip_duplicates=False) for pathxml in xls_dict: if xls_dict[pathxml]['mark'] == '1': folder, file = os.path.split(pathxml) copydir(folder, path_out) print('COPY: %s' % folder) if save_pms: datamask1 = scene(pathxml, imsys).datamask() id = xls_dict[pathxml]['id'] id_pms = id.replace('.MS', '.PMS') for ascene in proc.scenes: if ascene.meta.id == id_pms: if ShapesIntersect(datamask1, ascene.datamask()): copydir(ascene.path, path_out) print('COPY: %s' % ascene.path)
xls_path = r'\\172.21.195.2\thematic\Sadkov_SA\kanopus_dymka\102_2020_1159\report (new_code).xls' raster_data_path = r'd:\rks\s3\kanopus_missed\1159_PMS' folder_out = r'e:\rks\kanopus_dymka\1159' correct_vals = [20, 21, 22, 3] xls_dict = xls_to_dict(xls_path) raster_files = folder_paths(raster_data_path, 1, 'tif') for folder in xls_dict: test = re.search('[0-9]', xls_dict[folder].get('mark')) if test: test = int(test.group()) id = xls_dict[folder].get('id') if id: test = int(test) proc = process().input(folder) for ascene in proc.scenes: for file in raster_files: if split3(file)[1] == id: if IntersectRaster(file, ascene.datamask()): folder_name = os.path.split(path_in) copydir(folder, folder_out) shutil.copyfile( file, fullpath(r'%s\%s\%s.tif' % (folder_out, folder_name, id))) print('Folder copyied: %s' % id) continue else: print('Raster mismatch: %s' % id) else:
"4": "Turn left to go around obj", "5": "Turn right to go around obj", "6": "Go forward around obj", "7": "Go forward around obj", "8": "Turn right back to obj", "9": "Turn left back to obj" } turn_granularity = 0.3 move_distance = 5 last_behavior = -1 last_turn = agent.rotate_left while True: img = agent.get_image() fuzzy_vars = image_processor.process(img, agent.obst_hue, agent.dest_hue, agent.image_width) print("Fuzzy vars: " + str(fuzzy_vars)) behavior = controller.get_behavior(fuzzy_vars, last_behavior) print("Behavior: " + behaviors[str(behavior)]) last_behavior = behavior if behavior <= 0: last_turn(turn_granularity) if behavior <= -2: last_behavior = 0 agent.move_forward(move_distance) elif behavior == 1 or behavior == 6 or behavior == 7: agent.move_forward(move_distance) elif behavior == 2 or behavior == 4 or behavior == 9: agent.rotate_left(turn_granularity)