def GetUnmatchingScenes(source_scenes, s3_scenes, pms = False):
unmatched = OrderedDict()
matched = OrderedDict()
for id in source_scenes:
for s3id in s3_scenes:
source_id, meta_path = s3_scenes[s3id]
if id==source_id:
scene_dict = source_scenes[id]
scene_fullpath = GetSceneFullpath(scene_dict)
if scene_fullpath:
source_process = process().input(scene_fullpath)
temp_json = tempname('json')
source_process.GetCoverJSON(temp_json)
source_ds, source_lyr = get_lyr_by_path(temp_json)
s3_ds, s3_lyr = get_lyr_by_path(meta_path)
if source_lyr and s3_lyr:
source_feat = source_lyr.GetNextFeature()
s3_feat = s3_lyr.GetNextFeature()
match = CoverMatch(source_feat, s3_feat, pms = pms)
if match:
print('SCENE ALREADY EXIST: %s' % s3id)
matched[id] = source_scenes[id]
continue
else:
print('COVER ERROR: %s' % s3id)
else:
print('CANNOT OPEN SCENE: %s' % id)
unmatched[id] = source_scenes[id]
return matched, unmatched
def TotalCover(pout, files, srs = None):
driver = ogr.GetDriverByName('GeoJSON')
ds_out = driver.CreateDataSource(pout)
if srs is None:
srs = osr.SpatialReference()
srs.ImportFromEPSG(4326)
lyr_out = ds_out.CreateLayer('', srs, ogr.wkbMultiPolygon)
lyr_out.CreateField(ogr.FieldDefn('id', 4))
lyr_out.CreateField(ogr.FieldDefn('satid', 4))
feat_defn = lyr_out.GetLayerDefn()
for file in files:
ds_ = gdal.Open(file)
if ds_:
f,id,e = split3(file)
feat = ogr.Feature(feat_defn)
feat.SetField('id', id)
geom = RasterGeometry(ds_, reference=srs)
feat.SetGeometry(geom)
proc = process().input(f)
ascene = proc.scenes[0]
if ascene:
satid = ascene.meta.sat
feat.SetField('satid', satid)
lyr_out.CreateFeature(feat)
ds_out = None
def FindScenes(path_in, imsys_list = None, skip_duplicates = False, v_cover = None):
scenes = OrderedDict()
for folder_in in obj2list(path_in):
proc = process().input(folder_in, imsys_list=imsys_list, skip_duplicates = skip_duplicates)
for ascene in proc.scenes:
id = ascene.meta.id
if '.PAN' in id:
base_id = id.replace('.PAN','.PMS')
type = 'PAN'
elif '.MS' in id:
base_id = id.replace('.MS','.PMS')
type = 'MS'
elif '.PMS' in id:
base_id = id
type = 'PMS'
else:
print('Unknown type: %s' % id)
continue
if base_id in scenes:
if type in scenes[base_id]:
print('Scene already exists: %s' % id)
continue
else:
scenes[base_id][type] = ascene.fullpath
else:
scenes[base_id] = {'id': base_id, type: ascene.fullpath}
# !!! Correct for multiple source folders!
if not v_cover:
v_cover = fullpath(folder_in, 'v_cover.json')
if not os.path.exists(v_cover):
# scroll(v_cover)
proc.GetCoverJSON(v_cover)
return scenes, v_cover
def GetMetaDG(id):
files = globals()['dg_files']
if files:
names = flist(files, lambda x: split3(x)[1])
if id in names:
file = files[names.index(id)]
dg_proc = process().input(os.path.dirname(file))
# scroll(dg_proc.get_ids(), header=id)
dg_scene = dg_proc.scenes[0]
satid = dg_scene.meta.name('[sat]')
date = dg_scene.meta.name('[date]')
time = dg_scene.meta.name('[time]')
loc = dg_scene.meta.name('[location]')
lvl = dg_scene.meta.name('[lvl]')
# print(satid, date, time, loc, lvl)
return satid, date, time, loc, lvl
else:
return parse_dg(id)
def TotalCover(pout, files, srs=None):
driver = ogr.GetDriverByName('GeoJSON')
ds_out = driver.CreateDataSource(pout)
if srs is None:
srs = osr.SpatialReference()
srs.ImportFromEPSG(4326)
lyr_out = ds_out.CreateLayer('', srs, ogr.wkbMultiPolygon)
lyr_out.CreateField(ogr.FieldDefn('id', 4))
lyr_out.CreateField(ogr.FieldDefn('path', 4))
lyr_out.CreateField(ogr.FieldDefn('satid', 4))
lyr_out.CreateField(ogr.FieldDefn('geomerror', 8))
feat_defn = lyr_out.GetLayerDefn()
i = 0
for file in files:
try:
ds_ = gdal.Open(file)
except:
ds_ = None
if ds_:
i += 1
f, id, e = split3(file)
feat = ogr.Feature(feat_defn)
feat.SetField('id', id)
feat.SetField('path', file)
try:
proc = process().input(f)
if len(proc) == 1:
ascene = proc.scenes[0]
satid = ascene.meta.sat
feat.SetField('satid', satid)
except:
feat.SetField('satid', None)
try:
geom = RasterGeometry(ds_, reference=srs)
feat.SetGeometry(geom)
feat.SetField('geomerror', False)
except:
feat.SetField('geomerror', True)
lyr_out.CreateFeature(feat)
ds_out = None
return i
if raster_geometry:
break
else:
raster_geometry = None
if raster_geometry is None:
raster_geometry = raster_geom(gdal.Open(file), reference=srs)
if raster_geometry is None:
print('UNABLE TO EXTRACT GEOMETRY FROM: %s' % n)
continue
for feat in lyr_cover:
vector_geometry = feat.GetGeometryRef()
if raster_geometry.Intersects(vector_geometry):
print('MATCH: %s' % n)
if copy_:
copy_file(file, fullpath(pout, n, e), id=n)
else:
copy_paths.append(file)
if not f in scene_folders:
scene_folders.append(f)
break
if target=='LIST COPY PATHS':
if len(copy_paths) > 0:
with open(fullpath(pout, 'copy_paths.txt'), 'w') as txt:
txt.write('\n'.join(copy_paths))
try:
proc = process().input(scene_folders, imsys_list=['RSP'])
proc.GetCoverJSON(fullpath(pout, 'vector_cover.json'))
except:
print('ERROR CREATING VECTOR COVER')
# -*- coding: utf-8 -*-
from image_processor import process
from geodata import *
import sys
din = r'e:\tos3'
dout = r'e:\rks\s3'
tmpt_snt = '^S2[AB]_MSI.+\d$'
tmpt_pln = '.*AnalyticMS(_SR)?$'
tmpt_pln_neuro = 'IM4-PLN.*'
source = r'e:\rks\source'
tcover = r'e:\rks\s3\cover.json'
process().input(source).GetCoverJSON(tcover)
def cleansplit(str_, sep, join=''):
ids = str_.split(sep)
for i, id in enumerate(ids):
if i:
ids[i] = join+id
return ids
def get_export_list(din):
export = OrderedDict()
for path in folder_paths(din,1,'tif'):
f, n, e = split3(path)
if re.search(tmpt_snt, n):
ids = cleansplit(n, '_S2', 'S2')
for i, id in enumerate(ids):
# r'd:\digital_earth\resurs-p_new\krym',
# None
]
for path_in, path_out, path_data in zip(path_in_list, path_out_list,
path_data_list):
ds_in, lyr_in = get_lyr_by_path(path_in)
feat_in_list = lyr_in
json_fields(path_out, ogr.wkbMultiPolygon, 4326, fields_dict, feat_in_list,
translator)
if path_data is not None:
proc = image_processor.process().input(path_data)
ds_out, lyr_out = get_lyr_by_path(path_out)
for feat in lyr_out:
''' For Planet '''
ascene = proc.get_scene(feat.GetField('id') + '_3B_AnalyticMS')
if ascene is not None:
metadata = ascene.meta.container.get('xmltree')
feat.SetField('id', ascene.meta.id)
feat.SetField('id_s', ascene.meta.name('[location]'))
feat.SetField(
'id_neuro',
ascene.meta.name('[fullsat]-[date]-[location]-[lvl]'))
feat.SetField('datetime',
get_from_tree(metadata, 'acquisitionDateTime'))
# feat.SetField('clouds', get_from_tree(metadata, 'acquisitionDateTime'))
# path_in_list = folder_paths(path_in,1,'tif')
rgblist = []
files_list = []
codes_list = []
def get_code(file_name):
vals = file_name.split('_')
if len(vals) == 6:
return vals[2]
return vals[3]
codes_dict = {}
for ascene in process().input(path_in, imsys_list=['PLN']).scenes:
file = ascene.get_raster_path('Analytic')
f, n, e = split3(file)
code = get_code(n)
if re.search(r'^\d{8}_.+_3B_AnalyticMS_clip$', n):
files_list.append(file)
codes_list.append(code)
path_rgb = fullpath(folder_out, n + '.RGB', e)
if code not in codes_dict.keys():
codes_dict[code] = [file]
else:
codes_dict[code].append(file)
rgblist.append(path_rgb)
print('appended: %s' % n)
else:
out_folder = r'd:\rks\kanopus_cloud_test\pan'
out_pms = r'd:\rks\kanopus_cloud_test\pms'
suredir(out_folder)
suredir(out_pms)
def GetMSids(source_folder):
ids = []
for name in folder_paths(source_folder,1,'tif'):
if re.search('MS.+\.tif$', name.lower()):
ids.append(name[:-4])
return ids
ms_paths = folder_paths(ms_folder,1,'tif')
ms_ids = flist(ms_paths, lambda x: split3(x)[1])
proc = process().input(source_folder, skip_duplicates=False)
pan_paths = []
pan_ids = []
for ascene in proc.scenes:
if ascene.meta.type=='PAN':
# path = ascene.get_raster_path('pan')
path = folder_paths(ascene.path,1,'tif')[0] or None
if path:
pan_paths.append(path)
pan_ids.append(ascene.meta.id.strip())
else:
print('PATH NOT FOUND: %s' % ascene.meta.id)
# scroll(pan_ids)
# scroll(ms_ids)
if dir_out is None:
dir_out = dir_in
# dir_in = r'\\172.21.195.2\FTP-Share\ftp\planet_imgs\LES\archive\Ostashkovskoe_lesnichestvo\Case1\ostashkovskoe_2020may_cut1_PSScene4band_b21bb6c5-2af9-439c-912b-cc9a8492c0b2'
# txt_ids = None#r'd:\digital_earth\KV_Tatarstan\kv_ids.txt'
# dir_out = r'd:\rks\PL_new_201023\ostashkovskoe_2020may_cut1'
preserve_original = True
make_rgb = False
vector_granule_path = r'\\172.21.195.2\FTP-Share\ftp\images\granules_grid.shp'
ms2pms = False
invert_red_blue = False
if json_cover is None:
json_cover = tempname('json')
process().input(dir_in, skip_duplicates=False).GetCoverJSON(json_cover)
pass
names_tmpt = {
'sentinel': ('^S2[AB]_MSI.+\d$', [3, 2, 1]),
'sentinel_rgbn': ('^S2[AB].+RGBN$', [1, 2, 3]),
'planet': ('.*Analytic(MS)?(_SR)?$', [3, 2, 1]),
'planet_neuro': ('IM4-PLN.*', [1, 2, 3]),
'kanopus': ('KV.+MS.+L2', [1, 2, 3]),
# 'resursp': ('RP.+L2$', [1,2,3]),
'resursp-grn': ('RP.+PMS\.L2\.GRN\d+$', [1, 2, 3]),
'resursp-grn_new': ('\d_+RP.+PMS\.L2$', [1, 2, 3]),
'kanopus_new_ref': ('^fr.+KV.+$', [1, 2, 3]),
'digital-globe': (r'\d+[A-Z]+\d+-.*P\d+$', [3, 2, 1]),
}
def GetRasterPercentileUInteger(files,
min_p=0.02,
max_p=0.98,
bands=[1, 2, 3],
nodata=0,
max=65536,
manage_clouds=False):
max = int(max)
count = np.zeros((len(bands), max), np.uint64)
sums = np.zeros((len(bands)), int)
for file in files:
raster = gdal.Open(file)
if raster:
if manage_clouds:
f, n, e = split3(file)
proc = process().input(f)
for ascene in proc.scenes:
if file == ascene.get_raster_path('Analytic'):
clouds = float(
get_from_tree(ascene.meta.container.get('xmltree'),
'cloudCoverPercentage')[0]) / 100.0
for i, band in enumerate(bands):
values, numbers = np.unique(
raster.GetRasterBand(band).ReadAsArray(),
return_counts=True)
nodatamatch = np.where(values == nodata)
if len(nodatamatch) > 0:
for val in nodatamatch:
numbers[val] = 0
sum = np.sum(numbers)
if manage_clouds and (clouds > 0):
nocloud_sum = int(sum * (1 - clouds))
sum_ = 0
sums[i] += nocloud_sum
else:
clouds = False
sums[i] += sum
for val, num in zip(values, numbers):
if clouds:
sum_ += num
if sum_ >= nocloud_sum:
count[i, val] += num - sum_ + nocloud_sum
break
count[i, val] += num
del values
del numbers
result = []
for sum, hystogram in zip(sums, count):
min_num = sum * min_p
max_num = sum * max_p
cur_min_sum = 0
for i, num in enumerate(hystogram):
cur_min_sum += num
if cur_min_sum < min_num:
continue
else:
min_val = i
break
cur_max_sum = sum
for i, num in enumerate(hystogram[::-1]):
cur_max_sum -= num
if cur_max_sum > max_num:
continue
else:
max_val = max - i
break
result.append((min_val, max_val))
return result
from geodata import *
from image_processor import process, scene
proc_path = r'\\172.21.195.215\thematic\source\ntzomz\102_2020_1913'
xls_dict = xls_to_dict(
r'\\172.21.195.215\thematic\source\ntzomz\102_2020_1913\cloud_rsp.xls')
path_out = r'\\172.21.195.2\FTP-Share\ftp\image_exchange\rsp_cloud'
imsys = 'RSP'
save_pms = True
proc = process().input(proc_path, imsys_list=[imsys], skip_duplicates=False)
for pathxml in xls_dict:
if xls_dict[pathxml]['mark'] == '1':
folder, file = os.path.split(pathxml)
copydir(folder, path_out)
print('COPY: %s' % folder)
if save_pms:
datamask1 = scene(pathxml, imsys).datamask()
id = xls_dict[pathxml]['id']
id_pms = id.replace('.MS', '.PMS')
for ascene in proc.scenes:
if ascene.meta.id == id_pms:
if ShapesIntersect(datamask1, ascene.datamask()):
copydir(ascene.path, path_out)
print('COPY: %s' % ascene.path)
xls_path = r'\\172.21.195.2\thematic\Sadkov_SA\kanopus_dymka\102_2020_1159\report (new_code).xls'
raster_data_path = r'd:\rks\s3\kanopus_missed\1159_PMS'
folder_out = r'e:\rks\kanopus_dymka\1159'
correct_vals = [20, 21, 22, 3]
xls_dict = xls_to_dict(xls_path)
raster_files = folder_paths(raster_data_path, 1, 'tif')
for folder in xls_dict:
test = re.search('[0-9]', xls_dict[folder].get('mark'))
if test:
test = int(test.group())
id = xls_dict[folder].get('id')
if id:
test = int(test)
proc = process().input(folder)
for ascene in proc.scenes:
for file in raster_files:
if split3(file)[1] == id:
if IntersectRaster(file, ascene.datamask()):
folder_name = os.path.split(path_in)
copydir(folder, folder_out)
shutil.copyfile(
file,
fullpath(r'%s\%s\%s.tif' %
(folder_out, folder_name, id)))
print('Folder copyied: %s' % id)
continue
else:
print('Raster mismatch: %s' % id)
else:
"4": "Turn left to go around obj",
"5": "Turn right to go around obj",
"6": "Go forward around obj",
"7": "Go forward around obj",
"8": "Turn right back to obj",
"9": "Turn left back to obj"
}
turn_granularity = 0.3
move_distance = 5
last_behavior = -1
last_turn = agent.rotate_left
while True:
img = agent.get_image()
fuzzy_vars = image_processor.process(img, agent.obst_hue, agent.dest_hue, agent.image_width)
print("Fuzzy vars: " + str(fuzzy_vars))
behavior = controller.get_behavior(fuzzy_vars, last_behavior)
print("Behavior: " + behaviors[str(behavior)])
last_behavior = behavior
if behavior <= 0:
last_turn(turn_granularity)
if behavior <= -2:
last_behavior = 0
agent.move_forward(move_distance)
elif behavior == 1 or behavior == 6 or behavior == 7:
agent.move_forward(move_distance)
elif behavior == 2 or behavior == 4 or behavior == 9:
agent.rotate_left(turn_granularity)
