def run():
print("-----------------------------------------")
print("local ssd")
ssd = "X:/cam_front_center"
process_folder(ssd) # on a local ssd.
print("-----------------------------------------")
print("nas")
with f.timing("get camera folders", units=1):
folders, jsons, labels = f.get_camera_folders(f.get_path(),
root=f.get_root())
print(len(folders), " camera folders")
for folder in folders:
process_folder(folder)
if True:
break
print("-----------------------------------------")
[print("local ssd")]
process_folder(ssd) # on a local ssd.ls ..
print("-----------------------------------------")
print("just read some files in a camera folder.")
print("nas")
with f.timing("get camera folders", units=1):
folders, jsons, labels = f.get_camera_folders(f.get_path(),
root=f.get_root())
print(len(folders), " camera folders")
for folder in folders:
process_folder(folder)
if True:
break
def conflicts(source):
root = functions.get_root(cons.sprpckg_dir+source+".xml")
""" Resolving conflicts """
text.text_formatting(_(">> Check for conflicts..."), 0, 'info')
conflictcount = len(root[1])
conflicts = root[1]
if conflictcount > 0:
text.text_formatting(str(conflictcount) + _(" conflicts have found."), 1, 'info')
text.text_formatting(_("Following suprapackage(s) will remove. "), 0, 'info')
for conf in root[1]:
try:
text.text_formatting(conf.text + " ==> " + get.description(conf.text), 1, '')
except:
text.text_formatting(conf.text + _(" ==> There is no description for this suprapackage"), 1, 'warning')
if os.path.isfile(cons.sprpckg_dir + conf.text + ".xml"):
answer = input(_("Are you want to remove these suprapackages (Y/N): "))
answer.upper()
elif cons.confm:
answer = _("Y")
else:
answer = _("Y")
if answer == _("Y"):
for conf in root[1]:
package = conf.text
if suprapackage_check(package):
remove(package,"","")
else:
text.text_formatting(package + _(" is not installed"), 0, 'warning')
else:
text.text_formatting(package + _("Suprapackage couldn't installed."), 0, 'error')
sys.exit(1)
else:
text.text_formatting(_("There is a no conflict"), 1, 'info')
def dependencies(source,action):
text.text_formatting(_(">> Resolving dependencies.."), 0, 'info')
root = functions.get_root(cons.sprpckg_dir+source+".xml")
#Bağımlılıklar çözülüyor
dependcount = len(root[2])
dependencies = root[2]
if dependcount > 0:
text.text_formatting(str(dependcount) + _(" dependencies have found."), 1, 'info')
#Tamamen kaldırma işlemi için ....
if action == "remove":
text.text_formatting(_("Following suprapackage(s) will remove. "), 1, 'info')
for dep in root[2]:
try:
text.text_formatting(dep.text + " ==> " + get_description(dep.text), 1, 'info')
except:
text.text_formatting(dep.text + _(" ==> There is no description for this suprapackage"), 1, 'warning')
if cons.confm:
answer = _("Y")
else:
answer = input(_("Are you want to remove these suprapackages (Y/N): "))
answer = answer.upper()
if answer == _("Y"):
for dep in root[2]:
package = dep.text
functions.remove(package,"skip",source)
else:
text.text_formatting(_("Dependencies coulnd't removed so remove process couldn't continue."), 0, 'error')
sys.exit(1)
else:
text.text_formatting(_("Following suprapackage(s) will install. "), 0, 'info')
for dep in root[2]:
if not search.srch_pynr(dep.text,'Peynir/Name','absolute') and not suprapackage_check(str(dep.text)):
text.text_formatting(dep.text + _(" couldn't found in repository, you can install it manually"), 1, 'warning')
os.remove(cons.sprpckg_dir + source + ".xml")
sys.exit(1)
else:
try:
text.text_formatting(dep.text + " ==> " + get_description(dep.text), 1, 'info')
except:
text.text_formatting(dep.text + _(" ==> There is no description for this suprapackage"), 1, 'info')
if cons.confm:
answer = _("Y")
else:
answer = input(_("Are you want to install these suprapackages (Y/N): "))
answer = answer.upper()
if answer == _("Y"):
for dep in root[2]:
package = dep.text
if not functions.suprapackage_check(package):
functions.install(package,"","") #İç içe bağımlılık sorunu olacak o neden ilave bir fonksiyon paramatresi ise bu soruyu bir kez sordurulabilir
else:
text.text_formatting(">> " + package + _(" is already installed"), 0, 'info')
mdfy.modify_add(cons.sprpckg_dir+package+".xml","<Dependencies","'>"," " + source,"previous")
else:
os.remove(cons.sprpckg_dir+source+".xml")
text.text_formatting(_("Dependencies coulnd't installed so install process couldn't continue."), 0, 'error')
sys.exit(1)
else:
text.text_formatting(_("There is a no dependencies"), 1, 'info')
def local_dependency(source,package):
"""Get local dependencies when suprapackage removing from system"""
dep_list = []
source_root = functions.get_root(cons.sprpckg_dir+source+".xml")
local_dep = source_root[2].attrib['local']
for i in local_dep.strip().split(" "):
if i != package and len(local_dep.strip().split(" ")) > 0:
dep_list.append(i)
return dep_list
def local_dependency(source, package):
"""Get local dependencies when suprapackage removing from system"""
dep_list = []
source_root = functions.get_root(cons.sprpckg_dir + source + ".xml")
local_dep = source_root[2].attrib['local']
for i in local_dep.strip().split(" "):
if i != package and len(local_dep.strip().split(" ")) > 0:
dep_list.append(i)
return dep_list
def rmv_local_dependencies(source):
root = functions.get_root(cons.sprpckg_dir+source+".xml")
dependcount = len(root[2])
dependencies = root[2]
if dependcount > 0:
for dep in dependencies:
if os.path.isfile(cons.sprpckg_dir + dep.text +".xml"):
mdfy.modify_rmv(cons.sprpckg_dir + dep.text +".xml","<Dependencies ",""," " + source,"next","",4)
else:
text.text_formatting(_("There is no file to remove local dependencies"), 1, 'warning')
def get_files(units=None):
images, jsons, labels = bf.get_camera_folders(bf.get_path(),
root=bf.get_root())
print(len(images), ' folders for images')
l = []
for folder in images:
l = get_pngs(folder)
break # just do one now.
folder = images[0]
l = l if units is None else l[:units]
return l
def get_description(package):
repo_root = functions.get_root(cons.repo)
repo_search = repo_root[1].findall('Peynir/Name')
repo_search1 = repo_root[1].findall('Peynir/Summary')
for comp in range(len(repo_search)):
if repo_search[comp].text == package:
sayi = comp
break
try:
return repo_search1[(sayi * 2) + 1].text
except:
return _("There is no description for this suprapackage")
def get_description(package):
repo_root = functions.get_root(cons.repo)
repo_search = repo_root[1].findall('Peynir/Name')
repo_search1 = repo_root[1].findall('Peynir/Summary')
for comp in range(len(repo_search)):
if repo_search[comp].text == package:
sayi = comp
break
try:
return repo_search1[(sayi*2)+1].text
except:
return _("There is no description for this suprapackage")
def rmv_local_dependencies(source):
root = functions.get_root(cons.sprpckg_dir + source + ".xml")
dependcount = len(root[2])
dependencies = root[2]
if dependcount > 0:
for dep in dependencies:
if os.path.isfile(cons.sprpckg_dir + dep.text + ".xml"):
mdfy.modify_rmv(cons.sprpckg_dir + dep.text + ".xml",
"<Dependencies ", "", " " + source, "next", "",
4)
else:
text.text_formatting(
_("There is no file to remove local dependencies"), 1,
'warning')
def srch_pynr(srch,node,action):
"""Search suprapackage in repository"""
repo_root = functions.get_root(cons.repo)
repo_search = repo_root[1].findall(node)
if srch == "all":
text.text_formatting(str(len(repo_root[1])) + " suprapackages have found.", 0, 'info')
for rep in repo_root[1]:
text.text_formatting(rep[0].text + " ==> " + functions.get_description(rep[0].text), 1, 'info')
else:
for comp in range(len(repo_search)):
if action == "find":
if similarity(str(repo_search[comp].text),str(srch)) > 0.45 and int(len(repo_search)) >0:
text.text_formatting(_("-> Found ") + repo_search[comp].text + _(" similarity is ") + str(similarity(str(repo_search[comp].text),str(srch))*100)+"%", 1, 'info')
if repo_search[comp].text == srch and action == "absolute":
return True
break
def conflicts(source):
root = functions.get_root(cons.sprpckg_dir + source + ".xml")
""" Resolving conflicts """
text.text_formatting(_(">> Check for conflicts..."), 0, 'info')
conflictcount = len(root[1])
conflicts = root[1]
if conflictcount > 0:
text.text_formatting(
str(conflictcount) + _(" conflicts have found."), 1, 'info')
text.text_formatting(_("Following suprapackage(s) will remove. "), 0,
'info')
for conf in root[1]:
try:
text.text_formatting(
conf.text + " ==> " + get.description(conf.text), 1, '')
except:
text.text_formatting(
conf.text +
_(" ==> There is no description for this suprapackage"), 1,
'warning')
if os.path.isfile(cons.sprpckg_dir + conf.text + ".xml"):
answer = input(
_("Are you want to remove these suprapackages (Y/N): "))
answer.upper()
elif cons.confm:
answer = _("Y")
else:
answer = _("Y")
if answer == _("Y"):
for conf in root[1]:
package = conf.text
if suprapackage_check(package):
remove(package, "", "")
else:
text.text_formatting(package + _(" is not installed"), 0,
'warning')
else:
text.text_formatting(
package + _("Suprapackage couldn't installed."), 0, 'error')
sys.exit(1)
else:
text.text_formatting(_("There is a no conflict"), 1, 'info')
def run():
print("--------------------------------------------")
x,y,z=f.get_camera_folders(f.get_path(),root=f.get_root())
title="cars by folder:"
for i,folder in enumerate(x):
files=f.get_files(folder,"*.png")
units=len(files)
print(i,"folder: "+folder,"has:",len(files),"files.")
with f.timing("folder: "+folder+" ["+str(i)+"] has: "+str(len(files))+" files.",units,title):
ds=tff.make_tensor_slices_dataset_list(files)
print("----------------------------------------------")
for i,folder in enumerate(x):
files=f.get_files(folder,"*.png")
units=len(files)
print(i,"folder: "+folder,"has:",len(files),"files.")
with f.timing("folder: "+folder+" ["+str(i)+"] has: "+str(len(files))+" files.",units,title):
ds=tff.make_tensor_slices_dataset_list(files)
mapped=ds.map(tff.parse1and,tff.autotune)
tff.do_enumeration(mapped,parse2=tff.write_file)
print("--------------------------------------------")
def srch_pynr(srch, node, action):
"""Search suprapackage in repository"""
repo_root = functions.get_root(cons.repo)
repo_search = repo_root[1].findall(node)
if srch == "all":
text.text_formatting(
str(len(repo_root[1])) + " suprapackages have found.", 0, 'info')
for rep in repo_root[1]:
text.text_formatting(
rep[0].text + " ==> " + functions.get_description(rep[0].text),
1, 'info')
else:
for comp in range(len(repo_search)):
if action == "find":
if similarity(str(repo_search[comp].text),
str(srch)) > 0.45 and int(len(repo_search)) > 0:
text.text_formatting(
_("-> Found ") + repo_search[comp].text +
_(" similarity is ") + str(
similarity(str(repo_search[comp].text), str(srch))
* 100) + "%", 1, 'info')
if repo_search[comp].text == srch and action == "absolute":
return True
break
else:
print("length is correct")
#missing=set(['L:/a2d2/camera_lidar_semantic/20181016_095036/camera/cam_front_center','L:/a2d2/camera_lidar_semantic/20181204_191844/camera/cam_front_center'])
missing_ = [
'20181016_095036/camera/cam_front_center',
'20181204_191844/camera/cam_front_center'
]
missing = [join(path, part) for part in missing_]
print("we expect problems with:", missing)
# L:/a2d2/camera_lidar_semantic/20181016_095036/camera/cam_front_center
# L:/a2d2/camera_lidar_semantic/20181204_191844/camera/cam_front_center
return get_files_from_camera_folders(
(camera_folders, )) # do not remove the ','
root = f.get_root()
path = f.get_path()
(filename, image_filename, json_filename,
label_filename) = f.get_my_filenames()
image_filename = filename
actual = []
if os.path.exists(image_filename):
print(image_filename + " exists")
print("reading:", image_filename)
actual = f.read_filenames_from_disk(image_filename)
else:
print(image_filename + " does not exist.")
print("examining camera folder for files")
good = get_filenames_from_camera_folders(path, root=root)
print(type(good), len(good), "file names")
print(good[0])
def dependencies(source, action):
text.text_formatting(_(">> Resolving dependencies.."), 0, 'info')
root = functions.get_root(cons.sprpckg_dir + source + ".xml")
#Bağımlılıklar çözülüyor
dependcount = len(root[2])
dependencies = root[2]
if dependcount > 0:
text.text_formatting(
str(dependcount) + _(" dependencies have found."), 1, 'info')
#Tamamen kaldırma işlemi için ....
if action == "remove":
text.text_formatting(_("Following suprapackage(s) will remove. "),
1, 'info')
for dep in root[2]:
try:
text.text_formatting(
dep.text + " ==> " + get_description(dep.text), 1,
'info')
except:
text.text_formatting(
dep.text +
_(" ==> There is no description for this suprapackage"
), 1, 'warning')
if cons.confm:
answer = _("Y")
else:
answer = input(
_("Are you want to remove these suprapackages (Y/N): "))
answer = answer.upper()
if answer == _("Y"):
for dep in root[2]:
package = dep.text
functions.remove(package, "skip", source)
else:
text.text_formatting(
_("Dependencies coulnd't removed so remove process couldn't continue."
), 0, 'error')
sys.exit(1)
else:
text.text_formatting(_("Following suprapackage(s) will install. "),
0, 'info')
for dep in root[2]:
if not search.srch_pynr(dep.text, 'Peynir/Name',
'absolute') and not suprapackage_check(
str(dep.text)):
text.text_formatting(
dep.text +
_(" couldn't found in repository, you can install it manually"
), 1, 'warning')
os.remove(cons.sprpckg_dir + source + ".xml")
sys.exit(1)
else:
try:
text.text_formatting(
dep.text + " ==> " + get_description(dep.text), 1,
'info')
except:
text.text_formatting(
dep.text +
_(" ==> There is no description for this suprapackage"
), 1, 'info')
if cons.confm:
answer = _("Y")
else:
answer = input(
_("Are you want to install these suprapackages (Y/N): "))
answer = answer.upper()
if answer == _("Y"):
for dep in root[2]:
package = dep.text
if not functions.suprapackage_check(package):
functions.install(
package, "", ""
) #İç içe bağımlılık sorunu olacak o neden ilave bir fonksiyon paramatresi ise bu soruyu bir kez sordurulabilir
else:
text.text_formatting(
">> " + package + _(" is already installed"), 0,
'info')
mdfy.modify_add(cons.sprpckg_dir + package + ".xml",
"<Dependencies", "'>", " " + source,
"previous")
else:
os.remove(cons.sprpckg_dir + source + ".xml")
text.text_formatting(
_("Dependencies coulnd't installed so install process couldn't continue."
), 0, 'error')
sys.exit(1)
else:
text.text_formatting(_("There is a no dependencies"), 1, 'info')
others=list(set(maybies)-set(files))
printffo(files,folders,others)
return (files,folders,others)
def printijo(imageFiles,jsonFiles,otherFiles):
print("image_files:",imageFiles[:3])
print("json_files:",jsonFiles[:3])
print("others:",otherFiles[:3])
def getijo(l:list): # images, jsons, others - should only be given files.
imageFiles=[e for e in l if '.png' in e]
maybeFiles=list(set(l)-set(imageFiles))
jsonFiles=[e for e in maybeFiles if '.json' in e]
otherFiles=list(set(maybeFiles)-set(jsonFiles))
if len(maybeFiles)!=0: print("maybies:",maybeFiles[:3])
printijo(imageFiles,jsonFiles,otherFiles)
return imageFiles,jsonFiles,otherFiles
path=op.join(f.get_root(),f.get_path()) # these will be sorted
(folders,y,z)=f.get_camera_folders(path)
print('--------------------------')
f.ppl0(folders,name="folders from get camera folders",print_=True)
print('--------------------------')
files,folders,others=getffo(path)
print('--------------------------')
for folder in folders:
(files2,folders2,others2)=getffo(op.join(path,folder))
for folder2 in folders2:
print("process folder:",folder2)
files3,folders3,others3=getffo(op.join(path,folder,folder2)) # may need camera or lifar sun directory?
for folder3 in folders3:
print("process folder:",folder2)
files4,folders4,others4=getffo(op.join(path,folder,folder2,folder3)) # may need camera or lifar sun directory?
(imageFiles,jsonFiles,otherFiles)=getijo(files4)
with f.timing("import tensorflow",1):
import tensorflow as tf
print(tf.__version__)
import tffunctions as tff
x,y,z=f.get_lists_of_filenames()
print("got (",len(x),len(y),len(z),") files.",flush=True)
first=x[0]
print("first file:",first,os.path.exists(first))
path=f.path_head(first)
filename=f.path_leaf(first)
print(path,filename)
maybe='L:/ss/sem/20180807_145028/camera/cam_front_center/20180807145028_camera_frontcenter_000000091.png'
if maybe==first:
print("maybe =")
print("maybe file:",maybe,os.path.exists(maybe))
x,y,z=f.get_camera_folders(f.get_path(),root=f.get_root())
title="cars by folder:"
#with f.timing("folder: "+folder+" ["+str(i)+"] has: "+str(len(files))+" files.",units,title):
# ds=tff.make_tensor_slices_dataset_list(files)
filesets=f.getFilesets(x,"*.png")
print(len(filesets),"filesets.")
with f.timing("make datasets from filename sets.",len(x),title):
datasets=tff.makeDatasets(filesets)
print(len(datasets),"datasets, type:",type(datasets))
tff.time_enumeration(datasets,units=len(datasets)) # was 1892?
# this will make the 44 datasets, one for each camera folder
# naybe this is enough for this file.
exit()
print("---------------------------------------")
def concat_datasets(datasets):
ds0 = tf.data.Dataset.from_tensors(datasets[0])