def naming_indexing(fname):
'''
Names and indexes images according to category.
Saves image's infos to .xlsx format in current working directory.
Use when all images are ready to be catalogued.
If not, use 'proper_index2.py' instead to avoid erasing original names.
Parameters
----------
fname: type=str
Name of the image folder
Returns:
--------
pandas DataFrame:
columns: name, website, online image ID, extension
'''
SOURCES = '/home/francois/Desktop/neuromod_image_bank/neuromod_image_bank_docs/sources.csv'
refs = pd.read_csv(SOURCES)['reference'].tolist()
urls = pd.read_csv(SOURCES)['URL'].tolist()
imageinfos_to_df = []
for subd in os.listdir(join(getcwd(), fname)):
for subdd in tqdm(os.listdir(join(getcwd(), fname, subd))):
if subd in subdd:
newsubdd = subdd[subdd.find(subd) + len(str(subd)) + 1:]
if 'bodypart' in subd: #corrections to uniformize labels
newsubdd = subd.replace('bodypart', 'body_part')
else:
newsubdd = subdd
os.rename(join(getcwd(), fname, subd, subdd),
join(getcwd(), fname, subd, newsubdd))
for allpics in os.walk(join(getcwd(), fname)):
counter = 1
for picture in tqdm(allpics[2]):
if os.path.isfile(join(allpics[0], picture)):
if counter <= 9:
okname = bname(dname(join(allpics[0], picture)))+ \
'0'+str(counter)+splitext(join(allpics[0],
picture))[1]
elif counter >= 10:
okname = bname(dname(join(allpics[0], picture)))+ \
str(counter)+splitext(join(allpics[0], picture))[1]
for ref in refs.__iter__():
if picture.find(ref) != -1:
longpath, ext = splitext(join(allpics[0], picture))
image_id = longpath[longpath.find(ref) + len(ref):]
imageinfos_to_df.append(
(picture, okname, ref, urls[refs.index(ref)],
image_id, ext))
os.rename(join(allpics[0], picture), join(allpics[0], okname))
counter += 1
imageinfos_to_df = pd.DataFrame(imageinfos_to_df,
columns=[
'picture', 'name', 'website', 'url',
'online_image_id', 'extension'
])
# imageinfos_to_df.columns =
imageinfos_to_df.to_excel(join(getcwd(), fname + 'DF.xlsx'))
def square_resize(fname, dimensions=(500, 500)):
'''
Resizes square aspect-ratio images to desired dimensions.
Doesn't overwrite the images; instead, a prefix corresponding
to 'dimensions' parameter is added before each image's and folder's
name (i.e: folder 'aquatic_mammal' --> '500_aquatic_mammal').
Parameters
----------
fname: type = str
Name of category images directory (ex: 'outdoor_sport')
dimensions: type = tuple
Tuple (width, length) indicating desired size in pixels
type(width) and type(length) = int
'width' & 'length' should be equal
Returns
-------
None'''
imdirect.monkey_patch() #Fixes unexpected image rotation while saving
prefix = str(dimensions[0]) + '_'
nfpath = join(getcwd(), prefix + fname)
os.system("mkdir {}".format(nfpath))
for categs in ls(join(getcwd(), fname)):
newcatpath = join(nfpath, prefix + categs)
os.system("mkdir {}".format(newcatpath))
for synname in ls(join(getcwd(), fname, categs)):
os.system("mkdir {}".format(join(newcatpath, prefix + synname)))
for allim in os.walk(join(getcwd(), fname, categs, synname)):
for subd in allim[1]:
os.system("mkdir {}".format(
join(newcatpath, prefix + synname, prefix + subd)))
for im_name in tqdm(allim[2]):
impath = join(allim[0], im_name)
image = Image.open(impath)
newpath = impath.replace(fname, prefix + fname, 1)
newpath = newpath.replace(categs, prefix + categs, 1)
newpath = newpath.replace(synname, prefix + synname, 1)
newpath = newpath.replace(bname(dname(newpath)),
prefix + bname(dname(newpath)),
1)
image.convert("RGB")
if image.size[0] > image.size[1]:
image = image.crop(
((image.size[0] - image.size[1]) / 2, 0,
(image.size[0] + image.size[1]) / 2,
image.size[1]))
elif image.size[0] < image.size[1]:
image = image.crop(
(0, (image.size[1] - image.size[0]) / 2,
image.size[0],
(image.size[1] + image.size[0]) / 2))
im_resized = image.resize(dimensions, Image.ANTIALIAS)
im_resized.save(join(dname(newpath), prefix + im_name),
'JPEG',
quality=90)
def square_image(dimensions=(500, 500)):
'''
Crops all images in a directory to square aspect ratio and resizes them
to 'dimensions'. Doesn't overwrite the original image directory.
'''
imdirect.monkey_patch()
pre = str(dimensions[0])+'_'
newdir = join(dname(os.getcwd()), pre+bname(os.getcwd()))
os.system("mkdir {}".format(newdir))
for allim in os.walk(os.getcwd()):
for picture in tqdm(allim[2]):
picpath = join(allim[0], picture)
if os.path.isdir(dname(picpath)):
subdir = join(newdir, pre+bname(dname(dname(dname(picpath)))))
subdir2 = join(subdir, pre+bname(dname(dname(picpath))))
subdir3 = join(subdir2, pre+bname(dname(picpath)))
print(newdir+'\n', subdir+'\n', subdir2+'\n', subdir3+'\n')
os.system("mkdir {}".format(subdir))
os.system("mkdir {}".format(subdir2))
os.system("mkdir {}".format(subdir3))
image = Image.open(picpath)
if image.mode != 'RGB':
image.convert("RGB")
if image.size[0] > image.size[1]:
image = image.crop(((image.size[0]-image.size[1])/2,
0,
(image.size[0]+image.size[1])/2,
image.size[1]))
elif image.size[0] < image.size[1]:
image = image.crop((0,
(image.size[1]-image.size[0])/2,
image.size[0],
(image.size[1]+image.size[0])/2))
im_resized = image.resize(dimensions, Image.ANTIALIAS)
im_resized.save(join(subdir3, pre+picture), 'JPEG', quality=90)
def proper_index2(folder_name):
'''
Names and indexes images according to category.
Doesn't affect the image's web reference as would 'naming_indexing'
would (useful to inventoriate and unify labels across categories).
'''
sources = pd.read_csv(join(dname(os.getcwd()), 'sources.csv'))
references = sources['reference'].tolist()
folderpath = join(os.getcwd(), folder_name)
for root, dirs, files in os.walk(folderpath):
counter = 1
for file in files:
dir_path = os.path.dirname(join(root, file))
dir_name = os.path.basename(dir_path)
if 'bodypart' in dir_name:
dir_name = dir_name.replace('bodypart', 'body_part')
for ref in references.__iter__():
if file.find(ref) != -1:
suffix = file[file.find(ref):]
else:
suffix = ''
if counter <= 9:
ok_name = dir_name + '0' + str(counter) + suffix
elif counter >= 10:
ok_name = dir_name + str(counter) + suffix
ok_path = join(root, ok_name)
os.rename(join(root, file), os.path.splitext(ok_path)[0])
counter += 1
print(len(os.listdir(dir_path)))
def test_adv_summary(self):
fpath = apath(
pjoin(dname(__file__), 'resources', 'tripod', 'north', 'adv',
'MCR13N1T05adv1s-cal.nc'))
shutil.copy2(fpath, self.tmpf)
for of in adv.enhance_summary(self.tmpf):
logger.info('Wrote output: {}'.format(of))
def test_adcp_wvs(self):
fpath = apath(
pjoin(dname(__file__), 'resources', 'tripod', 'north', 'adcp',
'MCR13N1T_wvs.nc'))
shutil.copy2(fpath, self.tmpf)
for of in adcp.enhance_wvs(self.tmpf):
logger.info('Wrote output: {}'.format(of))
def setup_method(self, method):
outdir = pjoin(dname(__file__), 'output', type(self).__name__)
try:
os.makedirs(outdir)
except OSError:
pass
self.tmpf = pjoin(outdir, '{}.nc'.format(method.__name__))
def setup_method(self, method):
outdir = pjoin(dname(__file__), 'output', type(self).__name__)
try:
os.makedirs(outdir)
except OSError:
pass
self.tmpf = pjoin(outdir, '{}.nc'.format(method.__name__))
def run(self, fLog):
"""Basically this is the same code as the info.py request handler it
just is not triggered from apache so none of the CGI stuff is available
"""
# Assume it doesn't work, until we know it does
self.nRetCode = 13
fLog.write(" Exec: %s" % self.uCmd)
self.proc = subprocess.Popen(self.uCmd,
shell=True,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True)
(sStdOut, sStdErr) = self.proc.communicate()
fLog.write(sStdErr)
if self.proc.returncode != 0:
fLog.write(u"Error detected in command %s" % self.uCmd)
self.nRetCode = self.proc.returncode
return
sDir = dname(self.sOutPath)
if not os.path.isdir(sDir):
os.makedirs(sDir)
# Read the output in as a JSON object (may throw and that's okay)
dOut = json.loads(sStdOut, encoding="utf-8")
if self.sDas2Url != None:
dOut['x_links'] = [{
"tag": "das2Stream",
"description":
"Access to the upstream Das2 data source for this HAPI endpoint",
"mime-type": "application/vnd.das2.das2stream",
"url": self.sDas2Url
}]
# Override the description to match the sub source if needed
if self.sDescription:
dOut['description'] = self.sDescription
sJsonOut = json.dumps(dOut,
ensure_ascii=False,
sort_keys=True,
indent=3)
fOut = open(self.sOutPath, 'w')
fOut.write(sJsonOut)
fOut.close()
fLog.write("Cache file %s written" % self.sOutPath)
self.nRetCode = 0
self.proc = None
def _fileOutWithEx(sFile, tData):
fLog = tData[0]
sPrefix = tData[1]
lOut = tData[2]
sRelPath = None
sDescription = None
if not sFile.lower().endswith('.dsdf'):
return True
if sFile.lower().endswith('_dirinfo_.dsdf'):
return True
sDir = "%s/"%dname(sFile).replace(sPrefix, '')
# Remove items that are in directories name test from the
# output
if sDir.find(g_sTestDir) != -1:
return True
fIn = codecs.open(sFile, 'rb', encoding='utf-8')
#fLog.write("INFO: Parsing %s"%sFile)
for sLine in fIn:
iComment = sLine.find(';')
if iComment != -1:
sLine = sLine[:iComment]
if sLine.find('exampleRange') != -1:
sRelPath = sFile.replace(sPrefix, '').replace('.dsdf','')
if sDir != "/" and sDir not in lOut:
lOut.append(sDir)
if sLine.find('description') != -1:
lLine = sLine.split('=')
if len(lLine) > 1:
sDescription = "".join(lLine[1:])
sDescription = sDescription.strip("\"' \r\n\t")
continue
if sLine.find('rename') != -1:
lLine = sLine.split('=')
if len(lLine) > 1 and lLine[0].strip().lower() == 'rename':
fIn.close()
return True
if sRelPath:
if sDescription:
lOut.append( u"%s|%s"%(sRelPath,sDescription ) )
else:
lOut.append( sRelPath )
return True
def imcount():
imdict = {}
for allimages in os.walk(os.path.relpath('neuromod_image_bank')):
for folder in enumerate(allimages[1]):
folderdict = {
bname(dname(join(allimages[0], folder))):
(folder, len(ls(join(allimages[0], folder[1]))))
}
imdict[folder[0]] = folderdict
print(folderdict)
print(folder, len(os.listdir(os.path.join(allimages[0], folder))))
def load_companies():
companies = []
for company_file_path in glob(join(dname(__file__), 'companies', '*.csv')):
with open(company_file_path) as company_file:
reader = csv.reader(company_file)
reader.next() # skip the header row
for company in reader:
companies.append((len(companies) + 1, company[1]))
return companies
def test_adv_summary(self):
fpath = apath(pjoin(
dname(__file__),
'resources',
'tripod',
'north',
'adv',
'MCR13N1T05adv1s-cal.nc'
))
shutil.copy2(fpath, self.tmpf)
for of in adv.enhance_summary(self.tmpf):
logger.info('Wrote output: {}'.format(of))
def test_adcp_wvs(self):
fpath = apath(pjoin(
dname(__file__),
'resources',
'tripod',
'north',
'adcp',
'MCR13N1T_wvs.nc'
))
shutil.copy2(fpath, self.tmpf)
for of in adcp.enhance_wvs(self.tmpf):
logger.info('Wrote output: {}'.format(of))
def square_image(folderpath, length=500):
'''
Alt square_image.py using os.makedirs() insead of os.system
Crops all images in a directory to square aspect ratio and resizes them
to 'dimensions'. Doesn't overwrite the original image directory.
'''
imdirect.monkey_patch()
pre = str(length) + '_'
impaths = loadimages(folderpath)
newpaths = []
for impath in enumerate(impaths):
newparts = []
parts = splitall(impath[1])[1:-1]
for part in parts:
newparts.append(pre + part)
newpaths.append(
(impath[1], join(*[pre + part for part in splitall(impath[1])]),
dname(join(*[pre + part for part in splitall(impath[1])]))))
for newpath in tqdm(newpaths):
if not os.path.exists(newpath[2]):
os.mkdir(newpath[1])
image = Image.open(newpath[1])
# newdir = join(dname(folderpath), pre+bname(folderpath)
# os.system("mkdir {}".format(newdir))
# for allim in os.walk(os.getcwd()):
# for picture in tqdm(allim[2]):
# picpath = join(allim[0], picture)
# if os.path.isdir(dname(picpath)):
# subdir = join(newdir, pre+bname(dname(dname(dname(picpath)))))
# subdir2 = join(subdir, pre+bname(dname(dname(picpath))))
# subdir3 = join(subdir2, pre+bname(dname(picpath)))
# print(newdir+'\n', subdir+'\n', subdir2+'\n', subdir3+'\n')
# os.makedirs(subdir3, exist_ok=True)
# os.system("mkdir {}".format(subdir), exist_ok=True)
# os.system("mkdir {}".format(subdir2), exist_ok=True)
# os.system("mkdir {}".format(subdir3), exist_ok=True)
if image.mode != 'RGB':
image.convert("RGB")
if image.size[0] > image.size[1]:
image = image.crop(
((image.size[0] - image.size[1]) / 2, 0,
(image.size[0] + image.size[1]) / 2, image.size[1]))
elif image.size[0] < image.size[1]:
image = image.crop(
(0, (image.size[1] - image.size[0]) / 2, image.size[0],
(image.size[1] + image.size[0]) / 2))
im_resized = image.resize((length, length), Image.ANTIALIAS)
im_resized.save(newpath[1], 'JPEG', quality=100)
def get_images(datas):
''' Description
-----------
DF containing each image_concept\
Returns
-------
ImageBank.images
'''
images = pd.DataFrame((row[1] for row in datas.iterrows()
if pd.isnull(row[1]['subordinates']).all()),
columns=list(datas.columns))
images['names'] = [bname(row[1]['path']) for row in images.iterrows()]
images['folders'] = [
bname(dname(row[1].path)) for row in images.iterrows()
]
images = images.set_index('folders')
# images_ind = images.set_index('names', append=True).index
return images.sort_index(kind='mergesort')
def getDirUri(U, fLog, dConf, dDirInfo, sCatDir):
if "uri" in dDirInfo:
sUri = dDirInfo["uri"].strip("\"' /r/n")
fLog.write("INFO: Using exlicit URI for directory %s, %s" %
(sPath, sUri))
return sUri
sRelPath = None
_sOrigCatDir = sCatDir
while sCatDir != dConf['DSDF_ROOT']:
# Go up one
if sRelPath == None:
sRelPath = bname(sCatDir)
else:
sRelPath = "%s/%s" % (bname(sCatDir), sRelPath)
sCatDir = dname(sCatDir)
sCatDsdf = pjoin(sCatDir, '_dirinfo_.dsdf')
if os.path.isfile(sCatDsdf):
fIn = open(sCatDsdf, "rb")
dCatDsdf = U.dsdf.readDsdf(fIn, fLog)
fIn.close()
if "uri" in dCatDsdf:
fLog.write(
"INFO: Directory %s URI set relative to directory %s URI"
% (_sOrigCatDir, sCatDir))
sUri = dDsdf["uri"].strip("\"' /r/n")
return "%s/%s" % (sUri, sRelPath)
# Still here huh, okay
if "SITE_PATHURI" not in dConf:
U.webio.serverError(fLog,
"No pathUri setting along the path of _dirinfo_.dsdf files leading "+\
"the path to file %s and fall back value SITE_PATHURI not set in %s"%(
pjoin(_sOrigCatDir, '_dirinfo_.dsdf'), dConf['__file__']))
return None
fLog.write(
"INFO: Directory %s URI set relative to config file SITE_PATHURI: %s"
% (_sOrigCatDir, dConf['SITE_PATHURI']))
return "%s/%s" % (dConf['SITE_PATHURI'], sRelPath)
from os.path import dirname as dname
from os.path import join
from os import listdir as ls
import pandas as pd
from pandas import DataFrame as df
from taskfunctions import flatten
from taskfunctions import loadimages
from taskfunctions import splitall
with open('../docs/index_names.json', 'r') as json_file:
read_names = json.load(json_file)
datas_json = json.dumps(cib_json)
allim = loadimages()
tags = flatten([splitall(dname(fpath).split(IMDIR)[1])[1:] for fpath in allim])
freqs = [dict((tag, Counter(bname(dname(tag))).values()) for tag in flatten(tags))]
freqs = pd.Series(dict(Counter(flatten(tags)))).sort_index()
json_mapper = json.dumps(cib_json)
json_mapper
json_freqs = freqs.to_json()
freq_json = json.dumps(freqs, indent=4)
def flatten_json(dictionary):
flattened = []
def flat(data, name=''):
if type(data) is dict:
for d in data:
flat(data[d], name + d + ',')
elif type(data) is list:
def renamefaces(
facepath='/home/francois/cib/images/animate_being/animal/animal_face'):
flist = list(dict.fromkeys([dname(item) for item in loadimages(facepath)]))
for item in flist:
if '_face' not in bname(item):
os.rename(item, join(dname(item), bname(item) + '_face'))
