def naming_indexing(fname): ''' Names and indexes images according to category. Saves image's infos to .xlsx format in current working directory. Use when all images are ready to be catalogued. If not, use 'proper_index2.py' instead to avoid erasing original names. Parameters ---------- fname: type=str Name of the image folder Returns: -------- pandas DataFrame: columns: name, website, online image ID, extension ''' SOURCES = '/home/francois/Desktop/neuromod_image_bank/neuromod_image_bank_docs/sources.csv' refs = pd.read_csv(SOURCES)['reference'].tolist() urls = pd.read_csv(SOURCES)['URL'].tolist() imageinfos_to_df = [] for subd in os.listdir(join(getcwd(), fname)): for subdd in tqdm(os.listdir(join(getcwd(), fname, subd))): if subd in subdd: newsubdd = subdd[subdd.find(subd) + len(str(subd)) + 1:] if 'bodypart' in subd: #corrections to uniformize labels newsubdd = subd.replace('bodypart', 'body_part') else: newsubdd = subdd os.rename(join(getcwd(), fname, subd, subdd), join(getcwd(), fname, subd, newsubdd)) for allpics in os.walk(join(getcwd(), fname)): counter = 1 for picture in tqdm(allpics[2]): if os.path.isfile(join(allpics[0], picture)): if counter <= 9: okname = bname(dname(join(allpics[0], picture)))+ \ '0'+str(counter)+splitext(join(allpics[0], picture))[1] elif counter >= 10: okname = bname(dname(join(allpics[0], picture)))+ \ str(counter)+splitext(join(allpics[0], picture))[1] for ref in refs.__iter__(): if picture.find(ref) != -1: longpath, ext = splitext(join(allpics[0], picture)) image_id = longpath[longpath.find(ref) + len(ref):] imageinfos_to_df.append( (picture, okname, ref, urls[refs.index(ref)], image_id, ext)) os.rename(join(allpics[0], picture), join(allpics[0], okname)) counter += 1 imageinfos_to_df = pd.DataFrame(imageinfos_to_df, columns=[ 'picture', 'name', 'website', 'url', 'online_image_id', 'extension' ]) # imageinfos_to_df.columns = imageinfos_to_df.to_excel(join(getcwd(), fname + 'DF.xlsx'))
def square_resize(fname, dimensions=(500, 500)): ''' Resizes square aspect-ratio images to desired dimensions. Doesn't overwrite the images; instead, a prefix corresponding to 'dimensions' parameter is added before each image's and folder's name (i.e: folder 'aquatic_mammal' --> '500_aquatic_mammal'). Parameters ---------- fname: type = str Name of category images directory (ex: 'outdoor_sport') dimensions: type = tuple Tuple (width, length) indicating desired size in pixels type(width) and type(length) = int 'width' & 'length' should be equal Returns ------- None''' imdirect.monkey_patch() #Fixes unexpected image rotation while saving prefix = str(dimensions[0]) + '_' nfpath = join(getcwd(), prefix + fname) os.system("mkdir {}".format(nfpath)) for categs in ls(join(getcwd(), fname)): newcatpath = join(nfpath, prefix + categs) os.system("mkdir {}".format(newcatpath)) for synname in ls(join(getcwd(), fname, categs)): os.system("mkdir {}".format(join(newcatpath, prefix + synname))) for allim in os.walk(join(getcwd(), fname, categs, synname)): for subd in allim[1]: os.system("mkdir {}".format( join(newcatpath, prefix + synname, prefix + subd))) for im_name in tqdm(allim[2]): impath = join(allim[0], im_name) image = Image.open(impath) newpath = impath.replace(fname, prefix + fname, 1) newpath = newpath.replace(categs, prefix + categs, 1) newpath = newpath.replace(synname, prefix + synname, 1) newpath = newpath.replace(bname(dname(newpath)), prefix + bname(dname(newpath)), 1) image.convert("RGB") if image.size[0] > image.size[1]: image = image.crop( ((image.size[0] - image.size[1]) / 2, 0, (image.size[0] + image.size[1]) / 2, image.size[1])) elif image.size[0] < image.size[1]: image = image.crop( (0, (image.size[1] - image.size[0]) / 2, image.size[0], (image.size[1] + image.size[0]) / 2)) im_resized = image.resize(dimensions, Image.ANTIALIAS) im_resized.save(join(dname(newpath), prefix + im_name), 'JPEG', quality=90)
def square_image(dimensions=(500, 500)): ''' Crops all images in a directory to square aspect ratio and resizes them to 'dimensions'. Doesn't overwrite the original image directory. ''' imdirect.monkey_patch() pre = str(dimensions[0])+'_' newdir = join(dname(os.getcwd()), pre+bname(os.getcwd())) os.system("mkdir {}".format(newdir)) for allim in os.walk(os.getcwd()): for picture in tqdm(allim[2]): picpath = join(allim[0], picture) if os.path.isdir(dname(picpath)): subdir = join(newdir, pre+bname(dname(dname(dname(picpath))))) subdir2 = join(subdir, pre+bname(dname(dname(picpath)))) subdir3 = join(subdir2, pre+bname(dname(picpath))) print(newdir+'\n', subdir+'\n', subdir2+'\n', subdir3+'\n') os.system("mkdir {}".format(subdir)) os.system("mkdir {}".format(subdir2)) os.system("mkdir {}".format(subdir3)) image = Image.open(picpath) if image.mode != 'RGB': image.convert("RGB") if image.size[0] > image.size[1]: image = image.crop(((image.size[0]-image.size[1])/2, 0, (image.size[0]+image.size[1])/2, image.size[1])) elif image.size[0] < image.size[1]: image = image.crop((0, (image.size[1]-image.size[0])/2, image.size[0], (image.size[1]+image.size[0])/2)) im_resized = image.resize(dimensions, Image.ANTIALIAS) im_resized.save(join(subdir3, pre+picture), 'JPEG', quality=90)
def proper_index2(folder_name): ''' Names and indexes images according to category. Doesn't affect the image's web reference as would 'naming_indexing' would (useful to inventoriate and unify labels across categories). ''' sources = pd.read_csv(join(dname(os.getcwd()), 'sources.csv')) references = sources['reference'].tolist() folderpath = join(os.getcwd(), folder_name) for root, dirs, files in os.walk(folderpath): counter = 1 for file in files: dir_path = os.path.dirname(join(root, file)) dir_name = os.path.basename(dir_path) if 'bodypart' in dir_name: dir_name = dir_name.replace('bodypart', 'body_part') for ref in references.__iter__(): if file.find(ref) != -1: suffix = file[file.find(ref):] else: suffix = '' if counter <= 9: ok_name = dir_name + '0' + str(counter) + suffix elif counter >= 10: ok_name = dir_name + str(counter) + suffix ok_path = join(root, ok_name) os.rename(join(root, file), os.path.splitext(ok_path)[0]) counter += 1 print(len(os.listdir(dir_path)))
def test_adv_summary(self): fpath = apath( pjoin(dname(__file__), 'resources', 'tripod', 'north', 'adv', 'MCR13N1T05adv1s-cal.nc')) shutil.copy2(fpath, self.tmpf) for of in adv.enhance_summary(self.tmpf): logger.info('Wrote output: {}'.format(of))
def test_adcp_wvs(self): fpath = apath( pjoin(dname(__file__), 'resources', 'tripod', 'north', 'adcp', 'MCR13N1T_wvs.nc')) shutil.copy2(fpath, self.tmpf) for of in adcp.enhance_wvs(self.tmpf): logger.info('Wrote output: {}'.format(of))
def setup_method(self, method): outdir = pjoin(dname(__file__), 'output', type(self).__name__) try: os.makedirs(outdir) except OSError: pass self.tmpf = pjoin(outdir, '{}.nc'.format(method.__name__))
def run(self, fLog): """Basically this is the same code as the info.py request handler it just is not triggered from apache so none of the CGI stuff is available """ # Assume it doesn't work, until we know it does self.nRetCode = 13 fLog.write(" Exec: %s" % self.uCmd) self.proc = subprocess.Popen(self.uCmd, shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) (sStdOut, sStdErr) = self.proc.communicate() fLog.write(sStdErr) if self.proc.returncode != 0: fLog.write(u"Error detected in command %s" % self.uCmd) self.nRetCode = self.proc.returncode return sDir = dname(self.sOutPath) if not os.path.isdir(sDir): os.makedirs(sDir) # Read the output in as a JSON object (may throw and that's okay) dOut = json.loads(sStdOut, encoding="utf-8") if self.sDas2Url != None: dOut['x_links'] = [{ "tag": "das2Stream", "description": "Access to the upstream Das2 data source for this HAPI endpoint", "mime-type": "application/vnd.das2.das2stream", "url": self.sDas2Url }] # Override the description to match the sub source if needed if self.sDescription: dOut['description'] = self.sDescription sJsonOut = json.dumps(dOut, ensure_ascii=False, sort_keys=True, indent=3) fOut = open(self.sOutPath, 'w') fOut.write(sJsonOut) fOut.close() fLog.write("Cache file %s written" % self.sOutPath) self.nRetCode = 0 self.proc = None
def _fileOutWithEx(sFile, tData): fLog = tData[0] sPrefix = tData[1] lOut = tData[2] sRelPath = None sDescription = None if not sFile.lower().endswith('.dsdf'): return True if sFile.lower().endswith('_dirinfo_.dsdf'): return True sDir = "%s/"%dname(sFile).replace(sPrefix, '') # Remove items that are in directories name test from the # output if sDir.find(g_sTestDir) != -1: return True fIn = codecs.open(sFile, 'rb', encoding='utf-8') #fLog.write("INFO: Parsing %s"%sFile) for sLine in fIn: iComment = sLine.find(';') if iComment != -1: sLine = sLine[:iComment] if sLine.find('exampleRange') != -1: sRelPath = sFile.replace(sPrefix, '').replace('.dsdf','') if sDir != "/" and sDir not in lOut: lOut.append(sDir) if sLine.find('description') != -1: lLine = sLine.split('=') if len(lLine) > 1: sDescription = "".join(lLine[1:]) sDescription = sDescription.strip("\"' \r\n\t") continue if sLine.find('rename') != -1: lLine = sLine.split('=') if len(lLine) > 1 and lLine[0].strip().lower() == 'rename': fIn.close() return True if sRelPath: if sDescription: lOut.append( u"%s|%s"%(sRelPath,sDescription ) ) else: lOut.append( sRelPath ) return True
def imcount(): imdict = {} for allimages in os.walk(os.path.relpath('neuromod_image_bank')): for folder in enumerate(allimages[1]): folderdict = { bname(dname(join(allimages[0], folder))): (folder, len(ls(join(allimages[0], folder[1])))) } imdict[folder[0]] = folderdict print(folderdict) print(folder, len(os.listdir(os.path.join(allimages[0], folder))))
def load_companies(): companies = [] for company_file_path in glob(join(dname(__file__), 'companies', '*.csv')): with open(company_file_path) as company_file: reader = csv.reader(company_file) reader.next() # skip the header row for company in reader: companies.append((len(companies) + 1, company[1])) return companies
def test_adv_summary(self): fpath = apath(pjoin( dname(__file__), 'resources', 'tripod', 'north', 'adv', 'MCR13N1T05adv1s-cal.nc' )) shutil.copy2(fpath, self.tmpf) for of in adv.enhance_summary(self.tmpf): logger.info('Wrote output: {}'.format(of))
def test_adcp_wvs(self): fpath = apath(pjoin( dname(__file__), 'resources', 'tripod', 'north', 'adcp', 'MCR13N1T_wvs.nc' )) shutil.copy2(fpath, self.tmpf) for of in adcp.enhance_wvs(self.tmpf): logger.info('Wrote output: {}'.format(of))
def square_image(folderpath, length=500): ''' Alt square_image.py using os.makedirs() insead of os.system Crops all images in a directory to square aspect ratio and resizes them to 'dimensions'. Doesn't overwrite the original image directory. ''' imdirect.monkey_patch() pre = str(length) + '_' impaths = loadimages(folderpath) newpaths = [] for impath in enumerate(impaths): newparts = [] parts = splitall(impath[1])[1:-1] for part in parts: newparts.append(pre + part) newpaths.append( (impath[1], join(*[pre + part for part in splitall(impath[1])]), dname(join(*[pre + part for part in splitall(impath[1])])))) for newpath in tqdm(newpaths): if not os.path.exists(newpath[2]): os.mkdir(newpath[1]) image = Image.open(newpath[1]) # newdir = join(dname(folderpath), pre+bname(folderpath) # os.system("mkdir {}".format(newdir)) # for allim in os.walk(os.getcwd()): # for picture in tqdm(allim[2]): # picpath = join(allim[0], picture) # if os.path.isdir(dname(picpath)): # subdir = join(newdir, pre+bname(dname(dname(dname(picpath))))) # subdir2 = join(subdir, pre+bname(dname(dname(picpath)))) # subdir3 = join(subdir2, pre+bname(dname(picpath))) # print(newdir+'\n', subdir+'\n', subdir2+'\n', subdir3+'\n') # os.makedirs(subdir3, exist_ok=True) # os.system("mkdir {}".format(subdir), exist_ok=True) # os.system("mkdir {}".format(subdir2), exist_ok=True) # os.system("mkdir {}".format(subdir3), exist_ok=True) if image.mode != 'RGB': image.convert("RGB") if image.size[0] > image.size[1]: image = image.crop( ((image.size[0] - image.size[1]) / 2, 0, (image.size[0] + image.size[1]) / 2, image.size[1])) elif image.size[0] < image.size[1]: image = image.crop( (0, (image.size[1] - image.size[0]) / 2, image.size[0], (image.size[1] + image.size[0]) / 2)) im_resized = image.resize((length, length), Image.ANTIALIAS) im_resized.save(newpath[1], 'JPEG', quality=100)
def get_images(datas): ''' Description ----------- DF containing each image_concept\ Returns ------- ImageBank.images ''' images = pd.DataFrame((row[1] for row in datas.iterrows() if pd.isnull(row[1]['subordinates']).all()), columns=list(datas.columns)) images['names'] = [bname(row[1]['path']) for row in images.iterrows()] images['folders'] = [ bname(dname(row[1].path)) for row in images.iterrows() ] images = images.set_index('folders') # images_ind = images.set_index('names', append=True).index return images.sort_index(kind='mergesort')
def getDirUri(U, fLog, dConf, dDirInfo, sCatDir): if "uri" in dDirInfo: sUri = dDirInfo["uri"].strip("\"' /r/n") fLog.write("INFO: Using exlicit URI for directory %s, %s" % (sPath, sUri)) return sUri sRelPath = None _sOrigCatDir = sCatDir while sCatDir != dConf['DSDF_ROOT']: # Go up one if sRelPath == None: sRelPath = bname(sCatDir) else: sRelPath = "%s/%s" % (bname(sCatDir), sRelPath) sCatDir = dname(sCatDir) sCatDsdf = pjoin(sCatDir, '_dirinfo_.dsdf') if os.path.isfile(sCatDsdf): fIn = open(sCatDsdf, "rb") dCatDsdf = U.dsdf.readDsdf(fIn, fLog) fIn.close() if "uri" in dCatDsdf: fLog.write( "INFO: Directory %s URI set relative to directory %s URI" % (_sOrigCatDir, sCatDir)) sUri = dDsdf["uri"].strip("\"' /r/n") return "%s/%s" % (sUri, sRelPath) # Still here huh, okay if "SITE_PATHURI" not in dConf: U.webio.serverError(fLog, "No pathUri setting along the path of _dirinfo_.dsdf files leading "+\ "the path to file %s and fall back value SITE_PATHURI not set in %s"%( pjoin(_sOrigCatDir, '_dirinfo_.dsdf'), dConf['__file__'])) return None fLog.write( "INFO: Directory %s URI set relative to config file SITE_PATHURI: %s" % (_sOrigCatDir, dConf['SITE_PATHURI'])) return "%s/%s" % (dConf['SITE_PATHURI'], sRelPath)
from os.path import dirname as dname from os.path import join from os import listdir as ls import pandas as pd from pandas import DataFrame as df from taskfunctions import flatten from taskfunctions import loadimages from taskfunctions import splitall with open('../docs/index_names.json', 'r') as json_file: read_names = json.load(json_file) datas_json = json.dumps(cib_json) allim = loadimages() tags = flatten([splitall(dname(fpath).split(IMDIR)[1])[1:] for fpath in allim]) freqs = [dict((tag, Counter(bname(dname(tag))).values()) for tag in flatten(tags))] freqs = pd.Series(dict(Counter(flatten(tags)))).sort_index() json_mapper = json.dumps(cib_json) json_mapper json_freqs = freqs.to_json() freq_json = json.dumps(freqs, indent=4) def flatten_json(dictionary): flattened = [] def flat(data, name=''): if type(data) is dict: for d in data: flat(data[d], name + d + ',') elif type(data) is list:
def renamefaces( facepath='/home/francois/cib/images/animate_being/animal/animal_face'): flist = list(dict.fromkeys([dname(item) for item in loadimages(facepath)])) for item in flist: if '_face' not in bname(item): os.rename(item, join(dname(item), bname(item) + '_face'))