def create_images_histogram_collection(self):
models = Models()
number_of_files = 0
for dir_name, dir_names, file_names in os.walk(Constants.LOCAL_DIR_SAVE_PHOTO):
#walking inside all ads
for subdir_name in dir_names:
aviso_json = {"id_aviso":subdir_name, "photos":[]}
for dir_name_, dir_names_, file_names_ in os.walk(os.path.join(dir_name, subdir_name)):
#walking inside all photos of the specific ad
for file_name_ in file_names_:
try:
#generating the histogram and adding it to the json to be added to mongo
hist = self.get_histogram(os.path.join(dir_name, subdir_name, file_name_))
hist_json = {"photo_path":subdir_name + "/" + file_name_, "histogram":json.dumps(hist.tolist())}
aviso_json["photos"].append(hist_json)
except:
pass
if number_of_files%1000==0:
print number_of_files
number_of_files +=1
#calling model to add item to mongo
models.add_image_histogram(aviso_json)
print "[OK] Created histograms for " + str(number_of_files) + " photos."
def create_images_histogram_from_online_ads(self):
models = Models()
number_of_files = 0
print "Retrieving all online avisos. Please, wait..."
self.array_avisos_online = models.get_all_avisos_online()
print "[Ok]"
for aviso_id in self.array_avisos_online:
if number_of_files%100==0:
print number_of_files
number_of_files +=1
aviso_json = {"id_aviso":aviso_id, "photos":[]}
# print os.walk(Constants.LOCAL_DIR_SAVE_PHOTO + complete_folder)
try:
if len(str(aviso_id))<10:
aviso_id = format(int(aviso_id), "010")
aviso_id_splitted = re.findall(r'.{1,2}',str(aviso_id),re.DOTALL)
complete_folder = ""
for folder_name in aviso_id_splitted:
complete_folder += folder_name + "/"
for root, dirs, files in os.walk(Constants.LOCAL_DIR_SAVE_PHOTO + complete_folder):
folder_to_download = ""
for folder in dirs:
if folder == "100x75":
folder_to_download = "100x75"
break
elif folder == "1200x1200":
folder_to_download = "1200x1200"
break
folder_name = Constants.LOCAL_DIR_SAVE_PHOTO + complete_folder + folder_to_download
for file in os.listdir(folder_name):
if file.endswith(".jpg"):
hist = self.get_histogram(os.path.join(folder_name, file))
hist_json = {"photo_path":folder_name + "/" + file, "histogram":json.dumps(hist.tolist())}
aviso_json["photos"].append(hist_json)
if len(os.listdir(folder_name))>0:
models.add_image_histogram(aviso_json)
break
except:
pass
def create_images_histogram_from_images_backup_iw(self):
models = Models()
self.number_of_files = 0
print "Retrieving all online avisos. Please, wait..."
self.array_avisos_online = models.get_all_avisos_online()
print "[Ok]"
def step((ext, self), dir_name, files):
download_100x75 = True
aviso_id = dir_name[dir_name.rfind("Constants.LOCAL_DIR_SAVE_PHOTO")+len(Constants.LOCAL_DIR_SAVE_PHOTO)+2:dir_name.rfind("/")]
aviso_id = aviso_id.translate(None, "/")
if "100x75" in dir_name or "1200x1200" in dir_name:
dir100x75 = dir_name[:dir_name.rfind("/")] + "/100x75"
dir1200x1200 = dir_name[:dir_name.rfind("/")] + "/1200x1200"
try:
aviso_id_int = int(aviso_id)
except ValueError:
aviso_id_int = 0
if aviso_id_int in self.array_avisos_online:
if "1200x1200" in dir_name:
if os.path.isdir(dir100x75):
download_100x75 = False
if download_100x75:
aviso_json = {"id_aviso":aviso_id, "photos":[]}
for file_name in files:
if file_name.lower().endswith(ext):
try:
#generating the histogram and adding it to the json to be added to mongo
hist = self.get_histogram(os.path.join(dir_name, file_name))
hist_json = {"photo_path":dir_name + "/" + file_name, "histogram":json.dumps(hist.tolist())}
aviso_json["photos"].append(hist_json)
except:
pass
models.add_image_histogram(aviso_json)
if self.number_of_files%100==0:
print self.number_of_files
self.number_of_files +=1
else:
if self.number_of_files%100==0:
print self.number_of_files
self.number_of_files +=1
print aviso_id
