def outside_images(self): models = Models() number_of_files = 0 print "Retrieving all online avisos. Please, wait..." array_avisos_online = models.get_all_avisos_online() print '[Ok]' print "Retrieving all histogramed avisos. Please, wait..." array_avisos_with_hist = models.get_all_avisos_with_histogram() print '[Ok]' for aviso_online_id in array_avisos_online: is_on_hist_table = False for aviso_online_with_hist_id in array_avisos_with_hist: if aviso_online_id == aviso_online_with_hist_id: is_on_hist_table = True break if not is_on_hist_table: aviso_json = {"id_aviso":aviso_online_id} models.save_id_aviso_not_histogram_table(aviso_json) if number_of_files%100==0: print number_of_files number_of_files +=1
def create_images_histogram_from_online_ads(self): models = Models() number_of_files = 0 print "Retrieving all online avisos. Please, wait..." self.array_avisos_online = models.get_all_avisos_online() print "[Ok]" for aviso_id in self.array_avisos_online: if number_of_files%100==0: print number_of_files number_of_files +=1 aviso_json = {"id_aviso":aviso_id, "photos":[]} # print os.walk(Constants.LOCAL_DIR_SAVE_PHOTO + complete_folder) try: if len(str(aviso_id))<10: aviso_id = format(int(aviso_id), "010") aviso_id_splitted = re.findall(r'.{1,2}',str(aviso_id),re.DOTALL) complete_folder = "" for folder_name in aviso_id_splitted: complete_folder += folder_name + "/" for root, dirs, files in os.walk(Constants.LOCAL_DIR_SAVE_PHOTO + complete_folder): folder_to_download = "" for folder in dirs: if folder == "100x75": folder_to_download = "100x75" break elif folder == "1200x1200": folder_to_download = "1200x1200" break folder_name = Constants.LOCAL_DIR_SAVE_PHOTO + complete_folder + folder_to_download for file in os.listdir(folder_name): if file.endswith(".jpg"): hist = self.get_histogram(os.path.join(folder_name, file)) hist_json = {"photo_path":folder_name + "/" + file, "histogram":json.dumps(hist.tolist())} aviso_json["photos"].append(hist_json) if len(os.listdir(folder_name))>0: models.add_image_histogram(aviso_json) break except: pass
def create_images_histogram_from_images_backup_iw(self): models = Models() self.number_of_files = 0 print "Retrieving all online avisos. Please, wait..." self.array_avisos_online = models.get_all_avisos_online() print "[Ok]" def step((ext, self), dir_name, files): download_100x75 = True aviso_id = dir_name[dir_name.rfind("Constants.LOCAL_DIR_SAVE_PHOTO")+len(Constants.LOCAL_DIR_SAVE_PHOTO)+2:dir_name.rfind("/")] aviso_id = aviso_id.translate(None, "/") if "100x75" in dir_name or "1200x1200" in dir_name: dir100x75 = dir_name[:dir_name.rfind("/")] + "/100x75" dir1200x1200 = dir_name[:dir_name.rfind("/")] + "/1200x1200" try: aviso_id_int = int(aviso_id) except ValueError: aviso_id_int = 0 if aviso_id_int in self.array_avisos_online: if "1200x1200" in dir_name: if os.path.isdir(dir100x75): download_100x75 = False if download_100x75: aviso_json = {"id_aviso":aviso_id, "photos":[]} for file_name in files: if file_name.lower().endswith(ext): try: #generating the histogram and adding it to the json to be added to mongo hist = self.get_histogram(os.path.join(dir_name, file_name)) hist_json = {"photo_path":dir_name + "/" + file_name, "histogram":json.dumps(hist.tolist())} aviso_json["photos"].append(hist_json) except: pass models.add_image_histogram(aviso_json) if self.number_of_files%100==0: print self.number_of_files self.number_of_files +=1 else: if self.number_of_files%100==0: print self.number_of_files self.number_of_files +=1 print aviso_id