def download_image(prod, feature_name, category_id, max_images):
downloaded_images = 0
directory = os.path.join(category_id, feature_name)
try:
downloaded_images = len([name for name in os.listdir(directory) if os.path.isfile(name)])
except:
pass
if downloaded_images < max_images:
xlarge_url = prod['image']['sizes']['XLarge']['url']
img_arr = Utils.get_cv2_img_array(xlarge_url)
if img_arr is None:
logging.warning("Could not download image at url: {0}".format(xlarge_url))
return
relevance = background_removal.image_is_relevant(img_arr)
if relevance.is_relevant:
logging.info("Image is relevant...")
filename = "{0}_{1}.jpg".format(feature_name, prod["id"])
filepath = os.path.join(directory, filename)
Utils.ensure_dir(directory)
logging.info("Attempting to save to {0}...".format(filepath))
success = cv2.imwrite(filepath, img_arr)
if not success:
logging.info("!!!!!COULD NOT SAVE IMAGE!!!!!")
return 0
# downloaded_images += 1
logging.info("Saved... Downloaded approx. {0} images in this category/feature combination"
.format(downloaded_images))
return 1
else:
# TODO: Count number of irrelevant images (for statistics)
return 0
def get_pd_results_and_save(url=None, filename=None):
if url is not None:
print('getting pd results for ' + url)
image = Utils.get_cv2_img_array(url)
elif filename is not None:
print('getting pd results for ' + filename)
image = cv2.imread(filename)
if image is None:
print('image came back none')
return None
try:
seg_res = pd_falcon_client.pd(image)
except:
print('exception in pd_falcon_client.pd ' + str(sys.exc_info()[0]))
return
# print('segres:'+str(seg_res))
if filename is not None:
imgfilename = os.path.basename(
filename
) #use the actual on-disk filename if thats what we started with, otherwise use the name generated by pd
#this will et saved to /home/jeremy/pd_output/filename
else:
try:
imgfilename = seg_res['filename'] + '.jpg'
except:
print('some error on imgfile name')
imgfilename = str(int(time.time())) + '.jpg'
print('filename ' + imgfilename)
if not 'mask' in seg_res:
print('pd did not return mask')
print seg_res
return
mask = seg_res['mask']
label_dict = seg_res['label_dict']
print('labels:' + str(label_dict))
conversion_utils.count_values(mask)
pose = seg_res['pose']
mask_np = np.array(mask, dtype=np.uint8)
print('masksize ' + str(mask_np.shape))
pose_np = np.array(pose, dtype=np.uint8)
print('posesize ' + str(pose_np.shape))
# print('returned url '+seg_res['url'])
converted_mask = convert_and_save_results(mask_np, label_dict, pose_np,
imgfilename, image, url)
dir = constants.pd_output_savedir
Utils.ensure_dir(dir)
full_name = os.path.join(dir, imgfilename)
# full_name = filename
bmp_name = full_name.strip('.jpg') + (
'.bmp') #this bmp should get generated by convert_and_save_results
# imutils.show_mask_with_labels(bmp_name,constants.fashionista_categories_augmented_zero_based,original_image = full_name,save_images=False)
return mask, label_dict, pose_np, converted_mask
def get_pd_results(url=None, filename=None, img_arr=None):
if url is not None:
print('getting pd results for ' + url)
image = Utils.get_cv2_img_array(url)
elif filename is not None:
print('getting pd results for ' + filename)
image = cv2.imread(filename)
elif img_arr is not None:
print('getting pd results for img_arr ')
image = img_arr
if image is None:
print('image came back none')
return None
try:
seg_res = pd_falcon_client.pd(image)
except:
print('exception in pd_falcon_client.pd ' + str(sys.exc_info()[0]))
return
# print('segres:'+str(seg_res))
if not 'mask' in seg_res:
print('pd did not return mask')
print seg_res
return
mask = seg_res['mask']
label_dict = seg_res['label_dict']
print('labels:' + str(label_dict))
# conversion_utils.count_values(mask)
pose = seg_res['pose']
mask_np = np.array(mask, dtype=np.uint8)
print('masksize ' + str(mask_np.shape))
pose_np = np.array(pose, dtype=np.uint8)
print('posesize ' + str(pose_np.shape))
return [mask, label_dict, pose_np]
def save_to_www(results):
# print('attempting to save results {}'.format(results))
try: #save locally in case i get chance to setup local server
filename = 'pipeline_output.html'
wwwpath = '/home/docker-user/appengine_api/output' #this issnt shared in docker... prob / wont be shared either
wwwpath = '/data/www'
wwwname = os.path.join(wwwpath,os.path.basename(filename))
print('WWW - saving json to '+wwwname)
Utils.ensure_file(wwwname)
with open(wwwname,'w') as fp:
# json.dump(results,fp,indent=4) #object is not json serializable for whatever reason
fp.write(str(results))
fp.close()
print('WWW - writing')
except:
# print(sys.exc_info()[0])
print(sys.exc_info())
try: #save to server already running
destname = '/data/www/'+filename
print('copying to 13.69.27.202:'+destname)
scpcmd = 'scp '+wwwname + ' [email protected]:'+destname
subprocess.call(scpcmd,shell=True)
except:
print(sys.exc_info())
#attempt direct ftp since local save doesnt work and cant scp without local file
try:
import paramiko
connection = paramiko.SSHClient()
connection.set_missing_host_key_policy(paramiko.AutoAddPolicy())
connection.connect(13.69.27.202, username='******')
ftp = connection.open_sftp()
f = ftp.open(destname, 'w+')
f.write(results)
f.close()
ftp.close()
connection.close()
except:
print(sys.exc_info())
def create_training_set_with_grabcut(collection):
coll = db[collection]
i = 1
total = db.training_images.count()
start = time.time()
for doc in coll.find():
if not i % 10:
print "did {0}/{1} documents in {2} seconds".format(
i, total,
time.time() - start)
print "average time for image = {0}".format(
(time.time() - start) / i)
url = doc['url'].split('/')[-1]
img_url = 'https://tg-training.storage.googleapis.com/tamara_berg_street2shop_dataset/images/' + url
image = Utils.get_cv2_img_array(img_url)
if image is None:
print "{0} is a bad image".format(img_url)
continue
i += 1
small_image, ratio = background_removal.standard_resize(image, 600)
# skin_mask = kassper.skin_detection_with_grabcut(small_image, small_image, skin_or_clothes='skin')
# mask = np.where(skin_mask == 255, 35, 0).astype(np.uint8)
mask = np.zeros(small_image.shape[:2], dtype=np.uint8)
for item in doc['items']:
try:
bb = [int(c / ratio) for c in item['bb']]
item_bb = get_margined_bb(small_image, bb, 0)
if item['category'] not in cats:
continue
category_num = cats.index(item['category'])
item_mask = background_removal.simple_mask_grabcut(
small_image, rect=item_bb)
except:
continue
mask = np.where(item_mask == 255, category_num, mask)
filename = 'tamara_berg_street2shop_dataset/masks/' + url[:-4] + '.txt'
save_to_storage(bucket, mask, filename)
coll.update_one({'_id': doc['_id']}, {
'$set': {
'mask_url':
'https://tg-training.storage.googleapis.com/' + filename
}
})
print "Done masking! took {0} seconds".format(time.time() - start)
def save_img_at_url(url,savename=None):
# download the image, convert it to a NumPy array, and then save
# it into OpenCV format using last part of url (will overwrite imgs of same name at different url)
if not savename:
savename = url.split('?')[0]
img_arr = Utils.get_cv2_img_array(url)
print('name:'+savename)
cv2.imwrite(savename,img_arr)
return img_arr
if url.count('jpg') > 1:
print('np jpg here captain')
return None
resp = requests.get(url)
print resp
# resp = urllib.urlopen(url)
image = np.asarray(bytearray(resp.read()), dtype="uint8")
if image.size == 0:
return None
new_image = cv2.imdecode(image, cv2.IMREAD_COLOR)
