def import_db():
i = 0
train_db = picpac.Writer('/data/scratch/lcai/Avito/db/train.db',
picpac.OVERWRITE)
val_db = picpac.Writer('/data/scratch/lcai/Avito/db/val.db',
picpac.OVERWRITE)
for chunk in pd.read_csv(input_file, chunksize=chunksize):
#with open('%s.list' % Set, 'r') as f:
for _, row in chunk.iterrows():
itemid = row.iloc[0]
fig = str(row.iloc[15])
label = row.iloc[17]
error = 0
if os.path.isfile(fig_path + fig + ".jpg"):
file_path = fig_path + fig + ".jpg"
elif os.path.isfile(fig_path + fig):
file_path = fig_path + fig
else:
print("Error!\t", fig, "\t", itemid)
error = 1
if error != 1:
with open(file_path, 'rb') as f2:
buf = f2.read()
if i % 20 == 0:
val_db.append(float(label), buf)
val_array.append(itemid)
#print ("val",i)
else:
train_db.append(float(label), buf)
#print ("train",i)
'''
path = fig_path + fig + '*'
file_path = glob.glob(path)
if len(file_path) != 1:
print ("Error!!")
print (file_path)
else:
with open(glob.glob(path)[0], 'rb') as f2:
buf = f2.read()
if i % 20 == 0:
val_db.append(float(label), buf)
print ("val",i)
else:
train_db.append(float(label), buf)
print ("train",i)
'''
i += 1
pass
pass
del db
pickle.dump(val_array, open('/data/scratch/lcai/Avito/db/val_id.pkl',
'wb'))
pass
def import_db(db_path, img_path):
db = picpac.Writer(db_path, picpac.OVERWRITE)
for image in glob(img_path + "*/*.jpg"):
file_name = image.split("/")[-1]
if file_name in anno.keys():
db.append(anno[file_name], load_file(image), image.encode('ascii'))
pass
def import_db (path, tasks):
with open(path + '.list','w') as f:
db = picpac.Writer(path, picpac.OVERWRITE)
for p, l in tqdm(list(tasks)):
f.write('%s,%d\n' % (p, l))
image = cv2.imread(p, -1)
image = cv2.resize(image, None, fx=0.5, fy=0.5)
image_buffer = cv2.imencode('.jpg', image)[1].tostring()
db.append(float(l), image_buffer)
pass
def import_db(Set):
db = picpac.Writer('%s.db' % Set, picpac.OVERWRITE)
with open('%s.list' % Set, 'r') as f:
for l in f:
path, label = l.strip().split('\t')
with open(path, 'rb') as f2:
buf = f2.read()
db.append(float(label), buf)
pass
pass
del db
pass
def import_db(Cls, Set):
X, Y = load_list(Cls, Set, difficult=False)
db = picpac.Writer('db/%s.%s' % (Cls, Set), picpac.OVERWRITE)
C = 0
print("%s %s loading %d images..." % (Cls, Set, len(Y)))
for path, label in tqdm(list(zip(X, Y))):
with open(path, 'rb') as f:
buf = f.read()
db.append(float(label), buf)
pass
pass
pass
def import_db(split): #db_path, img_path, labelid_path):
db_path = 'scratch/cityscape/%s.db' % split
tasks = glob('data/cityscape/leftImg8bit/%s/*/*.png' % split)
print('importing %s: %d images' % (split, len(tasks)))
db = picpac.Writer(db_path, picpac.OVERWRITE)
for image_path in tqdm(tasks):
# image_path: data/cityscape/leftImg8bit/train/jena/jena_000050_000019_leftImg8bit.png
image = cv2.imread(image_path, -1)
if image is None:
continue
if not image.shape[0] > 0:
continue
with open(image_path, 'rb') as f:
image_buf = f.read()
pass
if split == 'test': # we don't need to load labels for test images
db.append(0, image_buf)
continue
label_path = image_path.replace('leftImg8bit', 'gtFine').replace(
'.png', '_polygons.json')
assert os.path.exists(label_path)
with open(label_path, 'r') as f:
anno = json.loads(f.read())
pass
H = anno['imgHeight']
W = anno['imgWidth']
shapes = []
for obj in anno['objects']:
label_txt = obj['label']
label = LABEL_MAP.get(label_txt, None)
if label is None:
UNKNOWN[label_txt] = 1
else:
points = []
for x, y in obj['polygon']:
points.append({'x': x / W, 'y': y / H})
pass
order = ORDER_MAP[label]
shapes.append({
'type': 'polygon',
'label': label,
'order': order,
'geometry': {
'points': points
}
})
pass
shapes.sort(key=lambda x: x['order'])
label_buffer = json.dumps({'shapes': shapes}).encode('ascii')
db.append(0, image_buf, label_buffer)
pass
def import_db (db_path, list_path):
db = picpac.Writer(db_path, picpac.OVERWRITE)
with open(list_path, 'r') as f:
for l in f:
path = l.strip()
part = path.split('/')[2]
#print(path)
if 'positive' in path:
l = 1
elif 'negative' in path:
l = 0
else:
assert 0
pass
assert part in PARTS
k = PARTS[part]
label = k * 2 + l
db.append(label, load_file('data/' + path), path.encode('ascii'))
pass
pass
def handle(self, *args, **options):
#hours = options['hours'] + 24 * options['days']
#check_and_import(hours, not options['run'], options['check'])
output = options['output'][0]
all = {}
if options['all']:
for image in Image.objects.filter(viewed=True):
all[image] = {"shapes": []}
pass
for anno in Annotation.objects.all():
shapes = all.setdefault(
anno.image, {"shapes": []})["shapes"].append(
fix_transpose(json.loads(anno.anno)["shapes"][0]))
pass
if os.path.exists(output):
os.remove(output)
pp = picpac.Writer(output, picpac.OVERWRITE)
for image, anno in all.items():
with open(image.path, 'rb') as f:
buf = f.read()
pp.append(0, buf, json.dumps(anno).encode('ascii'))
print(anno)
pass
def import_db(db_path, list_path):
db = picpac.Writer(db_path, picpac.OVERWRITE)
tasks = read_list(list_path)
for number in tqdm(tasks):
image_path = os.path.join('data/training/image_2', '%06d.png' % number)
label_path = os.path.join('data/training/label_2', '%06d.txt' % number)
image = cv2.imread(image_path, -1)
label = load_label(label_path)
H, W = image.shape[:2]
shapes = []
for obj in label:
if obj.cat != 'Car':
continue
#print(obj.bbox)
x1, y1, x2, y2 = obj.bbox
x = x1 / W
y = y1 / H
w = (x2 - x1) / W
h = (y2 - y1) / H
shapes.append({
'type': 'rect',
'geometry': {
'x': x,
'y': y,
'width': w,
'height': h
}
})
anno = {'shapes': shapes, 'number': number}
anno_buf = json.dumps(anno).encode('ascii')
#print(anno_buf)
db.append(0, load_file(image_path), anno_buf)
pass
# example.
from __future__ import absolute_import, division, print_function
import os
import cv2
import simplejson as json
import picpac
# create a picpac db
# make sure the file doesn't already exist
try:
os.remove('test.db')
except:
pass
db = picpac.Writer('test.db')
# append images for classification
label = 1
# directly append file content
with open('lenna.jpg', 'rb') as f:
# append label, image
buf = f.read()
db.append(label, buf)
image = cv2.imread('lenna.jpg', cv2.IMREAD_COLOR)
# encode np.array image to string and append
buf = cv2.imencode('.png', image)[1].tostring()
db.append(label, buf)
import simplejson as json
import cv2
import numpy as np
from skimage import measure
import scipy
from utils import *
from lyft import *
json.encoder.FLOAT_REPR = lambda f: ("%.3f" % f)
json.encoder.c_make_encoder = None
BASE = os.path.abspath(os.path.dirname(__file__))
assert len(LABEL_MAP) == 1
db = picpac.Writer('scratch/train.db', picpac.OVERWRITE)
CC = 0
with open(os.path.join(BASE, 'train.list'), 'r') as f:
for line in f:
stem = line.strip()
image_path = os.path.join(BASE, 'data/Train/CameraRGB/%s.png' % stem)
seg_path = os.path.join(BASE, 'data/Train/CameraSeg/%s.png' % stem)
print(seg_path)
assert os.path.exists(image_path)
assert os.path.exists(seg_path)
#image = cv2.imread(image_path)
seg= cv2.imread(seg_path)
#assert image.shape == seg.shape
['m_E_0.db', 'm_E_1.db', 'm_E_3.db', 'm_E_4.db', 'm_E_5.db', 'm_E_6.db'])
]
RX = 0.2 # reduce to 0.2 of original size
for out_path, in_paths in DBS:
print "Creating", out_path
l = 1
try:
os.remove(out_path)
except:
pass
outb = picpac.Writer(out_path)
for in_path in in_paths:
# iterate one database
inb = picpac.Reader(os.path.join('db', in_path))
cc = 0
for _, _, _, fields in inb:
#print pid, label, len(fields)
image_buf, anno_buf = fields
arr = np.fromstring(image_buf, np.uint8)
image = cv2.imdecode(arr, cv2.CV_LOAD_IMAGE_COLOR)
# down size image
image2 = cv2.resize(image, None, fx=RX, fy=RX)
#print image.shape, '=>', image2.shape