def test_datetime(self):
obj = datetime(2013, 3, 27, 23, 5)
mobj = msgpack.loads(msgpack.dumps(obj))
jobj = json.loads(json.dumps(obj))
assert obj == mobj
assert obj == jobj
eastern = pytz.timezone('US/Eastern')
obj = datetime(2013, 3, 27, 23, 5, tzinfo=eastern)
mobj = msgpack.loads(msgpack.dumps(obj))
jobj = json.loads(json.dumps(obj))
assert obj == mobj
assert obj == jobj
def test_date(self):
obj = date(2013, 3, 27)
mobj = msgpack.loads(msgpack.dumps(obj))
jobj = json.loads(json.dumps(obj))
assert obj == mobj
assert obj == jobj
def test_float(self):
obj = 3.14159
mobj = msgpack.loads(msgpack.dumps(obj))
jobj = json.loads(json.dumps(obj))
assert obj == mobj
assert isinstance(mobj, float)
assert obj == jobj
assert isinstance(jobj, float)
def test_dict(self):
obj = {'a': 1, 'b': 2, 'c': 3}
mobj = msgpack.loads(msgpack.dumps(obj))
jobj = json.loads(json.dumps(obj))
assert obj == mobj
assert isinstance(mobj, dict)
assert obj == jobj
assert isinstance(jobj, dict)
def test_set(self):
obj = set([1, 2, 3])
mobj = msgpack.loads(msgpack.dumps(obj))
jobj = json.loads(json.dumps(obj))
assert obj == mobj
assert isinstance(mobj, set)
assert obj == jobj
assert isinstance(jobj, set)
def test_list(self):
obj = [1, 2, 3]
mobj = msgpack.loads(msgpack.dumps(obj))
jobj = json.loads(json.dumps(obj))
assert obj == mobj
assert isinstance(mobj, list)
assert obj == jobj
assert isinstance(jobj, list)
def test_str(self):
obj = "abc"
mobj = msgpack.loads(msgpack.dumps(obj))
jobj = json.loads(json.dumps(obj))
assert obj == mobj
assert isinstance(mobj, (str, unicode))
assert obj == jobj
assert isinstance(jobj, (str, unicode))
def test_int(self):
obj = 1
mobj = msgpack.loads(msgpack.dumps(obj))
jobj = json.loads(json.dumps(obj))
assert obj == mobj
assert isinstance(mobj, int)
assert obj == jobj
assert isinstance(jobj, int)
def test_complex(self):
obj = complex(123)
mobj = msgpack.loads(msgpack.dumps(obj))
jobj = json.loads(json.dumps(obj))
assert obj == mobj
assert isinstance(mobj, complex)
assert obj == jobj
assert isinstance(jobj, complex)
def test_dict_int_keys(self):
obj = {1: 1, 2: 2, 3: 3}
mobj = msgpack.loads(msgpack.dumps(obj))
jobj = json.loads(json.dumps(obj))
assert obj == mobj
assert isinstance(mobj, dict)
# we cannot test this as JSON will convert int keys to strings
#assert obj == jobj
assert isinstance(jobj, dict)
def test_ndarray(self):
obj = np.array([1, 2, 3], dtype=np.float32)
mobj = msgpack.loads(msgpack.dumps(obj))
jobj = json.loads(json.dumps(obj))
assert obj.dtype == np.float32
assert (obj == mobj).all()
assert obj.dtype == mobj.dtype
assert isinstance(mobj, np.ndarray)
assert (obj == jobj).all()
assert obj.dtype == jobj.dtype
assert isinstance(jobj, np.ndarray)
def test_jaweson_migration(self):
class A(jaweson.Serialisable, Migrant):
def __init__(self):
self.a = 1
a = A()
assert a.a is 1
assert not hasattr(a, "b")
a = json.loads(json.dumps(a))
assert a.a is 1
assert not hasattr(a, "b")
a = a.migrate(a)
assert a.a is 1
assert hasattr(a, "b")
assert a.b is 2
a = json.loads(json.dumps(a))
assert a.a is 1
assert hasattr(a, "b")
assert a.b is 2
def test_serialisable_constructor(self):
class SerialisableConstructorObject(jaweson.Serialisable):
def __init__(self, a):
self.a = a
obj = SerialisableConstructorObject(2)
mobj = msgpack.loads(msgpack.dumps(obj))
jobj = json.loads(json.dumps(obj))
assert obj.a == 2
assert obj.a == mobj.a
assert obj.a == jobj.a
def test_serialisable(self):
class SerialisableObject(jaweson.Serialisable):
def __init__(self):
self.a = 1
obj = SerialisableObject()
mobj = msgpack.loads(msgpack.dumps(obj))
jobj = json.loads(json.dumps(obj))
assert obj.a == 1
assert obj.a == mobj.a
assert obj.a == jobj.a
def test_npgeneric(self):
obj = np.float32(1)
mobj = msgpack.loads(msgpack.dumps(obj))
jobj = json.loads(json.dumps(obj))
assert obj.dtype == np.float32
assert obj == mobj
assert obj.dtype == mobj.dtype
assert isinstance(mobj, np.generic)
assert obj == jobj
assert obj.dtype == jobj.dtype
assert isinstance(jobj, np.generic)
def init():
#print sys.argv[1]
inputFromPhp = json.loads(sys.argv[1])
file_path = inputFromPhp['image_file_path']
operation = inputFromPhp['operation']
data = inputFromPhp['data']
sender = RpcSender()
im = array(Image.open(file_path))
msg = {"image_path" : file_path, "image" : im, "operation" : operation, "model_parameters" : data}
json_msg = json.dumps(msg)
response = sender.call(json_msg)
print response
def on_post(self, req, resp):
print "Reached hydra on_post"
gpu = req.get_param('gpu')
ret = {"success": False}
#
try:
# data = msgpack.loads(req.stream.read())
data = req.stream.read()
# print('data coming into hydra:'+str(data))
print('hydra falcon')
dict = json.loads(data)
img = dict.get("image")
# img = data['name']
# img = data.split('"')[1]
# img = data
if isinstance(img, basestring):
print('url coming to hydra falcon:' + str(img))
else:
print('img arr into hydra falcon size:' + str(img.shape))
# img_arr=Utils.get_cv2_img_array(img)
# frcnn_output = self.get_fcrnn_output(self,img)
# hydra_output = multilabel_from_hydra_hls.get_hydra_output(img,detection_threshold=0.9)
hydra_output = multilabel_from_hydra_tg.get_hydra_output(img)
if "sweater_binary_h_iter_50000" in hydra_output:
del hydra_output["sweater_binary_h_iter_50000"]
if "sweatshirt_binary_h_iter_14000" in hydra_output:
del hydra_output["sweatshirt_binary_h_iter_14000"]
if "backpack_hydra_iter_2000" in hydra_output:
del hydra_output["backpack_hydra_iter_2000"]
del hydra_output[
"url"] #dont need this , its an array anyway lately
ret["output"] = hydra_output
if ret["output"] is not None:
ret["success"] = True
else:
ret["error"] = "No output from mlb"
# self.write_log(url,output)
except Exception as e:
traceback.print_exc()
ret["error"] = traceback.format_exc()
# resp.data = msgpack.dumps(ret)
# resp.content_type = 'application/x-msgpack'
resp.data = json.dumps(ret) #maybe dump instead of dumps?
resp.content_type = 'application/json'
resp.status = falcon.HTTP_200
def test_class_variable(self):
class ClassVariableObject(jaweson.Serialisable):
__classname = 'TestImHere'
a = 1
obj = ClassVariableObject()
mobj = msgpack.loads(msgpack.dumps(obj))
jobj = json.loads(json.dumps(obj))
assert obj.a == 1
assert obj.a == mobj.a
assert obj.a == jobj.a
assert '_Serialisable__a' not in json.dumps(obj)
def test_classname_inheritance(self):
class ClassA(json.Serialisable):
__classname = 'ThisClass'
def __init__(self):
self.a = 1
class ClassB(ClassA):
def __init__(self):
self.a = 2
a = ClassA()
b = ClassB()
assert a.a is 1
assert b.a is 2
ja = json.dumps(a)
jb = json.dumps(b)
assert 'ThisClass' in ja
assert 'ClassB' in jb
a2 = json.loads(ja)
b2 = json.loads(jb)
assert a2.a is 1
assert b2.a is 2
def get_hydra_output(self, subimage):
'''
get hydra details on an image
:param subimage: np array , e..g a crop of the original which fcrnn has found
:return:
'''
data = json.dumps({"image": subimage})
print('defense falcon is attempting to get response from hydra at ' + str(HYDRA_CLASSIFIER_ADDRESS))
try:
resp = requests.post(HYDRA_CLASSIFIER_ADDRESS, data=data)
dict = json.loads(resp.content)
return dict['output']
except:
print('couldnt get hydra output')
return None
def test_modified_class_variable(self):
class ClassVariableOverrideObject(jaweson.Serialisable):
a = 1
def __init__(self):
self.a = 2
obj = ClassVariableOverrideObject()
mobj = msgpack.loads(msgpack.dumps(obj))
jobj = json.loads(json.dumps(obj))
assert obj.a == 2
assert obj.a == mobj.a
assert obj.a == jobj.a
assert '_Serialisable__a' not in json.dumps(obj)
assert 'a' in json.dumps(obj)
def test_dodgy_constructor(self):
class DodgyConstructor(jaweson.Serialisable):
def __init__(self, a, b):
# flip the order
self.a = b
self.b = a
obj = DodgyConstructor(1, 2)
mobj = msgpack.loads(msgpack.dumps(obj))
jobj = json.loads(json.dumps(obj))
assert obj.a == 2
assert obj.b == 1
assert obj.a == mobj.a
assert obj.b == mobj.b
assert obj.a == jobj.a
assert obj.b == jobj.b
def test_hierarchy(self):
class Node(jaweson.Serialisable):
def __init__(self, name, child=None):
self.name = name
self.child = child
obj = Node(1, Node(2, Node(3)))
mobj = msgpack.loads(msgpack.dumps(obj))
jobj = json.loads(json.dumps(obj))
assert obj.name == 1
assert obj.child.name == 2
assert obj.child.child.name == 3
assert isinstance(obj, Node)
assert obj.name == mobj.name
assert obj.child.name == mobj.child.name
assert obj.child.child.name == mobj.child.child.name
assert obj.name == jobj.name
assert obj.child.name == jobj.child.name
assert obj.child.child.name == jobj.child.child.name
def test_custom_serialiser(self):
class CustomSerialiserObject(jaweson.Serialisable):
@classmethod
def to_json(cls, obj):
return {'c': obj.a, 'd': obj.b}
@classmethod
def from_json(cls, obj):
return cls(int(obj['c']), int(obj['d']))
def __init__(self, a, b):
self.a = a
self.b = b
obj = CustomSerialiserObject(1, 2)
mobj = msgpack.loads(msgpack.dumps(obj))
jobj = json.loads(json.dumps(obj))
assert obj.a == 1
assert obj.b == 2
assert obj.a == mobj.a
assert obj.b == mobj.b
assert obj.a == jobj.a
assert obj.b == jobj.b
def init():
#print sys.argv[1]
try:
inputFromPhp = json.loads(sys.argv[1])
except Exception, e:
print str(e)
def on_post(self, req, res):
#untested
print('\nStarting combine_gunicorn tf hls (got a post request)')
start_time = time.time()
tmpfile = '/data/jeremy/image_dbs/variant/viettel_demo/'
N = 10
randstring = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(N))
tmpfile = os.path.join(tmpfile, randstring + '.jpg')
sent_timestamp = 0
# serializer = msgpack
# res.content_type = "application/x-msgpack"
try:
json_data = json.loads(req.stream.read().decode('utf8'))
if 'sent_timestamp' in json_data:
sent_timestamp = float(json_data['sent_timestamp'])
print('sent timestamp {}'.format(sent_timestamp))
else:
sent_timestamp = 0 #
xfer_time = time.time() - sent_timestamp
print('xfer time:{}'.format(xfer_time))
base64encoded_image = json_data['image_data']
data = base64.b64decode(base64encoded_image)
# print('data type {}'.format(type(data)))
with open(tmpfile, "wb") as fh:
fh.write(data)
print('wrote file to {}, elapsed time for xfer {}'.format(
tmpfile,
time.time() - start_time))
decode_time = time.time() - start_time
try:
print('db2')
# imgpath = '/data/jeremy/tensorflow/tmp.jpg'
# tmpfile = cv2.imwrite(imgpath,img_arr)
detected = tf_detect.analyze_image(tmpfile, thresh=0.2)
except:
raise falcon.HTTPBadRequest(
"Something went wrong in get section 3:(",
traceback.format_exc())
try:
print('db4')
res.data = json.dumps(detected)
res.status = falcon.HTTP_200
except:
raise falcon.HTTPBadRequest(
"Something went wrong in get section 4:(",
traceback.format_exc())
try:
self.write_log('id', detected)
except:
raise falcon.HTTPBadRequest(
"Something went wrong in get section 5 (wrte log):(",
traceback.format_exc())
# stream = req.stream.read()
# print('stream {}'.format(stream))
# data = serializer.loads(stream)
# print('data:{}'.format(data))
# img_arr = data.get("image")
# print('img arr shape {}'.format(img_arr.shape))
# # detected = self.detect_yolo_pyyolo(img_arr)
# cv2.imwrite(tmpfile,img_arr)
# detected = self.tracker.next_frame(tmpfile)
# resp.data = serializer.dumps({"data": detected})
# resp.status = falcon.HTTP_200
except:
raise falcon.HTTPBadRequest("Something went wrong in post :(",
traceback.format_exc())
res.status = falcon.HTTP_203
res.body = json.dumps({
'status': 1,
'message': 'success',
'data': json.dumps(detected)
})
def on_request(self, ch, method, props, body):
'''
This function serves three api calls 1) identify_objects 2) save_image 3) learn_features
'''
image = json.loads(body)
#gets the api call
if "operation" in image:
operation = image["operation"]
ROOT = dirname(abspath(__file__))
image_store_dir = join(ROOT, '../examples/images')
caffe_input_file = join(ROOT, '../_temp/det_input.txt')
if "image_path" in image:
#retrive image_name from rabbitmq and generate string(path) to save the image
image_name = image["image_path"]
image_file_path = image_store_dir + '/' + image_name.split('/')[-1]+'.jpg'
if "image" in image:
#retrive image and store in temporary location
im = image["image"]
[height, width, depth] = im.shape
img = Image.fromarray(im)
img.save(image_file_path)
#dummy response
#response = deep_learning("I")
#Using RCNN detector
#ROOT = dirname(abspath(__file__))
#image_store_dir = join(ROOT, '../examples/images')
#caffe_input_file = join(ROOT, '../_temp/det_input.txt')
#cmd1 = 'echo ' + image_file_path + ' > ' + caffe_input_file
#system(cmd1)
#caffe_output_file = join(ROOT, '../_temp/_output.h5')
#response = detect(caffe_input_file , caffe_output_file)
#Using Faster RCNN Detector
if operation == "identify_objects":
response = detect_objects(image_file_path)
print response
elif operation == "save":
data = image["data"]
image_id = data["image_id"]
labels = data["labels"]
image_shape = [height, width, depth]
save_path = "/var/services/homes/kchakka/py-faster-rcnn/VOCdevkit/VOC2007/Annotations"
response = create_xml(image_id, labels, image_shape, save_path)
fast_rcnn_imagedb = "/var/services/homes/kchakka/py-faster-rcnn/VOCdevkit/VOC2007/JPEGImages/" + response+".jpg"
img.save(fast_rcnn_imagedb)
rcnn_image_file_path = "/var/services/homes/kchakka/py-faster-rcnn/VOCdevkit/VOC2007/ImageSets/Main/train.txt"
cmd1 = 'echo ' + response + '>>' + rcnn_image_file_path
system(cmd1)
print response
elif operation == "learn_features":
#cmd = "cd /var/services/homes/kchakka/py-faster-rcnn"
#system(cmd)
print "Operation is learn_features"
net_name = "VGG16"
weights = "/var/services/homes/kchakka/py-faster-rcnn/data/faster_rcnn_models/VGG16_faster_rcnn_final.caffemodel"
imdb = "voc_2007_train"
train_script = "/var/services/homes/kchakka/py-faster-rcnn/tools/train_faster_rcnn_alt_opt.py"
cmd1 = "python " + train_script + " --net_name=" + net_name + " --weights=" + weights + " --imdb=" + imdb
system(cmd1)
print "Training the network"
#cmd = "cd /var/services/homes/kchakka/caffe/argus"
#system(cmd)
response = "Training done"
ch.basic_publish(exchange='', routing_key=props.reply_to, properties=pika.BasicProperties(correlation_id = props.correlation_id), body=str(response))
ch.basic_ack(delivery_tag=method.delivery_tag)
def on_post(self, req, resp):
print "Reached on_post"
ret = {"success": False}
try:
## data = msgpack.loads(req.stream.read())
# img = data.get("image")
print('in try of onpost')
data = json.loads(req.stream.read())
print('data recd:'+str(data))
filename = data["filename"]
img_string = data["img_string"]
try:
taggername = data["strUser"]
except:
taggername = "unknown"
print('strUser not sent from webtool')
imagedata = img_string.split(',')[-1].decode('base64')
print('filename {} tagger {}'.format(filename,taggername))
#save new mask under old name and send to extremeli
# with open(filename, 'wb') as f:
# f.write(imagedata)
# f.close()
# command_string = 'scp '+filename+' [email protected]:/var/www/js-segment-annotator/data/pd_output'
# subprocess.call(command_string, shell=True)
#save new mask with 'finished_mask' filename and send to extremeli
outfilename = filename.replace('.png','_finished_mask.png').replace('.bmp','_finished_mask.bmp').replace('\n','')
# outfilename = outfilename.replace('_finished_mask','_finished_mask_'+taggername)
outfile2 = outfilename.replace('_finished_mask.png','_finished_mask_'+taggername+'.txt')
print('writing r-only img to '+outfilename)
with open(outfilename, 'wb') as f:
f.write(imagedata)
f.close()
with open(outfile2, 'w') as f:
f.write(taggername)
f.close()
command_string = 'scp '+outfilename+' [email protected]:/var/www/js-segment-annotator/data/pd_output'
subprocess.call(command_string, shell=True)
command_string = 'scp '+outfile2+' [email protected]:/var/www/js-segment-annotator/data/pd_output'
subprocess.call(command_string, shell=True)
#convert from 'webtool' format (index in red channel of 3chan img) to 'regular' format - 1 chan img that
#cv2 reads in as 3chan with identical info in all chans
#then send to extremeli. remove 'webtool' from name, as now its regular format
img_arr = cv2.imread(outfilename)
h,w = img_arr.shape[0:2]
data = img_arr
if len(img_arr.shape) == 3:
data = img_arr[:,:,2]
outfilename = filename.replace('.png','_finished_mask.png').replace('_webtool','').replace('.bmp','_finished_mask.bmp')
print('writing rgb img to '+outfilename)
cv2.imwrite(outfilename,data)
command_string = 'scp '+outfilename+' [email protected]:/var/www/js-segment-annotator/data/pd_output'
subprocess.call(command_string, shell=True)
ret["output"] = imagedata
if ret["output"] is not None:
ret["success"] = True
else:
ret["error"] = "No output from onpost"
except Exception as e:
traceback.print_exc()
ret["error"] = traceback.format_exc()
resp.data = msgpack.dumps(ret)
resp.content_type = 'application/x-msgpack'
resp.status = falcon.HTTP_200
def on_post(self, req, res):
#untested
print('\nStarting combine_gunicorn (got a post request)')
start_time=time.time()
tmpfile = '/data/jeremy/image_dbs/'
N=10
randstring = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(N))
tmpfile = os.path.join(tmpfile,randstring+'.jpg')
sent_timestamp=0
# serializer = msgpack
# res.content_type = "application/x-msgpack"
try:
json_data = json.loads(req.stream.read().decode('utf8'))
if 'sent_timestamp' in json_data:
sent_timestamp = float(json_data['sent_timestamp'])
print('sent timestamp {}'.format(sent_timestamp))
else:
sent_timestamp=0 #
if 'vietel_format' in json_data:
vietel_format = json_data['vietel_formt']
else:
vietel_format = False
xfer_time = time.time()-sent_timestamp
print('xfer time:{}'.format(xfer_time))
base64encoded_image = json_data['image_data']
data = base64.b64decode(base64encoded_image)
# print('data type {}'.format(type(data)))
with open(tmpfile, "wb") as fh:
fh.write(data)
print('wrote file to {}, elapsed time for xfer {}'.format(tmpfile,time.time()-start_time))
decode_time = time.time()-start_time
try:
print('db2')
# imgpath = '/data/jeremy/tensorflow/tmp.jpg'
# tmpfile = cv2.imwrite(imgpath,img_arr)
img_arr=cv2.imread(tmpfile)
detected = self.detect_yolo_pyyolo(img_arr)
##desired
#example results {"emotion": -0.027538346377856619, "n_currently_in_frame": 2, "transfer_time": 0.05023598670959473, "detect time": 1.9451310634613037,
# "detections": [{"bbox_xywh": [32, 68, 172, 306], "confidence": 0.997, "object": "person 0", "id": 0}, {"bbox_xywh": [254, 36, 205, 294], "confidence": 0.981, "object": "person 1", "id": 1}], "n_total": 2, "line_length": 0}
#actual
# {"data": [{"confidence": 0.6014, "object": "car", "bbox": [305, 178, 344, 195], "details": {"color": "gray"}}]}
#"detections": [{"bbox_x
if vietel_format:
detected_vietel_format={}
detected_vietel_format['emotion']=0
print('det data {}'.format(detected))
detected_vietel_format['detections']=detected
for det in detected_vietel_format['detections']:
det['bbox_xywh']=det['bbox']
del det['bbox']
detected = detected_vietel_format
# detected = tf_detect.analyze_image(tmpfile,thresh=0.2)
except:
raise falcon.HTTPBadRequest("Something went wrong in get section 3:(", traceback.format_exc())
try:
print('db4')
res.data = json.dumps(detected)
res.status = falcon.HTTP_200
except:
raise falcon.HTTPBadRequest("Something went wrong in get section 4:(", traceback.format_exc())
try:
self.write_log('id',detected)
except:
raise falcon.HTTPBadRequest("Something went wrong in get section 5 (wrte log):(", traceback.format_exc())
# stream = req.stream.read()
# print('stream {}'.format(stream))
# data = serializer.loads(stream)
# print('data:{}'.format(data))
# img_arr = data.get("image")
# print('img arr shape {}'.format(img_arr.shape))
# # detected = self.detect_yolo_pyyolo(img_arr)
# cv2.imwrite(tmpfile,img_arr)
# detected = self.tracker.next_frame(tmpfile)
# resp.data = serializer.dumps({"data": detected})
# resp.status = falcon.HTTP_200
except:
raise falcon.HTTPBadRequest("Something went wrong in post :(", traceback.format_exc())
res.status = falcon.HTTP_203
# res.body = json.dumps({'status': 1, 'message': 'success','data':json.dumps(detected)})
print('\n\nFINAL BEING SENT {}'.format(detected))
res.body = json.dumps(detected)
