def write_log(self, url, output): logfile = '/data/jeremy/caffenets/hydra/production/hydra/logged_hls_output.txt' print('logging output to ' + logfile) out = {'output': output, 'url': url} with open(logfile, 'a') as fp: # output.append = {'url':url} json.dumps(out, fp, indent=4)
def write_log(self, url, output): with open( '/data/jeremy/caffenets/hydra/production/hydra/tg_logged_output.txt', 'a+') as fp: #a+ not a to create if file doesnt exist out = output out['url'] = url json.dumps(out, fp, indent=4)
def test_exodus(self): adapter = DictAdapter({ 'a': [jaweson.dumps(TestObject()) for x in range(10)], 'b': [jaweson.dumps(AnotherObject()) for x in range(5)], }) Exodus.migrate_database(adapter) assert set(adapter.db.keys()) == set(['b', 'c', 'version']) assert len(filter(lambda obj: 'b' in obj, adapter.db['c'])) == len(adapter.db['c']) assert adapter.db['version'] == Exodus.highest_version()
def test_object_migration(self): # individual migration a = json.loads(jaweson.dumps(TestObject())) assert 'b' not in a a = Exodus.migrate_object(a) assert 'b' in a b = json.loads(jaweson.dumps(AnotherObject())) assert 'b' not in b Exodus.migrate_object(b) assert 'b' not in b
def test_non_serialisable(self): class NonSerialisableObject(object): def __init__(self): self.a = 1 obj = NonSerialisableObject() with self.assertRaises(TypeError): msgpack.dumps(obj) with self.assertRaises(TypeError): json.dumps(obj)
def test_class_variable(self): class ClassVariableObject(jaweson.Serialisable): __classname = 'TestImHere' a = 1 obj = ClassVariableObject() mobj = msgpack.loads(msgpack.dumps(obj)) jobj = json.loads(json.dumps(obj)) assert obj.a == 1 assert obj.a == mobj.a assert obj.a == jobj.a assert '_Serialisable__a' not in json.dumps(obj)
def test_datetime(self): obj = datetime(2013, 3, 27, 23, 5) mobj = msgpack.loads(msgpack.dumps(obj)) jobj = json.loads(json.dumps(obj)) assert obj == mobj assert obj == jobj eastern = pytz.timezone('US/Eastern') obj = datetime(2013, 3, 27, 23, 5, tzinfo=eastern) mobj = msgpack.loads(msgpack.dumps(obj)) jobj = json.loads(json.dumps(obj)) assert obj == mobj assert obj == jobj
def test_manual_object_migration(self): tm1 = self.migration1() tm2 = self.migration2() a = json.loads(jaweson.dumps(TestObject())) assert 'b' not in a tm1.migrate_object(a) assert 'b' in a tm2.migrate_object(a) assert 'b' in a b = json.loads(jaweson.dumps(AnotherObject())) tm1.migrate_object(b) assert 'b' not in b tm2.migrate_object(b) assert 'b' not in b
def test_date(self): obj = date(2013, 3, 27) mobj = msgpack.loads(msgpack.dumps(obj)) jobj = json.loads(json.dumps(obj)) assert obj == mobj assert obj == jobj
def on_get(self, req, resp): """Handles GET requests""" quote = { 'quote': 'just work already ', 'author': 'jeremy rutman' } resp.body = json.dumps(quote)
def test_modified_class_variable(self): class ClassVariableOverrideObject(jaweson.Serialisable): a = 1 def __init__(self): self.a = 2 obj = ClassVariableOverrideObject() mobj = msgpack.loads(msgpack.dumps(obj)) jobj = json.loads(json.dumps(obj)) assert obj.a == 2 assert obj.a == mobj.a assert obj.a == jobj.a assert '_Serialisable__a' not in json.dumps(obj) assert 'a' in json.dumps(obj)
def test_set(self): obj = set([1, 2, 3]) mobj = msgpack.loads(msgpack.dumps(obj)) jobj = json.loads(json.dumps(obj)) assert obj == mobj assert isinstance(mobj, set) assert obj == jobj assert isinstance(jobj, set)
def test_list(self): obj = [1, 2, 3] mobj = msgpack.loads(msgpack.dumps(obj)) jobj = json.loads(json.dumps(obj)) assert obj == mobj assert isinstance(mobj, list) assert obj == jobj assert isinstance(jobj, list)
def test_dict(self): obj = {'a': 1, 'b': 2, 'c': 3} mobj = msgpack.loads(msgpack.dumps(obj)) jobj = json.loads(json.dumps(obj)) assert obj == mobj assert isinstance(mobj, dict) assert obj == jobj assert isinstance(jobj, dict)
def test_float(self): obj = 3.14159 mobj = msgpack.loads(msgpack.dumps(obj)) jobj = json.loads(json.dumps(obj)) assert obj == mobj assert isinstance(mobj, float) assert obj == jobj assert isinstance(jobj, float)
def test_complex(self): obj = complex(123) mobj = msgpack.loads(msgpack.dumps(obj)) jobj = json.loads(json.dumps(obj)) assert obj == mobj assert isinstance(mobj, complex) assert obj == jobj assert isinstance(jobj, complex)
def test_str(self): obj = "abc" mobj = msgpack.loads(msgpack.dumps(obj)) jobj = json.loads(json.dumps(obj)) assert obj == mobj assert isinstance(mobj, (str, unicode)) assert obj == jobj assert isinstance(jobj, (str, unicode))
def on_get(self, req, resp): """Handles GET requests""" quote = { 'quote': 'I\'ve always been more interested in the future than in the past.', 'author': 'Grace Hopper' } resp.body = json.dumps(quote)
def test_int(self): obj = 1 mobj = msgpack.loads(msgpack.dumps(obj)) jobj = json.loads(json.dumps(obj)) assert obj == mobj assert isinstance(mobj, int) assert obj == jobj assert isinstance(jobj, int)
def test_dict_int_keys(self): obj = {1: 1, 2: 2, 3: 3} mobj = msgpack.loads(msgpack.dumps(obj)) jobj = json.loads(json.dumps(obj)) assert obj == mobj assert isinstance(mobj, dict) # we cannot test this as JSON will convert int keys to strings #assert obj == jobj assert isinstance(jobj, dict)
def test_classname(self): class NewClass(json.Serialisable): __classname = 'OldClass' def __init__(self): self.a = 1 a = NewClass() assert a.a is 1 j = json.dumps(a) assert 'OldClass' in j
def test_npgeneric(self): obj = np.float32(1) mobj = msgpack.loads(msgpack.dumps(obj)) jobj = json.loads(json.dumps(obj)) assert obj.dtype == np.float32 assert obj == mobj assert obj.dtype == mobj.dtype assert isinstance(mobj, np.generic) assert obj == jobj assert obj.dtype == jobj.dtype assert isinstance(jobj, np.generic)
def test_migrations(self): adapter = DictAdapter({ 'a': [jaweson.dumps(TestObject()) for x in range(10)], 'b': [jaweson.dumps(AnotherObject()) for x in range(5)], }) tm1 = self.migration1() tm1.migrate_database(adapter) assert set(adapter.db.keys()) == set(['a', 'b', 'version']) assert len(filter(lambda obj: json.loads(obj)['a'] == 1, adapter.db['a'])) == len(adapter.db['a']) assert len(filter(lambda obj: json.loads(obj)['a'] == 2, adapter.db['b'])) == len(adapter.db['b']) assert len(filter(lambda obj: 'b' in json.loads(obj), adapter.db['a'])) == len(adapter.db['a']) assert len(filter(lambda obj: 'b' not in json.loads(obj), adapter.db['b'])) == len(adapter.db['b']) tm2 = self.migration2() tm2.migrate_database(adapter) assert set(adapter.db.keys()) == set(['b', 'c', 'version']) assert len(filter(lambda obj: json.loads(obj)['a'] == 1, adapter.db['c'])) == len(adapter.db['c']) assert len(filter(lambda obj: json.loads(obj)['b'] == 2, adapter.db['c'])) == len(adapter.db['c']) assert len(filter(lambda obj: json.loads(obj)['a'] == 2, adapter.db['b'])) == len(adapter.db['b']) assert len(filter(lambda obj: 'b' not in json.loads(obj), adapter.db['b'])) == len(adapter.db['b'])
def test_ndarray(self): obj = np.array([1, 2, 3], dtype=np.float32) mobj = msgpack.loads(msgpack.dumps(obj)) jobj = json.loads(json.dumps(obj)) assert obj.dtype == np.float32 assert (obj == mobj).all() assert obj.dtype == mobj.dtype assert isinstance(mobj, np.ndarray) assert (obj == jobj).all() assert obj.dtype == jobj.dtype assert isinstance(jobj, np.ndarray)
def test_jaweson_migration(self): class A(jaweson.Serialisable, Migrant): def __init__(self): self.a = 1 a = A() assert a.a is 1 assert not hasattr(a, "b") a = json.loads(json.dumps(a)) assert a.a is 1 assert not hasattr(a, "b") a = a.migrate(a) assert a.a is 1 assert hasattr(a, "b") assert a.b is 2 a = json.loads(json.dumps(a)) assert a.a is 1 assert hasattr(a, "b") assert a.b is 2
def test_serialisable(self): class SerialisableObject(jaweson.Serialisable): def __init__(self): self.a = 1 obj = SerialisableObject() mobj = msgpack.loads(msgpack.dumps(obj)) jobj = json.loads(json.dumps(obj)) assert obj.a == 1 assert obj.a == mobj.a assert obj.a == jobj.a
def test_serialisable_constructor(self): class SerialisableConstructorObject(jaweson.Serialisable): def __init__(self, a): self.a = a obj = SerialisableConstructorObject(2) mobj = msgpack.loads(msgpack.dumps(obj)) jobj = json.loads(json.dumps(obj)) assert obj.a == 2 assert obj.a == mobj.a assert obj.a == jobj.a
def on_get(self, req, resp): ret = {"success": False} try: image_url = req.get_param("imageUrl") ret = { "data": self.feature.execute(image_url), "labels": self.labels, "success": True } except Exception as e: ret["error"] = str(e) ret["trace"] = traceback.format_exc() resp.body = json.dumps(ret)
def init(): #print sys.argv[1] inputFromPhp = json.loads(sys.argv[1]) file_path = inputFromPhp['image_file_path'] operation = inputFromPhp['operation'] data = inputFromPhp['data'] sender = RpcSender() im = array(Image.open(file_path)) msg = {"image_path" : file_path, "image" : im, "operation" : operation, "model_parameters" : data} json_msg = json.dumps(msg) response = sender.call(json_msg) print response
def on_post(self, req, resp): print "Reached hydra on_post" gpu = req.get_param('gpu') ret = {"success": False} # try: # data = msgpack.loads(req.stream.read()) data = req.stream.read() # print('data coming into hydra:'+str(data)) print('hydra falcon') dict = json.loads(data) img = dict.get("image") # img = data['name'] # img = data.split('"')[1] # img = data if isinstance(img, basestring): print('url coming to hydra falcon:' + str(img)) else: print('img arr into hydra falcon size:' + str(img.shape)) # img_arr=Utils.get_cv2_img_array(img) # frcnn_output = self.get_fcrnn_output(self,img) # hydra_output = multilabel_from_hydra_hls.get_hydra_output(img,detection_threshold=0.9) hydra_output = multilabel_from_hydra_tg.get_hydra_output(img) if "sweater_binary_h_iter_50000" in hydra_output: del hydra_output["sweater_binary_h_iter_50000"] if "sweatshirt_binary_h_iter_14000" in hydra_output: del hydra_output["sweatshirt_binary_h_iter_14000"] if "backpack_hydra_iter_2000" in hydra_output: del hydra_output["backpack_hydra_iter_2000"] del hydra_output[ "url"] #dont need this , its an array anyway lately ret["output"] = hydra_output if ret["output"] is not None: ret["success"] = True else: ret["error"] = "No output from mlb" # self.write_log(url,output) except Exception as e: traceback.print_exc() ret["error"] = traceback.format_exc() # resp.data = msgpack.dumps(ret) # resp.content_type = 'application/x-msgpack' resp.data = json.dumps(ret) #maybe dump instead of dumps? resp.content_type = 'application/json' resp.status = falcon.HTTP_200
def test_classname_inheritance(self): class ClassA(json.Serialisable): __classname = 'ThisClass' def __init__(self): self.a = 1 class ClassB(ClassA): def __init__(self): self.a = 2 a = ClassA() b = ClassB() assert a.a is 1 assert b.a is 2 ja = json.dumps(a) jb = json.dumps(b) assert 'ThisClass' in ja assert 'ClassB' in jb a2 = json.loads(ja) b2 = json.loads(jb) assert a2.a is 1 assert b2.a is 2
def get_hydra_output(self, subimage): ''' get hydra details on an image :param subimage: np array , e..g a crop of the original which fcrnn has found :return: ''' data = json.dumps({"image": subimage}) print('defense falcon is attempting to get response from hydra at ' + str(HYDRA_CLASSIFIER_ADDRESS)) try: resp = requests.post(HYDRA_CLASSIFIER_ADDRESS, data=data) dict = json.loads(resp.content) return dict['output'] except: print('couldnt get hydra output') return None
def test_dodgy_constructor(self): class DodgyConstructor(jaweson.Serialisable): def __init__(self, a, b): # flip the order self.a = b self.b = a obj = DodgyConstructor(1, 2) mobj = msgpack.loads(msgpack.dumps(obj)) jobj = json.loads(json.dumps(obj)) assert obj.a == 2 assert obj.b == 1 assert obj.a == mobj.a assert obj.b == mobj.b assert obj.a == jobj.a assert obj.b == jobj.b
def on_get(self, req, resp): ret = {"success": False} try: image_url = req.get_param("imageUrl") image = get_cv2_img_array(image_url) face_rects = self.face_detector(image, 1) faces = [[rect.left(), rect.top(), rect.width(), rect.height()] for rect in face_rects] ret = { "data": new_genderDetector.theDetector(image, faces[0]), "success": True } except Exception as e: ret["error"] = str(e) ret["trace"] = traceback.format_exc() resp.body = json.dumps(ret)
def test_hierarchy(self): class Node(jaweson.Serialisable): def __init__(self, name, child=None): self.name = name self.child = child obj = Node(1, Node(2, Node(3))) mobj = msgpack.loads(msgpack.dumps(obj)) jobj = json.loads(json.dumps(obj)) assert obj.name == 1 assert obj.child.name == 2 assert obj.child.child.name == 3 assert isinstance(obj, Node) assert obj.name == mobj.name assert obj.child.name == mobj.child.name assert obj.child.child.name == mobj.child.child.name assert obj.name == jobj.name assert obj.child.name == jobj.child.name assert obj.child.child.name == jobj.child.child.name
def test_custom_serialiser(self): class CustomSerialiserObject(jaweson.Serialisable): @classmethod def to_json(cls, obj): return {'c': obj.a, 'd': obj.b} @classmethod def from_json(cls, obj): return cls(int(obj['c']), int(obj['d'])) def __init__(self, a, b): self.a = a self.b = b obj = CustomSerialiserObject(1, 2) mobj = msgpack.loads(msgpack.dumps(obj)) jobj = json.loads(json.dumps(obj)) assert obj.a == 1 assert obj.b == 2 assert obj.a == mobj.a assert obj.b == mobj.b assert obj.a == jobj.a assert obj.b == jobj.b
def on_get(self, req, resp): print "Reached on_get" ret = {"success": False} # Query Params category_index = req.get_param_as_int('categoryIndex') threshold = req.get_param('threshold') get_multilabel_results = req.get_param_as_bool('getMultilabelResults') get_combined_results = req.get_param_as_bool('getCombinedResults') image_url = req.get_param('imageUrl', required=True) try: img = image_url # Utils.get_cv2_img_array(image_url) # multilabel alone if get_multilabel_results: multilabel_output = neurodoll.get_multilabel_output(img) ret['multilabel_output'] = multilabel_output print('multilabel output:' + str(multilabel_output)) if multilabel_output is not None: ret["success"] = True # ret["success"] = bool(multilabel_output) # combined multilabel and nd if get_combined_results: combined_output = neurodoll.combine_neurodoll_v3labels_and_multilabel( img) # ret['combined_output'] = combined_output # ret['mask'] = combined_output if combined_output is not None: ret['results_page'] = "http://13.69.27.202:8099/" ret['success'] = True ret['found_categories'] = list(np.unique(combined_output)) ret['bbs'] = imutils.mask_to_rects(combined_output) # yonti style - single category mask ret["label_dict"] = constants.ultimate_21_dict # regular neurodoll call if not get_multilabel_results and not get_combined_results and not category_index: print "No special params, inferring..." ret["mask"], labels = neurodoll.infer_one(img) if ret["mask"] is not None: ret["success"] = True ret["bbs"] = imutils.mask_to_rects(ret["mask"]) else: ret["error"] = "No mask from ND" except Exception as e: traceback.print_exc() ret["error"] = traceback.format_exc() if ret["success"] and ret.get("redirect"): raise falcon.HTTPFound("http://13.69.27.202:8099/") resp.data = json.dumps(ret) resp.content_type = 'application/json' resp.status = falcon.HTTP_200 return (ret)
def on_post(self, req, res): #untested print('\nStarting combine_gunicorn tf hls (got a post request)') start_time = time.time() tmpfile = '/data/jeremy/image_dbs/variant/viettel_demo/' N = 10 randstring = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(N)) tmpfile = os.path.join(tmpfile, randstring + '.jpg') sent_timestamp = 0 # serializer = msgpack # res.content_type = "application/x-msgpack" try: json_data = json.loads(req.stream.read().decode('utf8')) if 'sent_timestamp' in json_data: sent_timestamp = float(json_data['sent_timestamp']) print('sent timestamp {}'.format(sent_timestamp)) else: sent_timestamp = 0 # xfer_time = time.time() - sent_timestamp print('xfer time:{}'.format(xfer_time)) base64encoded_image = json_data['image_data'] data = base64.b64decode(base64encoded_image) # print('data type {}'.format(type(data))) with open(tmpfile, "wb") as fh: fh.write(data) print('wrote file to {}, elapsed time for xfer {}'.format( tmpfile, time.time() - start_time)) decode_time = time.time() - start_time try: print('db2') # imgpath = '/data/jeremy/tensorflow/tmp.jpg' # tmpfile = cv2.imwrite(imgpath,img_arr) detected = tf_detect.analyze_image(tmpfile, thresh=0.2) except: raise falcon.HTTPBadRequest( "Something went wrong in get section 3:(", traceback.format_exc()) try: print('db4') res.data = json.dumps(detected) res.status = falcon.HTTP_200 except: raise falcon.HTTPBadRequest( "Something went wrong in get section 4:(", traceback.format_exc()) try: self.write_log('id', detected) except: raise falcon.HTTPBadRequest( "Something went wrong in get section 5 (wrte log):(", traceback.format_exc()) # stream = req.stream.read() # print('stream {}'.format(stream)) # data = serializer.loads(stream) # print('data:{}'.format(data)) # img_arr = data.get("image") # print('img arr shape {}'.format(img_arr.shape)) # # detected = self.detect_yolo_pyyolo(img_arr) # cv2.imwrite(tmpfile,img_arr) # detected = self.tracker.next_frame(tmpfile) # resp.data = serializer.dumps({"data": detected}) # resp.status = falcon.HTTP_200 except: raise falcon.HTTPBadRequest("Something went wrong in post :(", traceback.format_exc()) res.status = falcon.HTTP_203 res.body = json.dumps({ 'status': 1, 'message': 'success', 'data': json.dumps(detected) })
def on_post(self, req, res): #untested print('\nStarting combine_gunicorn (got a post request)') start_time=time.time() tmpfile = '/data/jeremy/image_dbs/' N=10 randstring = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(N)) tmpfile = os.path.join(tmpfile,randstring+'.jpg') sent_timestamp=0 # serializer = msgpack # res.content_type = "application/x-msgpack" try: json_data = json.loads(req.stream.read().decode('utf8')) if 'sent_timestamp' in json_data: sent_timestamp = float(json_data['sent_timestamp']) print('sent timestamp {}'.format(sent_timestamp)) else: sent_timestamp=0 # if 'vietel_format' in json_data: vietel_format = json_data['vietel_formt'] else: vietel_format = False xfer_time = time.time()-sent_timestamp print('xfer time:{}'.format(xfer_time)) base64encoded_image = json_data['image_data'] data = base64.b64decode(base64encoded_image) # print('data type {}'.format(type(data))) with open(tmpfile, "wb") as fh: fh.write(data) print('wrote file to {}, elapsed time for xfer {}'.format(tmpfile,time.time()-start_time)) decode_time = time.time()-start_time try: print('db2') # imgpath = '/data/jeremy/tensorflow/tmp.jpg' # tmpfile = cv2.imwrite(imgpath,img_arr) img_arr=cv2.imread(tmpfile) detected = self.detect_yolo_pyyolo(img_arr) ##desired #example results {"emotion": -0.027538346377856619, "n_currently_in_frame": 2, "transfer_time": 0.05023598670959473, "detect time": 1.9451310634613037, # "detections": [{"bbox_xywh": [32, 68, 172, 306], "confidence": 0.997, "object": "person 0", "id": 0}, {"bbox_xywh": [254, 36, 205, 294], "confidence": 0.981, "object": "person 1", "id": 1}], "n_total": 2, "line_length": 0} #actual # {"data": [{"confidence": 0.6014, "object": "car", "bbox": [305, 178, 344, 195], "details": {"color": "gray"}}]} #"detections": [{"bbox_x if vietel_format: detected_vietel_format={} detected_vietel_format['emotion']=0 print('det data {}'.format(detected)) detected_vietel_format['detections']=detected for det in detected_vietel_format['detections']: det['bbox_xywh']=det['bbox'] del det['bbox'] detected = detected_vietel_format # detected = tf_detect.analyze_image(tmpfile,thresh=0.2) except: raise falcon.HTTPBadRequest("Something went wrong in get section 3:(", traceback.format_exc()) try: print('db4') res.data = json.dumps(detected) res.status = falcon.HTTP_200 except: raise falcon.HTTPBadRequest("Something went wrong in get section 4:(", traceback.format_exc()) try: self.write_log('id',detected) except: raise falcon.HTTPBadRequest("Something went wrong in get section 5 (wrte log):(", traceback.format_exc()) # stream = req.stream.read() # print('stream {}'.format(stream)) # data = serializer.loads(stream) # print('data:{}'.format(data)) # img_arr = data.get("image") # print('img arr shape {}'.format(img_arr.shape)) # # detected = self.detect_yolo_pyyolo(img_arr) # cv2.imwrite(tmpfile,img_arr) # detected = self.tracker.next_frame(tmpfile) # resp.data = serializer.dumps({"data": detected}) # resp.status = falcon.HTTP_200 except: raise falcon.HTTPBadRequest("Something went wrong in post :(", traceback.format_exc()) res.status = falcon.HTTP_203 # res.body = json.dumps({'status': 1, 'message': 'success','data':json.dumps(detected)}) print('\n\nFINAL BEING SENT {}'.format(detected)) res.body = json.dumps(detected)
self.corr_id = str(uuid.uuid4()) self.channel.basic_publish(exchange='', routing_key='_argus_queue', properties=pika.BasicProperties( reply_to = self.callback_queue, correlation_id = self.corr_id, ), body=n) while self.response is None: self.connection.process_data_events() return self.response def init(): #print sys.argv[1] try: inputFromPhp = json.loads(sys.argv[1]) except Exception, e: print str(e) operation = inputFromPhp['operation'] data = inputFromPhp['data'] sender = RpcSender() msg = {"operation" : operation, "data": data} json_msg = json.dumps(msg) response = sender.call(json_msg) print response if __name__ == "__main__": init()
def on_get(self, req, resp): # """Handles GET requests""" quote = {'msg': 'responding to posts... ', 'author': 'jeremy rutman'} resp.body = json.dumps(quote)