def encode(n, proto_factory=TBinaryProtocolFactory()): ab = make_addressbook() start = time.time() for i in range(n): serialize(ab, proto_factory) end = time.time() print("encode\t-> {}".format(end - start))
def testSerializeThenDeserialize(self): obj = Xtruct2(i32_thing=1, struct_thing=Xtruct(string_thing="foo")) s1 = serialize(obj) for i in range(10): self.assertEquals(s1, serialize(obj)) objcopy = Xtruct2() deserialize(objcopy, serialize(obj)) self.assertEquals(obj, objcopy) obj = Xtruct(string_thing="bar") objcopy = Xtruct() deserialize(objcopy, serialize(obj)) self.assertEquals(obj, objcopy)
def test_launchTask_deserialization_fail(self): # noqa proxy_driver = ProxyDriver() role = getpass.getuser() task_info = mesos_pb2.TaskInfo() task_info.name = task_info.task_id.value = "broken" task_info.data = serialize( AssignedTask( task=TaskConfig( job=JobKey(role=role, environment="env", name="name"), owner=Identity(role=role, user=role), executorConfig=ExecutorConfig(name=AURORA_EXECUTOR_NAME, data="garbage"), ) ) ) te = FastThermosExecutor( runner_provider=make_provider(safe_mkdtemp()), sandbox_provider=DefaultTestSandboxProvider() ) te.launchTask(proxy_driver, task_info) proxy_driver.wait_stopped() updates = proxy_driver.method_calls["sendStatusUpdate"] assert len(updates) == 2 assert updates[0][0][0].state == mesos_pb2.TASK_STARTING assert updates[1][0][0].state == mesos_pb2.TASK_FAILED
def render_quota(self, write_json, quota_resp): def get_quota_json(quota): result = {} result['cpu'] = quota.numCpus result['ram'] = float(quota.ramMb) / 1024 result['disk'] = float(quota.diskMb) / 1024 return result def get_quota_str(quota): result = [] result.append(' CPU: %s' % quota.numCpus) result.append(' RAM: %f GB' % (float(quota.ramMb) / 1024)) result.append(' Disk: %f GB' % (float(quota.diskMb) / 1024)) return result if write_json: return serialize( quota_resp.result.getQuotaResult, protocol_factory=TJSONProtocol.TSimpleJSONProtocolFactory()) else: quota_result = quota_resp.result.getQuotaResult result = ['Allocated:'] result += get_quota_str(quota_result.quota) if quota_result.prodConsumption: result.append('Production resources consumed:') result += get_quota_str(quota_result.prodConsumption) if quota_result.nonProdConsumption: result.append('Non-production resources consumed:') result += get_quota_str(quota_result.nonProdConsumption) return '\n'.join(result)
def render_quota(self, write_json, quota_resp): def get_quota_json(quota): result = {} result["cpu"] = quota.numCpus result["ram"] = float(quota.ramMb) / 1024 result["disk"] = float(quota.diskMb) / 1024 return result def get_quota_str(quota): result = [] result.append(" CPU: %s" % quota.numCpus) result.append(" RAM: %f GB" % (float(quota.ramMb) / 1024)) result.append(" Disk: %f GB" % (float(quota.diskMb) / 1024)) return result if write_json: return serialize( quota_resp.result.getQuotaResult, protocol_factory=TJSONProtocol.TSimpleJSONProtocolFactory() ) else: quota_result = quota_resp.result.getQuotaResult result = ["Allocated:"] result += get_quota_str(quota_result.quota) if quota_result.prodConsumption: result.append("Production resources consumed:") result += get_quota_str(quota_result.prodConsumption) if quota_result.nonProdConsumption: result.append("Non-production resources consumed:") result += get_quota_str(quota_result.nonProdConsumption) return "\n".join(result)
def render_quota(self, write_json, quota_resp): def get_quota_json(quota): result = {} result['cpu'] = quota.numCpus result['ram'] = float(quota.ramMb) / 1024 result['disk'] = float(quota.diskMb) / 1024 return result def get_quota_str(quota): result = [] result.append(' CPU: %s' % quota.numCpus) result.append(' RAM: %f GB' % (float(quota.ramMb) / 1024)) result.append(' Disk: %f GB' % (float(quota.diskMb) / 1024)) return result if write_json: return serialize(quota_resp.result.getQuotaResult, protocol_factory=TJSONProtocol.TSimpleJSONProtocolFactory()) else: quota_result = quota_resp.result.getQuotaResult result = ['Allocated:'] result += get_quota_str(quota_result.quota) if quota_result.prodConsumption: result.append('Production resources consumed:') result += get_quota_str(quota_result.prodConsumption) if quota_result.nonProdConsumption: result.append('Non-production resources consumed:') result += get_quota_str(quota_result.nonProdConsumption) return '\n'.join(result)
def render_task_json(scheduled_task): """Render a single task into json. This is baroque, but it uses thrift to give us all of the job status data, while allowing us to compose it with other stuff and pretty-print it. """ return json.loads(serialize(scheduled_task, protocol_factory=TJSONProtocol.TSimpleJSONProtocolFactory()))
def _bench_thrift(loops=1000): """Measure using a thrift-generated library N times. The target is a simple addressbook. We measure the following: * create an addressbook with 1 person in it * serialize it * deserialize it into a new addressbook For each iteration we repeat this 100 times. """ # proto_factory = TBinaryProtocolFactory() proto_factory = TBinaryProtocolAcceleratedFactory() elapsed = 0 times = [] for _ in range(loops): # This is a macro benchmark for a Python implementation # so "elapsed" covers more than just how long the Addressbook ops take. t0 = pyperf.perf_counter() for _ in range(100): # First, create the addressbook. ab = make_addressbook() # Then, round-trip through serialization. encoded = serialize(ab, proto_factory) ab2 = ttypes.AddressBook() deserialize(ab2, encoded, proto_factory) t1 = pyperf.perf_counter() elapsed += t1 - t0 times.append(t0) times.append(pyperf.perf_counter()) return elapsed, times
def render_quota(self, write_json, quota_resp): def get_quota_str(quota): resource_details = ResourceManager.resource_details_from_quota(quota) return (' %s: %s%s' % ( r.resource_type.display_name, r.value, r.resource_type.display_unit) for r in resource_details) if write_json: return serialize(quota_resp.result.getQuotaResult, protocol_factory=TJSONProtocol.TSimpleJSONProtocolFactory()) else: quota_result = quota_resp.result.getQuotaResult result = ['Allocated:'] result += get_quota_str(quota_result.quota) if quota_result.prodSharedConsumption: result.append('Production shared pool resources consumed:') result += get_quota_str(quota_result.prodSharedConsumption) if quota_result.prodDedicatedConsumption: result.append('Production dedicated pool resources consumed:') result += get_quota_str(quota_result.prodDedicatedConsumption) if quota_result.nonProdSharedConsumption: result.append('Non-production shared pool resources consumed:') result += get_quota_str(quota_result.nonProdSharedConsumption) if quota_result.nonProdDedicatedConsumption: result.append('Non-production dedicated pool resources consumed:') result += get_quota_str(quota_result.nonProdDedicatedConsumption) return '\n'.join(result)
def writeFiniteVolumeIndexData(finiteVolumeIndexFile, finiteVolumeIndexData): assert isinstance(finiteVolumeIndexData, FiniteVolumeIndexData) blob = serialize(finiteVolumeIndexData, protocol_factory=TBinaryProtocol.TBinaryProtocolFactory()) ff = open(finiteVolumeIndexFile, 'wb') ff.write(blob) ff.close() print("wrote finitevolume data to file " + str(finiteVolumeIndexFile))
def decode(n, proto_factory=TBinaryProtocolFactory()): ab = ttypes.AddressBook() ab_encoded = serialize(make_addressbook()) start = time.time() for i in range(n): deserialize(ab, ab_encoded, proto_factory) end = time.time() print("decode\t-> {}".format(end - start))
def next(self): if self.num <= self.count: cur, self.num = str(self.num), self.num + 1 msg = KafkaMessage(cur) s_msg = serialize(msg) return DictionaryMock(value=s_msg) else: raise StopIteration()
def writeMovingBoundaryIndexData(movingBoundaryIndexFile, movingBoundaryIndexData): assert isinstance(movingBoundaryIndexData, MovingBoundaryIndexData) blob = serialize(movingBoundaryIndexData, protocol_factory=TBinaryProtocol.TBinaryProtocolFactory()) ff = open(movingBoundaryIndexFile, 'wb') ff.write(blob) ff.close() print("wrote movingboundary data to file " + str(movingBoundaryIndexFile))
def create_address(host, port): """Serialize the given address to thrift ServerAddress. :type host: str :type port: int :rtype: str """ address = ServerAddress(host, port) return serialize(address)
def serialize_content(content): """ Серилизация контента товара для БД Cassandra. :param content: контент, который необходимо сериализовать :return: строка """ p = WarehouseMethods.build_dict_for_WareContentDto(content) content = WarehouseMethods.get_WareContentDto(**p) content_obj = serialize(content) service_log.put("Serialized content: %s" % str(content_obj)) return content_obj
def verify(self, serialized, factory): self.assertEqual(serialized, serialize(self.message, factory)) self.assertEqual( "hello thrift", deserialize(Message(), serialized, factory).body, ) self.assertEqual(42, deserialize(Message(), serialized, factory).num) self.assertRaises(EOFError, deserialize, Message(), b'', factory)
def next_tuple(self): try: msg_body = next(self.msg_bodies) msg = Message(type=MsgType.TYPE_A, body=msg_body, score=98.7654321) ser_msg = b2a_base64(serialize(msg)) self.log('emitting message: {}'.format(msg)) self.log('serialized message: {}'.format(ser_msg)) self.emit([ser_msg], tup_id=hash(msg)) except StopIteration: sleep(10) pass
def testSerializeThenDeserialize(self): obj = Xtruct2(i32_thing=1, struct_thing=Xtruct(string_thing="foo")) s1 = serialize(obj) for i in range(10): self.assertEquals(s1, serialize(obj)) objcopy = Xtruct2() deserialize(objcopy, serialize(obj)) self.assertEquals(obj, objcopy) obj = Xtruct(string_thing="bar") objcopy = Xtruct() deserialize(objcopy, serialize(obj)) self.assertEquals(obj, objcopy) # test booleans obj = Bools(im_true=True, im_false=False) objcopy = Bools() deserialize(objcopy, serialize(obj)) self.assertEquals(obj, objcopy) # test enums for num, name in Numberz._VALUES_TO_NAMES.items(): obj = Bonk(message='enum Numberz value %d is string %s' % (num, name), type=num) objcopy = Bonk() deserialize(objcopy, serialize(obj)) self.assertEquals(obj, objcopy)
def writeChomboIndexData(chomboIndexFile, chomboIndexData): """ Returns: None: """ assert isinstance(chomboIndexData, ChomboIndexData) blob = serialize(chomboIndexData, protocol_factory=TBinaryProtocol.TBinaryProtocolFactory()) ff = open(chomboIndexFile, 'wb') ff.write(blob) ff.close() print("wrote chomboIndex data to file " + str(chomboIndexFile))
def make_task(thermos_config, assigned_ports={}, **kw): role = getpass.getuser() task_id = thermos_config.task().name().get() + '-001' at = AssignedTask(taskId=task_id, task=TaskConfig(executorConfig=ExecutorConfig( name=AURORA_EXECUTOR_NAME, data=thermos_config.json_dumps()), owner=Identity(role=role, user=role)), assignedPorts=assigned_ports, **kw) td = mesos_pb2.TaskInfo() td.task_id.value = task_id td.name = thermos_config.task().name().get() td.data = serialize(at) return td
def test_launchTask_deserialization_fail(self): proxy_driver = ProxyDriver() task_info = mesos_pb.TaskInfo() task_info.name = task_info.task_id.value = 'broken' task_info.data = serialize( AssignedTask(task=TaskConfig(executorConfig=ExecutorConfig( name=AURORA_EXECUTOR_NAME, data='garbage')))) te = ThermosExecutor(runner_provider=make_provider(safe_mkdtemp()), sandbox_provider=DefaultTestSandboxProvider) te.launchTask(proxy_driver, task_info) updates = proxy_driver.method_calls['sendStatusUpdate'] assert len(updates) == 1 assert updates[0][0][0].state == mesos_pb.TASK_FAILED
def send(self, batch): """ Send batch of spans out via AsyncHTTPClient. Any exceptions thrown will be caught above in the exception handler of _submit(). """ headers = {'Content-Type': 'application/x-thrift'} if self.auth_token: headers['Authorization'] = 'Bearer {}'.format(self.auth_token) data = serialize(batch) headers['Content-Length'] = str(len(data)) try: self.client.post(url=self.url, headers=headers, data=data) except Exception as e: raise_with_value(e, 'POST to jaeger_endpoint failed: {}'.format(e))
def test_launchTask_deserialization_fail(self): proxy_driver = ProxyDriver() task_info = mesos_pb.TaskInfo() task_info.name = task_info.task_id.value = 'broken' task_info.data = serialize(AssignedTask(task=TaskConfig(executorConfig=ExecutorConfig( name=AURORA_EXECUTOR_NAME, data='garbage')))) te = ThermosExecutor( runner_provider=make_provider(safe_mkdtemp()), sandbox_provider=DefaultTestSandboxProvider) te.launchTask(proxy_driver, task_info) updates = proxy_driver.method_calls['sendStatusUpdate'] assert len(updates) == 1 assert updates[0][0][0].state == mesos_pb.TASK_FAILED
def make_task(thermos_config, assigned_ports={}, **kw): role = getpass.getuser() task_id = thermos_config.task().name().get() + '-001' at = AssignedTask( taskId=task_id, task=TaskConfig( executorConfig=ExecutorConfig( name=AURORA_EXECUTOR_NAME, data=thermos_config.json_dumps()), job=JobKey(role=role, environment='env', name='name')), assignedPorts=assigned_ports, **kw) td = mesos_pb2.TaskInfo() td.task_id.value = task_id td.name = thermos_config.task().name().get() td.data = serialize(at) return td
def render_task_json(scheduled_task): """Render a single task into json. This is baroque, but it uses thrift to give us all of the job status data, while allowing us to compose it with other stuff and pretty-print it. """ task = json.loads(serialize(scheduled_task, protocol_factory=TJSONProtocol.TSimpleJSONProtocolFactory())) # Now, clean it up: take all fields that are actually enums, and convert # their values to strings. task['status'] = ScheduleStatus._VALUES_TO_NAMES[task['status']] for event in task['taskEvents']: event['status'] = ScheduleStatus._VALUES_TO_NAMES[event['status']] # convert boolean fields to boolean value names. assigned = task['assignedTask'] task_config = assigned['task'] task_config['isService'] = (task_config['isService'] != 0) if 'production' in task_config: task_config['production'] = (task_config['production'] != 0) return task
def render_task_json(scheduled_task): """Render a single task into json. This is baroque, but it uses thrift to give us all of the job status data, while allowing us to compose it with other stuff and pretty-print it. """ task = json.loads(serialize(scheduled_task, protocol_factory=TJSONProtocol.TSimpleJSONProtocolFactory())) # Now, clean it up: take all fields that are actually enums, and convert # their values to strings. task["status"] = ScheduleStatus._VALUES_TO_NAMES[task["status"]] events = sorted(task["taskEvents"], key=lambda event: event["timestamp"]) for event in events: event["status"] = ScheduleStatus._VALUES_TO_NAMES[event["status"]] # convert boolean fields to boolean value names. assigned = task["assignedTask"] task_config = assigned["task"] task_config["isService"] = task_config["isService"] != 0 if "production" in task_config: task_config["production"] = task_config["production"] != 0 return task
def render_task_json(scheduled_task): """Render a single task into json. This is baroque, but it uses thrift to give us all of the job status data, while allowing us to compose it with other stuff and pretty-print it. """ task = json.loads(serialize(scheduled_task, protocol_factory=TJSONProtocol.TSimpleJSONProtocolFactory())) # Now, clean it up: take all fields that are actually enums, and convert # their values to strings. task["status"] = ScheduleStatus._VALUES_TO_NAMES[task["status"]] events = sorted(task["taskEvents"], key=lambda event: event["timestamp"]) for event in events: event["status"] = ScheduleStatus._VALUES_TO_NAMES[event["status"]] # convert boolean fields to boolean value names. assigned = task["assignedTask"] task_config = assigned["task"] task_config["isService"] = (task_config["isService"] != 0) if "production" in task_config: task_config["production"] = (task_config["production"] != 0) return task
def test_launchTask_deserialization_fail(self): # noqa proxy_driver = ProxyDriver() role = getpass.getuser() task_info = mesos_pb2.TaskInfo() task_info.name = task_info.task_id.value = 'broken' task_info.data = serialize( AssignedTask(task=TaskConfig( job=JobKey(role=role, environment='env', name='name'), executorConfig=ExecutorConfig(name=AURORA_EXECUTOR_NAME, data='garbage')))) te = FastThermosExecutor(runner_provider=make_provider(safe_mkdtemp()), sandbox_provider=DefaultTestSandboxProvider()) te.launchTask(proxy_driver, task_info) proxy_driver.wait_stopped() updates = proxy_driver.method_calls['sendStatusUpdate'] assert len(updates) == 2 assert updates[0][0][0].state == mesos_pb2.TASK_STARTING assert updates[1][0][0].state == mesos_pb2.TASK_FAILED
def render_task_json(scheduled_task): """Render a single task into json. This is baroque, but it uses thrift to give us all of the job status data, while allowing us to compose it with other stuff and pretty-print it. """ task = json.loads( serialize(scheduled_task, protocol_factory=TJSONProtocol. TSimpleJSONProtocolFactory())) # Now, clean it up: take all fields that are actually enums, and convert # their values to strings. task['status'] = ScheduleStatus._VALUES_TO_NAMES[task['status']] for event in task['taskEvents']: event['status'] = ScheduleStatus._VALUES_TO_NAMES[ event['status']] # convert boolean fields to boolean value names. assigned = task['assignedTask'] task_config = assigned['task'] task_config['isService'] = (task_config['isService'] != 0) if 'production' in task_config: task_config['production'] = (task_config['production'] != 0) return task
def iteration_cache(self, op, func, cache_type, TType, **kwargs): cache_list = [] for idx, (cache, name, cache_type_) in enumerate(self._c_cache): if cache and (not cache_type or cache_type == cache_type_): with self._debug_time: (value, _return) = func(self, cache, cache_type_, TType, **kwargs) v = kwargs['value'] if 'value' in kwargs else value _logging.flowpath_cache(op, kwargs['key'], v, name, perf_t=self._debug_time.use_time) if cache_type != None: return value # run on *get* method, set no return value if _return: if cache_list: # move cache expire = 600 key = kwargs['key'] extime = cache.ttl(key) if extime > 0: expire = extime for c, name, cache_type_ in cache_list: cache_value = value with self._debug_time: if cache_type_ != self.ProcessCache and TType: cache_value = serialize(value) c.set_value(key, value, expire=expire) _logging.flowpath_cache('|-set(expire:%s)' % expire, key, value, name, perf_t=self._debug_time.use_time) return value cache_list.append((cache, name, cache_type_)) return None
def testSerializeThenDeserialize(self): obj = Xtruct2(i32_thing=1, struct_thing=Xtruct(string_thing="foo")) s1 = serialize(obj) for i in range(10): self.assertEquals(s1, serialize(obj)) objcopy = Xtruct2() deserialize(objcopy, serialize(obj)) self.assertEquals(obj, objcopy) obj = Xtruct(string_thing="bar") objcopy = Xtruct() deserialize(objcopy, serialize(obj)) self.assertEquals(obj, objcopy) # test booleans obj = Bools(im_true=True, im_false=False) objcopy = Bools() deserialize(objcopy, serialize(obj)) self.assertEquals(obj, objcopy) # test enums def _enumerate_enum(enum_class): if hasattr(enum_class, '_VALUES_TO_NAMES'): # old-style enums for num, name in enum_class._VALUES_TO_NAMES.items(): yield (num, name) else: # assume Python 3.4+ IntEnum-based from enum import IntEnum self.assertTrue((issubclass(enum_class, IntEnum))) for num in enum_class: yield (num.value, num.name) for num, name in _enumerate_enum(Numberz): obj = Bonk(message='enum Numberz value %d is string %s' % (num, name), type=num) objcopy = Bonk() deserialize(objcopy, serialize(obj)) self.assertEquals(obj, objcopy)
def thrift_to_json(thrift_object): return json.loads( serialize( thrift_object, protocol_factory=TSimpleJSONProtocolFactory()).decode("utf-8"))
def encoder_map(self, data): return serialize(test_ttypes.Map(data))
def getThrift(self, domain, key): return self._exec(lambda client: client.get(domain, serialize(key)))
def encoder_array(self, data): return serialize(test_ttypes.Array(data))
def _func(self, cache, cache_type, TType, key, value, expire): if cache_type != self.ProcessCache and TType: value = serialize(value) cache.set_value(key, value, expire=expire) return (None, False)
def encoder_points_list(self, data): return serialize(test_ttypes.PointsList(data))
def _thrift_to_json(self, config): return json.loads( serialize( config, protocol_factory=TJSONProtocol.TSimpleJSONProtocolFactory()))
def encoder_integer(self, data): return serialize(test_ttypes.Int(data))
def encoder_simple_list(self, data): return serialize(test_ttypes.SimpleList(data))
def json2thrift(json_str, thrift_class): return json.loads(json_str, cls=ThriftJSONDecoder, thrift_class=thrift_class) if __name__ == '__main__': addr="ws://192.168.20.114:8013/?base64=0&mask=0&binary=1" ws = create_connection(addr) a = UserInfo() a.userId=12321 a.gameId=9999 a.clientId="H5_2.0_weixin.weixin.0-hall20418.weixin.dasfs" binStr=serialize(a) bindDict = { "cmd": "bind_user5", "subproto": [ "param/thrift/user.UserInfo" ], "param": binStr } packer = msgpack.packb(bindDict, use_bin_type=True) # 先建立用户连接 ws.send(packer) retData = ws.recv() print retData # 发送第一条消息 b = Action() b.userId= 12321
def encoder_boolean(self, data): return serialize(test_ttypes.Bool(data))
def encoder_struct_10(self, data): return serialize(test_ttypes.Struct10(**data))
def dumps(self, data, **kw): return serialize(data)
def encoder_bytes(self, data): return serialize(test_ttypes.Bin(data))
def main(): #sys.setrecursionlimit(100000) interresults = [None] try: #-------------------------------------------------------------------------- # parse input and deserialize the (thrift) optimization problem #-------------------------------------------------------------------------- parser = argparse.ArgumentParser() parser.add_argument("optfile",help="filename of input optimization file") parser.add_argument("resultfile",help="filename of optimization results") args = parser.parse_args() assert isinstance(args.resultfile, str) resultFile = args.resultfile dir_path = os.path.dirname(os.path.realpath(resultFile)) interresults[0]=os.path.join(dir_path,"interresults.txt") f_optfile = open(args.optfile, "rb") blob_opt = f_optfile.read() print("read "+str(len(blob_opt))+" bytes from "+args.optfile) f_optfile.close() ''' struct OptProblem { 1: required string mathModelSbmlFile; 2: required int numberOfOptimizationRuns; 3: required ParameterDescriptionList parameterDescriptionList; 4: required ReferenceVariableList referenceVariableList; 5: required string experimentalDataCSV; 6: required CopasiOptimizationMethod optimizationMethod; } ''' vcellOptProblem = VCELLOPT.OptProblem() protocol_factory = TBinaryProtocol.TBinaryProtocolFactory deserialize(vcellOptProblem, blob_opt, protocol_factory = protocol_factory()) print("done with deserialization") #----------------------------------------------------------------------------- # add CDataModel # assert(COPASI.CCopasiRootContainer.getRoot() != None) # create a datamodel dataModel = COPASI.CCopasiRootContainer.addDatamodel(); assert(isinstance(dataModel,COPASI.CCopasiDataModel)) assert(COPASI.CCopasiRootContainer.getDatamodelList().size() == 1) try: #sbmlFile = "C:\\COPASI-4.19.140-Source\\copasi\\bindings\\python\\examples\\exampleDeni.xml" sbmlString = vcellOptProblem.mathModelSbmlContents dataModel.importSBMLFromString(str(sbmlString)) print("data model loaded") except: e_info = sys.exc_info() traceback.print_exception(e_info[0],e_info[1],e_info[2],file=sys.stdout) sys.stderr.write("exception: error importing sbml file: "+str(e_info[0])+": "+str(e_info[1])+"\n") sys.stderr.flush() return -1 model = dataModel.getModel() assert(isinstance(model,COPASI.CModel)) model.compileIfNecessary() printModel(model) mathContainer = model.getMathContainer() assert(isinstance(mathContainer,COPASI.CMathContainer)) # dataModel.saveModel("optModel.cps", True) #--------------------------------------------------------------------------- # add CFitTask #--------------------------------------------------------------------------- fitTask = dataModel.addTask(COPASI.CTaskEnum.parameterFitting) assert(isinstance(fitTask, COPASI.CFitTask)) ''' enum OptimizationMethodType { EvolutionaryProgram, SRES, GeneticAlgorithm, GeneticAlgorithmSR, HookeJeeves, LevenbergMarquardt, NelderMead, ParticleSwarm, RandomSearch, SimulatedAnnealing, SteepestDescent, Praxis, TruncatedNewton } ''' methodTypeDict = dict() methodTypeDict[VCELLOPT.OptimizationMethodType.EvolutionaryProgram] = COPASI.CTaskEnum.EvolutionaryProgram methodTypeDict[VCELLOPT.OptimizationMethodType.SRES] = COPASI.CTaskEnum.SRES methodTypeDict[VCELLOPT.OptimizationMethodType.GeneticAlgorithm] = COPASI.CTaskEnum.GeneticAlgorithm methodTypeDict[VCELLOPT.OptimizationMethodType.GeneticAlgorithmSR] = COPASI.CTaskEnum.GeneticAlgorithmSR methodTypeDict[VCELLOPT.OptimizationMethodType.HookeJeeves] = COPASI.CTaskEnum.HookeJeeves methodTypeDict[VCELLOPT.OptimizationMethodType.LevenbergMarquardt] = COPASI.CTaskEnum.LevenbergMarquardt methodTypeDict[VCELLOPT.OptimizationMethodType.NelderMead] = COPASI.CTaskEnum.NelderMead methodTypeDict[VCELLOPT.OptimizationMethodType.ParticleSwarm] = COPASI.CTaskEnum.ParticleSwarm methodTypeDict[VCELLOPT.OptimizationMethodType.RandomSearch] = COPASI.CTaskEnum.RandomSearch methodTypeDict[VCELLOPT.OptimizationMethodType.SimulatedAnnealing] = COPASI.CTaskEnum.SimulatedAnnealing methodTypeDict[VCELLOPT.OptimizationMethodType.SteepestDescent] = COPASI.CTaskEnum.SteepestDescent methodTypeDict[VCELLOPT.OptimizationMethodType.Praxis] = COPASI.CTaskEnum.Praxis methodTypeDict[VCELLOPT.OptimizationMethodType.TruncatedNewton] = COPASI.CTaskEnum.TruncatedNewton # # set CFitMethod # copasiFitMethodType = methodTypeDict[vcellOptProblem.optimizationMethod.optimizationMethodType] if (copasiFitMethodType not in fitTask.getValidMethods()): print "fit method not allowed" return 1 fitTask.setMethodType(copasiFitMethodType) fitMethod = fitTask.getMethod() assert(isinstance(fitMethod,COPASI.COptMethod)) ''' enum OptimizationParameterType { Number_of_Generations, Number_of_Iterations, Population_Size, Random_Number_Generator, Seed, IterationLimit, Tolerance, Rho, Scale, Swarm_Size, Std_Deviation, Start_Temperature, Cooling_Factor, Pf } ''' methodParamDict = dict() methodParamDict[VCELLOPT.OptimizationParameterType.Number_of_Generations] = "Number of Generations" methodParamDict[VCELLOPT.OptimizationParameterType.Number_of_Iterations] = "Number of Iterations" methodParamDict[VCELLOPT.OptimizationParameterType.Population_Size] = "Population Size" methodParamDict[VCELLOPT.OptimizationParameterType.Random_Number_Generator] = "Random Number Generator" methodParamDict[VCELLOPT.OptimizationParameterType.Seed] = "Seed" methodParamDict[VCELLOPT.OptimizationParameterType.IterationLimit] = "Iteration Limit" methodParamDict[VCELLOPT.OptimizationParameterType.Tolerance] = "Tolerance" methodParamDict[VCELLOPT.OptimizationParameterType.Rho] = "Rho" methodParamDict[VCELLOPT.OptimizationParameterType.Scale] = "Scale" methodParamDict[VCELLOPT.OptimizationParameterType.Swarm_Size] = "Swarm Size" methodParamDict[VCELLOPT.OptimizationParameterType.Std_Deviation] = "Std Deviation" methodParamDict[VCELLOPT.OptimizationParameterType.Start_Temperature] = "Start Temperature" methodParamDict[VCELLOPT.OptimizationParameterType.Cooling_Factor] = "Cooling Factor" methodParamDict[VCELLOPT.OptimizationParameterType.Pf] = "Pf" # # set FitMethod parameters # ''' <CopasiOptimizationMethod name="Evolutionary Programming"> <CopasiOptimizationParameter name="Number of Generations" value="200.0" dataType="int"/> <CopasiOptimizationParameter name="Population Size" value="20.0" dataType="int"/> <CopasiOptimizationParameter name="Random Number Generator" value="1.0" dataType="int"/> <CopasiOptimizationParameter name="Seed" value="0.0" dataType="int"/> </CopasiOptimizationMethod> <Method name="Evolutionary Programming" type="EvolutionaryProgram"> <Parameter name="Number of Generations" type="unsignedInteger" value="444"/> <Parameter name="Population Size" type="unsignedInteger" value="77"/> <Parameter name="Random Number Generator" type="unsignedInteger" value="5"/> <Parameter name="Seed" type="unsignedInteger" value="44"/> </Method> ''' vcellOptParamList = vcellOptProblem.optimizationMethod.optimizationParameterList assert(isinstance(vcellOptParamList, list)) for vcellOptParam in vcellOptParamList: assert(isinstance(vcellOptParam,VCELLOPT.CopasiOptimizationParameter)) print methodParamDict[vcellOptParam.paramType] fitMethod.removeParameter(methodParamDict[vcellOptParam.paramType]) if (vcellOptParam.dataType == VCELLOPT.OptimizationParameterDataType.INT): fitMethod.addParameter(methodParamDict[vcellOptParam.paramType], COPASI.CCopasiParameter.INT) fitParameter = fitMethod.getParameter(methodParamDict[vcellOptParam.paramType]) assert (isinstance(fitParameter, COPASI.CCopasiParameter)) fitParameter.setIntValue(int(vcellOptParam.value)) else: fitMethod.addParameter(methodParamDict[vcellOptParam.paramType], COPASI.CCopasiParameter.DOUBLE) fitParameter = fitMethod.getParameter(methodParamDict[vcellOptParam.paramType]) assert (isinstance(fitParameter, COPASI.CCopasiParameter)) fitParameter.setDblValue(vcellOptParam.value) # # get FitProblem # fitProblem = fitTask.getProblem() assert(isinstance(fitProblem, COPASI.COptProblem)) # works for all COPASI builds >= 140 #assert(isinstance(fitProblem, COPASI.CFitProblem)) # not CFitProblem in COPASI build 140 #fitProblem.setRandomizeStartValues(True) experimentSet = fitProblem.getParameter("Experiment Set") assert(isinstance(experimentSet,COPASI.CExperimentSet)) assert(experimentSet.getExperimentCount() == 0) # first experiment experiment = COPASI.CExperiment(dataModel) assert(isinstance(experiment,COPASI.CExperiment)) experiment.setIsRowOriented(True) # Use the TemporaryFile context manager for easy clean-up tmpExportCSVFile = tempfile.NamedTemporaryFile(delete=False) varNameList = list(v.varName for v in vcellOptProblem.referenceVariableList) csvString = ", ".join(map(str, varNameList)) + "\n" for row in vcellOptProblem.experimentalDataSet.rows: csvString += ", ".join(map(str, row.data)) + "\n" num_lines = len(vcellOptProblem.experimentalDataSet.rows) + 1 tmpExportCSVFile.write(csvString) tmpExportCSVFile.close() experiment.setFileName(str(tmpExportCSVFile.name)) experiment.setFirstRow(1) experiment.setKeyValue("Experiment_1") experiment.setLastRow(num_lines) experiment.setHeaderRow(1) experiment.setSeparator(",") experiment.setExperimentType(COPASI.CTaskEnum.timeCourse) experiment.setNormalizeWeightsPerExperiment(True) vcellReferenceVariableList = vcellOptProblem.referenceVariableList assert(isinstance(vcellReferenceVariableList, list)) num_ref_variables = len(vcellReferenceVariableList) experiment.setNumColumns(num_ref_variables) # one independent (time), all the other dependent # experiment object map objectMap = experiment.getObjectMap() assert (isinstance(objectMap, COPASI.CExperimentObjectMap)) result = objectMap.setNumCols(num_ref_variables) assert result == True # map time column to model time ''' <variable type="independent" name="t"/> ''' result = objectMap.setRole(0, COPASI.CExperiment.time) assert (result==True) assert objectMap.getRole(0) == COPASI.CExperiment.time timeReference = model.getValueReference() #timeReference = model.getObject(COPASI.CCopasiObjectName("Reference=Time")) assert(timeReference!=None) assert(isinstance(timeReference,COPASI.CCopasiObject)) objectMap.setObjectCN(0,timeReference.getCN().getString()) # getObjectCN returns a string whereas getCN returns a CCopasiObjectName assert(objectMap.getObjectCN(0)==timeReference.getCN().getString()) # map rest of data columns as dependent variables ''' <variable type="dependent" name="C_cyt"/> <variable type="dependent" name="RanC_cyt"/> ''' for refIndex in range(1,num_ref_variables): # skip first columnn (time) refVar = vcellReferenceVariableList[refIndex] assert(isinstance(refVar,VCELLOPT.ReferenceVariable)) modelValue = getModelValue(model,str(refVar.varName)) assert(isinstance(modelValue,COPASI.CModelValue)) objectMap.setRole(refIndex, COPASI.CExperiment.dependent) modelValueReference = modelValue.getObject(COPASI.CCopasiObjectName("Reference=Value")) assert(isinstance(modelValueReference,COPASI.CCopasiObject)) print "modelValue CN is "+str(modelValue.getCN())+", modelValueReference CN is "+str(modelValueReference.getCN()) objectMap.setObjectCN(refIndex, modelValueReference.getCN().getString()) experimentSet.addExperiment(experiment) # addExperiment makes a copy assert experimentSet.getExperimentCount() == 1 experiment = experimentSet.getExperiment(0) # need to get the correct instance assert(isinstance(experiment,COPASI.CExperiment)) #--------------------------------------------------------------------------------------- # define CFitItems #--------------------------------------------------------------------------------------- ''' <parameterDescription> <parameter name="Kf" low="0.1" high="10.0" init="5.0" scale="5.0"/> <parameter name="Kr" low="100.0" high="10000.0" init="500.0" scale="500.0"/> </parameterDescription> ''' assert(fitProblem.getOptItemSize()==0) vcellParameterDescriptionList = vcellOptProblem.parameterDescriptionList for vcellParam in vcellParameterDescriptionList: assert(isinstance(vcellParam,VCELLOPT.ParameterDescription)) paramModelValue = getModelValue(model,str(vcellParam.name)) assert(isinstance(paramModelValue,COPASI.CModelValue)) paramModelValueRef = paramModelValue.getInitialValueReference() assert(isinstance(paramModelValueRef,COPASI.CCopasiObject)) fitItem = COPASI.CFitItem(dataModel) assert(isinstance(fitItem,COPASI.CFitItem)) fitItem.setObjectCN(paramModelValueRef.getCN()) fitItem.setStartValue(vcellParam.initialValue) fitItem.setLowerBound(COPASI.CCopasiObjectName(str(vcellParam.minValue))) fitItem.setUpperBound(COPASI.CCopasiObjectName(str(vcellParam.maxValue))) # todo: what about scale? # add the fit item to the correct parameter group optimizationItemGroup = fitProblem.getParameter("OptimizationItemList") assert(isinstance(optimizationItemGroup,COPASI.CCopasiParameterGroup)) optimizationItemGroup.addParameter(fitItem) # addParameter makes a copy of the fit item, so we have to get it back #fitItem = optimizationItemGroup.getParameter(0) #assert(isinstance(fitItem,COPASI.CFitItem)) model.compileIfNecessary() #print optimizationItemGroup.printToString() # debug anyway, not present in COPASI build 140 # -------------------------------------------------------------------------------------- # Run the optimization (N times) # -------------------------------------------------------------------------------------- leastError = 1e8 paramNames = [] paramValues = [] numObjFuncEvals = 0 numParamsToFit = fitProblem.getOptItemSize() # Create intermediate results file with open(interresults[0], 'a') as foutput: foutput.write(str(vcellOptProblem.numberOfOptimizationRuns)+"\n") foutput.close() for i in range(0, vcellOptProblem.numberOfOptimizationRuns): result = True try: print ("This can take some time...") initialize = (i==0) result = fitTask.processWithOutputFlags(initialize, COPASI.CCopasiTask.NO_OUTPUT) # NO_OUTPUT except: print "Unexpected error:", sys.exc_info()[0] return 1 if result == False: sys.stderr.write("An error occured while running the Parameter estimation.\n") # dataModel.saveModel('test_failed.cps', True) sys.stderr.write("fitTask warning: '" + str(fitTask.getProcessWarning()) + "'") sys.stderr.write("fitTask error: '" + str(fitTask.getProcessError()) + "'") # check if there are additional error messages if COPASI.CCopasiMessage.size() > 0: # print the messages in chronological order sys.stderr.write(COPASI.CCopasiMessage.getAllMessageText(True)) return 1 currentFuncValue = fitProblem.getSolutionValue() print "currFuncValue = " + str(currentFuncValue) if (currentFuncValue < leastError) or (i == 0): # current run has the smallest error so far bestObjectiveFunction = currentFuncValue numObjFuncEvals = fitProblem.getFunctionEvaluations() paramNames = [] paramValues = [] for j in range(0, numParamsToFit): optItem = fitProblem.getOptItemList()[j] paramName = optItem.getObject().getCN().getRemainder().getRemainder().getElementName(0) paramNames.append(paramName) paramValue = fitProblem.getSolutionVariables().get(j) paramValues.append(paramValue) print "param " + paramName + " --> " + str(paramValue) assert isinstance(currentFuncValue, float) leastError = currentFuncValue with open(interresults[0], 'a') as foutput: foutput.write(str(i)+" "+str(leastError)+" "+str(numObjFuncEvals)+"\n") foutput.close() #result = dataModel.saveModel('test_succeeded.cps', True) #assert(result==True) optRun = VCELLOPT.OptRun() optRun.optProblem = vcellOptProblem optRun.statusMessage = "complete" optRun.status = VCELLOPT.OptRunStatus.Complete optResultSet = VCELLOPT.OptResultSet() optResultSet.numFunctionEvaluations = numObjFuncEvals optResultSet.objectiveFunction = leastError optResultSet.optParameterValues = [] paramValueDict = dict(zip(paramNames,paramValues)) for paramName in paramNames: optParameterValue = VCELLOPT.OptParameterValue(paramName,paramValueDict[paramName]) optResultSet.optParameterValues.append(optParameterValue) optRun.optResultSet = optResultSet protocol_factory = TBinaryProtocol.TBinaryProtocolFactory optRunBlob = serialize(vcellOptProblem, protocol_factory = protocol_factory()) transportOut = TTransport.TMemoryBuffer() protocolOut = TBinaryProtocol.TBinaryProtocol(transportOut) optRun.write(protocolOut) with open(resultFile, 'wb') as foutput: foutput.write(transportOut.getvalue()) foutput.close() with open(interresults[0], 'a') as foutput: foutput.write("done") foutput.close() # writeOptSolverResultSet(resultFile, leastError, numObjFuncEvals, paramNames, paramValues) except: e_info = sys.exc_info() traceback.print_exception(e_info[0],e_info[1],e_info[2],file=sys.stdout) sys.stderr.write("exception: "+str(e_info[0])+": "+str(e_info[1])+"\n") sys.stderr.flush() if interresults[0] != None: with open(interresults[0], 'a') as foutput: foutput.write("exception: "+str(e_info[0])+": "+str(e_info[1])) foutput.close() return -1 else: return 0 finally: os.unlink(tmpExportCSVFile.name)
def send(self, topic, data): serialized_data = serialize(KafkaMessage(data)) self.producer.send(topic, serialized_data)
def encoder_float(self, data): return serialize(test_ttypes.Float(data))
def encoder_struct_map(self, data): return serialize(struct_map_ttypes.StructMap(**data))
def _thrift_to_json(self, config): return json.loads( serialize(config, protocol_factory=TJSONProtocol.TSimpleJSONProtocolFactory()))
def toBytes(self): return serialize(self.message)