def encode(n, proto_factory=TBinaryProtocolFactory()):
ab = make_addressbook()
start = time.time()
for i in range(n):
serialize(ab, proto_factory)
end = time.time()
print("encode\t-> {}".format(end - start))
def encode(n, proto_factory=TBinaryProtocolFactory()):
ab = make_addressbook()
start = time.time()
for i in range(n):
serialize(ab, proto_factory)
end = time.time()
print("encode\t-> {}".format(end - start))
def testSerializeThenDeserialize(self):
obj = Xtruct2(i32_thing=1, struct_thing=Xtruct(string_thing="foo"))
s1 = serialize(obj)
for i in range(10):
self.assertEquals(s1, serialize(obj))
objcopy = Xtruct2()
deserialize(objcopy, serialize(obj))
self.assertEquals(obj, objcopy)
obj = Xtruct(string_thing="bar")
objcopy = Xtruct()
deserialize(objcopy, serialize(obj))
self.assertEquals(obj, objcopy)
def testSerializeThenDeserialize(self):
obj = Xtruct2(i32_thing=1,
struct_thing=Xtruct(string_thing="foo"))
s1 = serialize(obj)
for i in range(10):
self.assertEquals(s1, serialize(obj))
objcopy = Xtruct2()
deserialize(objcopy, serialize(obj))
self.assertEquals(obj, objcopy)
obj = Xtruct(string_thing="bar")
objcopy = Xtruct()
deserialize(objcopy, serialize(obj))
self.assertEquals(obj, objcopy)
def test_launchTask_deserialization_fail(self): # noqa
proxy_driver = ProxyDriver()
role = getpass.getuser()
task_info = mesos_pb2.TaskInfo()
task_info.name = task_info.task_id.value = "broken"
task_info.data = serialize(
AssignedTask(
task=TaskConfig(
job=JobKey(role=role, environment="env", name="name"),
owner=Identity(role=role, user=role),
executorConfig=ExecutorConfig(name=AURORA_EXECUTOR_NAME, data="garbage"),
)
)
)
te = FastThermosExecutor(
runner_provider=make_provider(safe_mkdtemp()), sandbox_provider=DefaultTestSandboxProvider()
)
te.launchTask(proxy_driver, task_info)
proxy_driver.wait_stopped()
updates = proxy_driver.method_calls["sendStatusUpdate"]
assert len(updates) == 2
assert updates[0][0][0].state == mesos_pb2.TASK_STARTING
assert updates[1][0][0].state == mesos_pb2.TASK_FAILED
def render_quota(self, write_json, quota_resp):
def get_quota_json(quota):
result = {}
result['cpu'] = quota.numCpus
result['ram'] = float(quota.ramMb) / 1024
result['disk'] = float(quota.diskMb) / 1024
return result
def get_quota_str(quota):
result = []
result.append(' CPU: %s' % quota.numCpus)
result.append(' RAM: %f GB' % (float(quota.ramMb) / 1024))
result.append(' Disk: %f GB' % (float(quota.diskMb) / 1024))
return result
if write_json:
return serialize(
quota_resp.result.getQuotaResult,
protocol_factory=TJSONProtocol.TSimpleJSONProtocolFactory())
else:
quota_result = quota_resp.result.getQuotaResult
result = ['Allocated:']
result += get_quota_str(quota_result.quota)
if quota_result.prodConsumption:
result.append('Production resources consumed:')
result += get_quota_str(quota_result.prodConsumption)
if quota_result.nonProdConsumption:
result.append('Non-production resources consumed:')
result += get_quota_str(quota_result.nonProdConsumption)
return '\n'.join(result)
def render_quota(self, write_json, quota_resp):
def get_quota_json(quota):
result = {}
result["cpu"] = quota.numCpus
result["ram"] = float(quota.ramMb) / 1024
result["disk"] = float(quota.diskMb) / 1024
return result
def get_quota_str(quota):
result = []
result.append(" CPU: %s" % quota.numCpus)
result.append(" RAM: %f GB" % (float(quota.ramMb) / 1024))
result.append(" Disk: %f GB" % (float(quota.diskMb) / 1024))
return result
if write_json:
return serialize(
quota_resp.result.getQuotaResult, protocol_factory=TJSONProtocol.TSimpleJSONProtocolFactory()
)
else:
quota_result = quota_resp.result.getQuotaResult
result = ["Allocated:"]
result += get_quota_str(quota_result.quota)
if quota_result.prodConsumption:
result.append("Production resources consumed:")
result += get_quota_str(quota_result.prodConsumption)
if quota_result.nonProdConsumption:
result.append("Non-production resources consumed:")
result += get_quota_str(quota_result.nonProdConsumption)
return "\n".join(result)
def render_quota(self, write_json, quota_resp):
def get_quota_json(quota):
result = {}
result['cpu'] = quota.numCpus
result['ram'] = float(quota.ramMb) / 1024
result['disk'] = float(quota.diskMb) / 1024
return result
def get_quota_str(quota):
result = []
result.append(' CPU: %s' % quota.numCpus)
result.append(' RAM: %f GB' % (float(quota.ramMb) / 1024))
result.append(' Disk: %f GB' % (float(quota.diskMb) / 1024))
return result
if write_json:
return serialize(quota_resp.result.getQuotaResult,
protocol_factory=TJSONProtocol.TSimpleJSONProtocolFactory())
else:
quota_result = quota_resp.result.getQuotaResult
result = ['Allocated:']
result += get_quota_str(quota_result.quota)
if quota_result.prodConsumption:
result.append('Production resources consumed:')
result += get_quota_str(quota_result.prodConsumption)
if quota_result.nonProdConsumption:
result.append('Non-production resources consumed:')
result += get_quota_str(quota_result.nonProdConsumption)
return '\n'.join(result)
def render_task_json(scheduled_task):
"""Render a single task into json. This is baroque, but it uses thrift to
give us all of the job status data, while allowing us to compose it with
other stuff and pretty-print it.
"""
return json.loads(serialize(scheduled_task,
protocol_factory=TJSONProtocol.TSimpleJSONProtocolFactory()))
def _bench_thrift(loops=1000):
"""Measure using a thrift-generated library N times.
The target is a simple addressbook. We measure the following:
* create an addressbook with 1 person in it
* serialize it
* deserialize it into a new addressbook
For each iteration we repeat this 100 times.
"""
# proto_factory = TBinaryProtocolFactory()
proto_factory = TBinaryProtocolAcceleratedFactory()
elapsed = 0
times = []
for _ in range(loops):
# This is a macro benchmark for a Python implementation
# so "elapsed" covers more than just how long the Addressbook ops take.
t0 = pyperf.perf_counter()
for _ in range(100):
# First, create the addressbook.
ab = make_addressbook()
# Then, round-trip through serialization.
encoded = serialize(ab, proto_factory)
ab2 = ttypes.AddressBook()
deserialize(ab2, encoded, proto_factory)
t1 = pyperf.perf_counter()
elapsed += t1 - t0
times.append(t0)
times.append(pyperf.perf_counter())
return elapsed, times
def render_quota(self, write_json, quota_resp):
def get_quota_str(quota):
resource_details = ResourceManager.resource_details_from_quota(quota)
return (' %s: %s%s' % (
r.resource_type.display_name,
r.value,
r.resource_type.display_unit) for r in resource_details)
if write_json:
return serialize(quota_resp.result.getQuotaResult,
protocol_factory=TJSONProtocol.TSimpleJSONProtocolFactory())
else:
quota_result = quota_resp.result.getQuotaResult
result = ['Allocated:']
result += get_quota_str(quota_result.quota)
if quota_result.prodSharedConsumption:
result.append('Production shared pool resources consumed:')
result += get_quota_str(quota_result.prodSharedConsumption)
if quota_result.prodDedicatedConsumption:
result.append('Production dedicated pool resources consumed:')
result += get_quota_str(quota_result.prodDedicatedConsumption)
if quota_result.nonProdSharedConsumption:
result.append('Non-production shared pool resources consumed:')
result += get_quota_str(quota_result.nonProdSharedConsumption)
if quota_result.nonProdDedicatedConsumption:
result.append('Non-production dedicated pool resources consumed:')
result += get_quota_str(quota_result.nonProdDedicatedConsumption)
return '\n'.join(result)
def writeFiniteVolumeIndexData(finiteVolumeIndexFile, finiteVolumeIndexData):
assert isinstance(finiteVolumeIndexData, FiniteVolumeIndexData)
blob = serialize(finiteVolumeIndexData,
protocol_factory=TBinaryProtocol.TBinaryProtocolFactory())
ff = open(finiteVolumeIndexFile, 'wb')
ff.write(blob)
ff.close()
print("wrote finitevolume data to file " + str(finiteVolumeIndexFile))
def decode(n, proto_factory=TBinaryProtocolFactory()):
ab = ttypes.AddressBook()
ab_encoded = serialize(make_addressbook())
start = time.time()
for i in range(n):
deserialize(ab, ab_encoded, proto_factory)
end = time.time()
print("decode\t-> {}".format(end - start))
def decode(n, proto_factory=TBinaryProtocolFactory()):
ab = ttypes.AddressBook()
ab_encoded = serialize(make_addressbook())
start = time.time()
for i in range(n):
deserialize(ab, ab_encoded, proto_factory)
end = time.time()
print("decode\t-> {}".format(end - start))
def next(self):
if self.num <= self.count:
cur, self.num = str(self.num), self.num + 1
msg = KafkaMessage(cur)
s_msg = serialize(msg)
return DictionaryMock(value=s_msg)
else:
raise StopIteration()
def writeMovingBoundaryIndexData(movingBoundaryIndexFile,
movingBoundaryIndexData):
assert isinstance(movingBoundaryIndexData, MovingBoundaryIndexData)
blob = serialize(movingBoundaryIndexData,
protocol_factory=TBinaryProtocol.TBinaryProtocolFactory())
ff = open(movingBoundaryIndexFile, 'wb')
ff.write(blob)
ff.close()
print("wrote movingboundary data to file " + str(movingBoundaryIndexFile))
def create_address(host, port):
"""Serialize the given address to thrift ServerAddress.
:type host: str
:type port: int
:rtype: str
"""
address = ServerAddress(host, port)
return serialize(address)
def serialize_content(content):
""" Серилизация контента товара для БД Cassandra.
:param content: контент, который необходимо сериализовать
:return: строка
"""
p = WarehouseMethods.build_dict_for_WareContentDto(content)
content = WarehouseMethods.get_WareContentDto(**p)
content_obj = serialize(content)
service_log.put("Serialized content: %s" % str(content_obj))
return content_obj
def verify(self, serialized, factory):
self.assertEqual(serialized, serialize(self.message, factory))
self.assertEqual(
"hello thrift",
deserialize(Message(), serialized, factory).body,
)
self.assertEqual(42, deserialize(Message(), serialized, factory).num)
self.assertRaises(EOFError, deserialize, Message(), b'', factory)
def next_tuple(self):
try:
msg_body = next(self.msg_bodies)
msg = Message(type=MsgType.TYPE_A, body=msg_body, score=98.7654321)
ser_msg = b2a_base64(serialize(msg))
self.log('emitting message: {}'.format(msg))
self.log('serialized message: {}'.format(ser_msg))
self.emit([ser_msg], tup_id=hash(msg))
except StopIteration:
sleep(10)
pass
def testSerializeThenDeserialize(self):
obj = Xtruct2(i32_thing=1,
struct_thing=Xtruct(string_thing="foo"))
s1 = serialize(obj)
for i in range(10):
self.assertEquals(s1, serialize(obj))
objcopy = Xtruct2()
deserialize(objcopy, serialize(obj))
self.assertEquals(obj, objcopy)
obj = Xtruct(string_thing="bar")
objcopy = Xtruct()
deserialize(objcopy, serialize(obj))
self.assertEquals(obj, objcopy)
# test booleans
obj = Bools(im_true=True, im_false=False)
objcopy = Bools()
deserialize(objcopy, serialize(obj))
self.assertEquals(obj, objcopy)
# test enums
for num, name in Numberz._VALUES_TO_NAMES.items():
obj = Bonk(message='enum Numberz value %d is string %s' % (num, name), type=num)
objcopy = Bonk()
deserialize(objcopy, serialize(obj))
self.assertEquals(obj, objcopy)
def testSerializeThenDeserialize(self):
obj = Xtruct2(i32_thing=1, struct_thing=Xtruct(string_thing="foo"))
s1 = serialize(obj)
for i in range(10):
self.assertEquals(s1, serialize(obj))
objcopy = Xtruct2()
deserialize(objcopy, serialize(obj))
self.assertEquals(obj, objcopy)
obj = Xtruct(string_thing="bar")
objcopy = Xtruct()
deserialize(objcopy, serialize(obj))
self.assertEquals(obj, objcopy)
# test booleans
obj = Bools(im_true=True, im_false=False)
objcopy = Bools()
deserialize(objcopy, serialize(obj))
self.assertEquals(obj, objcopy)
# test enums
for num, name in Numberz._VALUES_TO_NAMES.items():
obj = Bonk(message='enum Numberz value %d is string %s' %
(num, name),
type=num)
objcopy = Bonk()
deserialize(objcopy, serialize(obj))
self.assertEquals(obj, objcopy)
def writeChomboIndexData(chomboIndexFile, chomboIndexData):
"""
Returns:
None:
"""
assert isinstance(chomboIndexData, ChomboIndexData)
blob = serialize(chomboIndexData,
protocol_factory=TBinaryProtocol.TBinaryProtocolFactory())
ff = open(chomboIndexFile, 'wb')
ff.write(blob)
ff.close()
print("wrote chomboIndex data to file " + str(chomboIndexFile))
def make_task(thermos_config, assigned_ports={}, **kw):
role = getpass.getuser()
task_id = thermos_config.task().name().get() + '-001'
at = AssignedTask(taskId=task_id,
task=TaskConfig(executorConfig=ExecutorConfig(
name=AURORA_EXECUTOR_NAME,
data=thermos_config.json_dumps()),
owner=Identity(role=role, user=role)),
assignedPorts=assigned_ports,
**kw)
td = mesos_pb2.TaskInfo()
td.task_id.value = task_id
td.name = thermos_config.task().name().get()
td.data = serialize(at)
return td
def test_launchTask_deserialization_fail(self):
proxy_driver = ProxyDriver()
task_info = mesos_pb.TaskInfo()
task_info.name = task_info.task_id.value = 'broken'
task_info.data = serialize(
AssignedTask(task=TaskConfig(executorConfig=ExecutorConfig(
name=AURORA_EXECUTOR_NAME, data='garbage'))))
te = ThermosExecutor(runner_provider=make_provider(safe_mkdtemp()),
sandbox_provider=DefaultTestSandboxProvider)
te.launchTask(proxy_driver, task_info)
updates = proxy_driver.method_calls['sendStatusUpdate']
assert len(updates) == 1
assert updates[0][0][0].state == mesos_pb.TASK_FAILED
def send(self, batch):
"""
Send batch of spans out via AsyncHTTPClient. Any exceptions thrown
will be caught above in the exception handler of _submit().
"""
headers = {'Content-Type': 'application/x-thrift'}
if self.auth_token:
headers['Authorization'] = 'Bearer {}'.format(self.auth_token)
data = serialize(batch)
headers['Content-Length'] = str(len(data))
try:
self.client.post(url=self.url, headers=headers, data=data)
except Exception as e:
raise_with_value(e, 'POST to jaeger_endpoint failed: {}'.format(e))
def test_launchTask_deserialization_fail(self):
proxy_driver = ProxyDriver()
task_info = mesos_pb.TaskInfo()
task_info.name = task_info.task_id.value = 'broken'
task_info.data = serialize(AssignedTask(task=TaskConfig(executorConfig=ExecutorConfig(
name=AURORA_EXECUTOR_NAME,
data='garbage'))))
te = ThermosExecutor(
runner_provider=make_provider(safe_mkdtemp()),
sandbox_provider=DefaultTestSandboxProvider)
te.launchTask(proxy_driver, task_info)
updates = proxy_driver.method_calls['sendStatusUpdate']
assert len(updates) == 1
assert updates[0][0][0].state == mesos_pb.TASK_FAILED
def make_task(thermos_config, assigned_ports={}, **kw):
role = getpass.getuser()
task_id = thermos_config.task().name().get() + '-001'
at = AssignedTask(
taskId=task_id,
task=TaskConfig(
executorConfig=ExecutorConfig(
name=AURORA_EXECUTOR_NAME,
data=thermos_config.json_dumps()),
job=JobKey(role=role, environment='env', name='name')),
assignedPorts=assigned_ports,
**kw)
td = mesos_pb2.TaskInfo()
td.task_id.value = task_id
td.name = thermos_config.task().name().get()
td.data = serialize(at)
return td
def render_task_json(scheduled_task):
"""Render a single task into json. This is baroque, but it uses thrift to
give us all of the job status data, while allowing us to compose it with
other stuff and pretty-print it.
"""
task = json.loads(serialize(scheduled_task,
protocol_factory=TJSONProtocol.TSimpleJSONProtocolFactory()))
# Now, clean it up: take all fields that are actually enums, and convert
# their values to strings.
task['status'] = ScheduleStatus._VALUES_TO_NAMES[task['status']]
for event in task['taskEvents']:
event['status'] = ScheduleStatus._VALUES_TO_NAMES[event['status']]
# convert boolean fields to boolean value names.
assigned = task['assignedTask']
task_config = assigned['task']
task_config['isService'] = (task_config['isService'] != 0)
if 'production' in task_config:
task_config['production'] = (task_config['production'] != 0)
return task
def render_task_json(scheduled_task):
"""Render a single task into json. This is baroque, but it uses thrift to
give us all of the job status data, while allowing us to compose it with
other stuff and pretty-print it.
"""
task = json.loads(serialize(scheduled_task, protocol_factory=TJSONProtocol.TSimpleJSONProtocolFactory()))
# Now, clean it up: take all fields that are actually enums, and convert
# their values to strings.
task["status"] = ScheduleStatus._VALUES_TO_NAMES[task["status"]]
events = sorted(task["taskEvents"], key=lambda event: event["timestamp"])
for event in events:
event["status"] = ScheduleStatus._VALUES_TO_NAMES[event["status"]]
# convert boolean fields to boolean value names.
assigned = task["assignedTask"]
task_config = assigned["task"]
task_config["isService"] = task_config["isService"] != 0
if "production" in task_config:
task_config["production"] = task_config["production"] != 0
return task
def render_task_json(scheduled_task):
"""Render a single task into json. This is baroque, but it uses thrift to
give us all of the job status data, while allowing us to compose it with
other stuff and pretty-print it.
"""
task = json.loads(serialize(scheduled_task,
protocol_factory=TJSONProtocol.TSimpleJSONProtocolFactory()))
# Now, clean it up: take all fields that are actually enums, and convert
# their values to strings.
task["status"] = ScheduleStatus._VALUES_TO_NAMES[task["status"]]
events = sorted(task["taskEvents"], key=lambda event: event["timestamp"])
for event in events:
event["status"] = ScheduleStatus._VALUES_TO_NAMES[event["status"]]
# convert boolean fields to boolean value names.
assigned = task["assignedTask"]
task_config = assigned["task"]
task_config["isService"] = (task_config["isService"] != 0)
if "production" in task_config:
task_config["production"] = (task_config["production"] != 0)
return task
def test_launchTask_deserialization_fail(self): # noqa
proxy_driver = ProxyDriver()
role = getpass.getuser()
task_info = mesos_pb2.TaskInfo()
task_info.name = task_info.task_id.value = 'broken'
task_info.data = serialize(
AssignedTask(task=TaskConfig(
job=JobKey(role=role, environment='env', name='name'),
executorConfig=ExecutorConfig(name=AURORA_EXECUTOR_NAME,
data='garbage'))))
te = FastThermosExecutor(runner_provider=make_provider(safe_mkdtemp()),
sandbox_provider=DefaultTestSandboxProvider())
te.launchTask(proxy_driver, task_info)
proxy_driver.wait_stopped()
updates = proxy_driver.method_calls['sendStatusUpdate']
assert len(updates) == 2
assert updates[0][0][0].state == mesos_pb2.TASK_STARTING
assert updates[1][0][0].state == mesos_pb2.TASK_FAILED
def render_task_json(scheduled_task):
"""Render a single task into json. This is baroque, but it uses thrift to
give us all of the job status data, while allowing us to compose it with
other stuff and pretty-print it.
"""
task = json.loads(
serialize(scheduled_task,
protocol_factory=TJSONProtocol.
TSimpleJSONProtocolFactory()))
# Now, clean it up: take all fields that are actually enums, and convert
# their values to strings.
task['status'] = ScheduleStatus._VALUES_TO_NAMES[task['status']]
for event in task['taskEvents']:
event['status'] = ScheduleStatus._VALUES_TO_NAMES[
event['status']]
# convert boolean fields to boolean value names.
assigned = task['assignedTask']
task_config = assigned['task']
task_config['isService'] = (task_config['isService'] != 0)
if 'production' in task_config:
task_config['production'] = (task_config['production'] != 0)
return task
def iteration_cache(self, op, func, cache_type,
TType, **kwargs):
cache_list = []
for idx, (cache, name, cache_type_) in enumerate(self._c_cache):
if cache and (not cache_type or cache_type == cache_type_):
with self._debug_time:
(value, _return) = func(self, cache, cache_type_, TType, **kwargs)
v = kwargs['value'] if 'value' in kwargs else value
_logging.flowpath_cache(op, kwargs['key'], v, name,
perf_t=self._debug_time.use_time)
if cache_type != None:
return value
# run on *get* method, set no return value
if _return:
if cache_list:
# move cache
expire = 600
key = kwargs['key']
extime = cache.ttl(key)
if extime > 0:
expire = extime
for c, name, cache_type_ in cache_list:
cache_value = value
with self._debug_time:
if cache_type_ != self.ProcessCache and TType:
cache_value = serialize(value)
c.set_value(key, value, expire=expire)
_logging.flowpath_cache('|-set(expire:%s)' % expire,
key, value,
name,
perf_t=self._debug_time.use_time)
return value
cache_list.append((cache, name, cache_type_))
return None
def testSerializeThenDeserialize(self):
obj = Xtruct2(i32_thing=1, struct_thing=Xtruct(string_thing="foo"))
s1 = serialize(obj)
for i in range(10):
self.assertEquals(s1, serialize(obj))
objcopy = Xtruct2()
deserialize(objcopy, serialize(obj))
self.assertEquals(obj, objcopy)
obj = Xtruct(string_thing="bar")
objcopy = Xtruct()
deserialize(objcopy, serialize(obj))
self.assertEquals(obj, objcopy)
# test booleans
obj = Bools(im_true=True, im_false=False)
objcopy = Bools()
deserialize(objcopy, serialize(obj))
self.assertEquals(obj, objcopy)
# test enums
def _enumerate_enum(enum_class):
if hasattr(enum_class, '_VALUES_TO_NAMES'):
# old-style enums
for num, name in enum_class._VALUES_TO_NAMES.items():
yield (num, name)
else:
# assume Python 3.4+ IntEnum-based
from enum import IntEnum
self.assertTrue((issubclass(enum_class, IntEnum)))
for num in enum_class:
yield (num.value, num.name)
for num, name in _enumerate_enum(Numberz):
obj = Bonk(message='enum Numberz value %d is string %s' %
(num, name),
type=num)
objcopy = Bonk()
deserialize(objcopy, serialize(obj))
self.assertEquals(obj, objcopy)
def thrift_to_json(thrift_object):
return json.loads(
serialize(
thrift_object,
protocol_factory=TSimpleJSONProtocolFactory()).decode("utf-8"))
def encoder_map(self, data):
return serialize(test_ttypes.Map(data))
def getThrift(self, domain, key):
return self._exec(lambda client: client.get(domain, serialize(key)))
def encoder_array(self, data):
return serialize(test_ttypes.Array(data))
def _func(self, cache, cache_type, TType, key, value, expire):
if cache_type != self.ProcessCache and TType:
value = serialize(value)
cache.set_value(key, value, expire=expire)
return (None, False)
def encoder_points_list(self, data):
return serialize(test_ttypes.PointsList(data))
def _thrift_to_json(self, config):
return json.loads(
serialize(
config,
protocol_factory=TJSONProtocol.TSimpleJSONProtocolFactory()))
def getThrift(self, domain, key):
return self._exec(lambda client: client.get(domain, serialize(key)))
def encoder_integer(self, data):
return serialize(test_ttypes.Int(data))
def encoder_simple_list(self, data):
return serialize(test_ttypes.SimpleList(data))
def json2thrift(json_str, thrift_class):
return json.loads(json_str, cls=ThriftJSONDecoder, thrift_class=thrift_class)
if __name__ == '__main__':
addr="ws://192.168.20.114:8013/?base64=0&mask=0&binary=1"
ws = create_connection(addr)
a = UserInfo()
a.userId=12321
a.gameId=9999
a.clientId="H5_2.0_weixin.weixin.0-hall20418.weixin.dasfs"
binStr=serialize(a)
bindDict = {
"cmd": "bind_user5",
"subproto": [
"param/thrift/user.UserInfo"
],
"param": binStr
}
packer = msgpack.packb(bindDict, use_bin_type=True)
# 先建立用户连接
ws.send(packer)
retData = ws.recv()
print retData
# 发送第一条消息
b = Action()
b.userId= 12321
def encoder_boolean(self, data):
return serialize(test_ttypes.Bool(data))
def encoder_struct_10(self, data):
return serialize(test_ttypes.Struct10(**data))
def dumps(self, data, **kw):
return serialize(data)
def encoder_bytes(self, data):
return serialize(test_ttypes.Bin(data))
def main():
#sys.setrecursionlimit(100000)
interresults = [None]
try:
#--------------------------------------------------------------------------
# parse input and deserialize the (thrift) optimization problem
#--------------------------------------------------------------------------
parser = argparse.ArgumentParser()
parser.add_argument("optfile",help="filename of input optimization file")
parser.add_argument("resultfile",help="filename of optimization results")
args = parser.parse_args()
assert isinstance(args.resultfile, str)
resultFile = args.resultfile
dir_path = os.path.dirname(os.path.realpath(resultFile))
interresults[0]=os.path.join(dir_path,"interresults.txt")
f_optfile = open(args.optfile, "rb")
blob_opt = f_optfile.read()
print("read "+str(len(blob_opt))+" bytes from "+args.optfile)
f_optfile.close()
'''
struct OptProblem {
1: required string mathModelSbmlFile;
2: required int numberOfOptimizationRuns;
3: required ParameterDescriptionList parameterDescriptionList;
4: required ReferenceVariableList referenceVariableList;
5: required string experimentalDataCSV;
6: required CopasiOptimizationMethod optimizationMethod;
}
'''
vcellOptProblem = VCELLOPT.OptProblem()
protocol_factory = TBinaryProtocol.TBinaryProtocolFactory
deserialize(vcellOptProblem, blob_opt, protocol_factory = protocol_factory())
print("done with deserialization")
#-----------------------------------------------------------------------------
# add CDataModel
#
assert(COPASI.CCopasiRootContainer.getRoot() != None)
# create a datamodel
dataModel = COPASI.CCopasiRootContainer.addDatamodel();
assert(isinstance(dataModel,COPASI.CCopasiDataModel))
assert(COPASI.CCopasiRootContainer.getDatamodelList().size() == 1)
try:
#sbmlFile = "C:\\COPASI-4.19.140-Source\\copasi\\bindings\\python\\examples\\exampleDeni.xml"
sbmlString = vcellOptProblem.mathModelSbmlContents
dataModel.importSBMLFromString(str(sbmlString))
print("data model loaded")
except:
e_info = sys.exc_info()
traceback.print_exception(e_info[0],e_info[1],e_info[2],file=sys.stdout)
sys.stderr.write("exception: error importing sbml file: "+str(e_info[0])+": "+str(e_info[1])+"\n")
sys.stderr.flush()
return -1
model = dataModel.getModel()
assert(isinstance(model,COPASI.CModel))
model.compileIfNecessary()
printModel(model)
mathContainer = model.getMathContainer()
assert(isinstance(mathContainer,COPASI.CMathContainer))
# dataModel.saveModel("optModel.cps", True)
#---------------------------------------------------------------------------
# add CFitTask
#---------------------------------------------------------------------------
fitTask = dataModel.addTask(COPASI.CTaskEnum.parameterFitting)
assert(isinstance(fitTask, COPASI.CFitTask))
'''
enum OptimizationMethodType {
EvolutionaryProgram,
SRES,
GeneticAlgorithm,
GeneticAlgorithmSR,
HookeJeeves,
LevenbergMarquardt,
NelderMead,
ParticleSwarm,
RandomSearch,
SimulatedAnnealing,
SteepestDescent,
Praxis,
TruncatedNewton
}
'''
methodTypeDict = dict()
methodTypeDict[VCELLOPT.OptimizationMethodType.EvolutionaryProgram] = COPASI.CTaskEnum.EvolutionaryProgram
methodTypeDict[VCELLOPT.OptimizationMethodType.SRES] = COPASI.CTaskEnum.SRES
methodTypeDict[VCELLOPT.OptimizationMethodType.GeneticAlgorithm] = COPASI.CTaskEnum.GeneticAlgorithm
methodTypeDict[VCELLOPT.OptimizationMethodType.GeneticAlgorithmSR] = COPASI.CTaskEnum.GeneticAlgorithmSR
methodTypeDict[VCELLOPT.OptimizationMethodType.HookeJeeves] = COPASI.CTaskEnum.HookeJeeves
methodTypeDict[VCELLOPT.OptimizationMethodType.LevenbergMarquardt] = COPASI.CTaskEnum.LevenbergMarquardt
methodTypeDict[VCELLOPT.OptimizationMethodType.NelderMead] = COPASI.CTaskEnum.NelderMead
methodTypeDict[VCELLOPT.OptimizationMethodType.ParticleSwarm] = COPASI.CTaskEnum.ParticleSwarm
methodTypeDict[VCELLOPT.OptimizationMethodType.RandomSearch] = COPASI.CTaskEnum.RandomSearch
methodTypeDict[VCELLOPT.OptimizationMethodType.SimulatedAnnealing] = COPASI.CTaskEnum.SimulatedAnnealing
methodTypeDict[VCELLOPT.OptimizationMethodType.SteepestDescent] = COPASI.CTaskEnum.SteepestDescent
methodTypeDict[VCELLOPT.OptimizationMethodType.Praxis] = COPASI.CTaskEnum.Praxis
methodTypeDict[VCELLOPT.OptimizationMethodType.TruncatedNewton] = COPASI.CTaskEnum.TruncatedNewton
#
# set CFitMethod
#
copasiFitMethodType = methodTypeDict[vcellOptProblem.optimizationMethod.optimizationMethodType]
if (copasiFitMethodType not in fitTask.getValidMethods()):
print "fit method not allowed"
return 1
fitTask.setMethodType(copasiFitMethodType)
fitMethod = fitTask.getMethod()
assert(isinstance(fitMethod,COPASI.COptMethod))
'''
enum OptimizationParameterType {
Number_of_Generations,
Number_of_Iterations,
Population_Size,
Random_Number_Generator,
Seed,
IterationLimit,
Tolerance,
Rho,
Scale,
Swarm_Size,
Std_Deviation,
Start_Temperature,
Cooling_Factor,
Pf
}
'''
methodParamDict = dict()
methodParamDict[VCELLOPT.OptimizationParameterType.Number_of_Generations] = "Number of Generations"
methodParamDict[VCELLOPT.OptimizationParameterType.Number_of_Iterations] = "Number of Iterations"
methodParamDict[VCELLOPT.OptimizationParameterType.Population_Size] = "Population Size"
methodParamDict[VCELLOPT.OptimizationParameterType.Random_Number_Generator] = "Random Number Generator"
methodParamDict[VCELLOPT.OptimizationParameterType.Seed] = "Seed"
methodParamDict[VCELLOPT.OptimizationParameterType.IterationLimit] = "Iteration Limit"
methodParamDict[VCELLOPT.OptimizationParameterType.Tolerance] = "Tolerance"
methodParamDict[VCELLOPT.OptimizationParameterType.Rho] = "Rho"
methodParamDict[VCELLOPT.OptimizationParameterType.Scale] = "Scale"
methodParamDict[VCELLOPT.OptimizationParameterType.Swarm_Size] = "Swarm Size"
methodParamDict[VCELLOPT.OptimizationParameterType.Std_Deviation] = "Std Deviation"
methodParamDict[VCELLOPT.OptimizationParameterType.Start_Temperature] = "Start Temperature"
methodParamDict[VCELLOPT.OptimizationParameterType.Cooling_Factor] = "Cooling Factor"
methodParamDict[VCELLOPT.OptimizationParameterType.Pf] = "Pf"
#
# set FitMethod parameters
#
'''
<CopasiOptimizationMethod name="Evolutionary Programming">
<CopasiOptimizationParameter name="Number of Generations" value="200.0" dataType="int"/>
<CopasiOptimizationParameter name="Population Size" value="20.0" dataType="int"/>
<CopasiOptimizationParameter name="Random Number Generator" value="1.0" dataType="int"/>
<CopasiOptimizationParameter name="Seed" value="0.0" dataType="int"/>
</CopasiOptimizationMethod>
<Method name="Evolutionary Programming" type="EvolutionaryProgram">
<Parameter name="Number of Generations" type="unsignedInteger" value="444"/>
<Parameter name="Population Size" type="unsignedInteger" value="77"/>
<Parameter name="Random Number Generator" type="unsignedInteger" value="5"/>
<Parameter name="Seed" type="unsignedInteger" value="44"/>
</Method>
'''
vcellOptParamList = vcellOptProblem.optimizationMethod.optimizationParameterList
assert(isinstance(vcellOptParamList, list))
for vcellOptParam in vcellOptParamList:
assert(isinstance(vcellOptParam,VCELLOPT.CopasiOptimizationParameter))
print methodParamDict[vcellOptParam.paramType]
fitMethod.removeParameter(methodParamDict[vcellOptParam.paramType])
if (vcellOptParam.dataType == VCELLOPT.OptimizationParameterDataType.INT):
fitMethod.addParameter(methodParamDict[vcellOptParam.paramType], COPASI.CCopasiParameter.INT)
fitParameter = fitMethod.getParameter(methodParamDict[vcellOptParam.paramType])
assert (isinstance(fitParameter, COPASI.CCopasiParameter))
fitParameter.setIntValue(int(vcellOptParam.value))
else:
fitMethod.addParameter(methodParamDict[vcellOptParam.paramType], COPASI.CCopasiParameter.DOUBLE)
fitParameter = fitMethod.getParameter(methodParamDict[vcellOptParam.paramType])
assert (isinstance(fitParameter, COPASI.CCopasiParameter))
fitParameter.setDblValue(vcellOptParam.value)
#
# get FitProblem
#
fitProblem = fitTask.getProblem()
assert(isinstance(fitProblem, COPASI.COptProblem)) # works for all COPASI builds >= 140
#assert(isinstance(fitProblem, COPASI.CFitProblem)) # not CFitProblem in COPASI build 140
#fitProblem.setRandomizeStartValues(True)
experimentSet = fitProblem.getParameter("Experiment Set")
assert(isinstance(experimentSet,COPASI.CExperimentSet))
assert(experimentSet.getExperimentCount() == 0)
# first experiment
experiment = COPASI.CExperiment(dataModel)
assert(isinstance(experiment,COPASI.CExperiment))
experiment.setIsRowOriented(True)
# Use the TemporaryFile context manager for easy clean-up
tmpExportCSVFile = tempfile.NamedTemporaryFile(delete=False)
varNameList = list(v.varName for v in vcellOptProblem.referenceVariableList)
csvString = ", ".join(map(str, varNameList)) + "\n"
for row in vcellOptProblem.experimentalDataSet.rows:
csvString += ", ".join(map(str, row.data)) + "\n"
num_lines = len(vcellOptProblem.experimentalDataSet.rows) + 1
tmpExportCSVFile.write(csvString)
tmpExportCSVFile.close()
experiment.setFileName(str(tmpExportCSVFile.name))
experiment.setFirstRow(1)
experiment.setKeyValue("Experiment_1")
experiment.setLastRow(num_lines)
experiment.setHeaderRow(1)
experiment.setSeparator(",")
experiment.setExperimentType(COPASI.CTaskEnum.timeCourse)
experiment.setNormalizeWeightsPerExperiment(True)
vcellReferenceVariableList = vcellOptProblem.referenceVariableList
assert(isinstance(vcellReferenceVariableList, list))
num_ref_variables = len(vcellReferenceVariableList)
experiment.setNumColumns(num_ref_variables) # one independent (time), all the other dependent
# experiment object map
objectMap = experiment.getObjectMap()
assert (isinstance(objectMap, COPASI.CExperimentObjectMap))
result = objectMap.setNumCols(num_ref_variables)
assert result == True
# map time column to model time
'''
<variable type="independent" name="t"/>
'''
result = objectMap.setRole(0, COPASI.CExperiment.time)
assert (result==True)
assert objectMap.getRole(0) == COPASI.CExperiment.time
timeReference = model.getValueReference()
#timeReference = model.getObject(COPASI.CCopasiObjectName("Reference=Time"))
assert(timeReference!=None)
assert(isinstance(timeReference,COPASI.CCopasiObject))
objectMap.setObjectCN(0,timeReference.getCN().getString())
# getObjectCN returns a string whereas getCN returns a CCopasiObjectName
assert(objectMap.getObjectCN(0)==timeReference.getCN().getString())
# map rest of data columns as dependent variables
'''
<variable type="dependent" name="C_cyt"/>
<variable type="dependent" name="RanC_cyt"/>
'''
for refIndex in range(1,num_ref_variables): # skip first columnn (time)
refVar = vcellReferenceVariableList[refIndex]
assert(isinstance(refVar,VCELLOPT.ReferenceVariable))
modelValue = getModelValue(model,str(refVar.varName))
assert(isinstance(modelValue,COPASI.CModelValue))
objectMap.setRole(refIndex, COPASI.CExperiment.dependent)
modelValueReference = modelValue.getObject(COPASI.CCopasiObjectName("Reference=Value"))
assert(isinstance(modelValueReference,COPASI.CCopasiObject))
print "modelValue CN is "+str(modelValue.getCN())+", modelValueReference CN is "+str(modelValueReference.getCN())
objectMap.setObjectCN(refIndex, modelValueReference.getCN().getString())
experimentSet.addExperiment(experiment) # addExperiment makes a copy
assert experimentSet.getExperimentCount() == 1
experiment = experimentSet.getExperiment(0) # need to get the correct instance
assert(isinstance(experiment,COPASI.CExperiment))
#---------------------------------------------------------------------------------------
# define CFitItems
#---------------------------------------------------------------------------------------
'''
<parameterDescription>
<parameter name="Kf" low="0.1" high="10.0" init="5.0" scale="5.0"/>
<parameter name="Kr" low="100.0" high="10000.0" init="500.0" scale="500.0"/>
</parameterDescription>
'''
assert(fitProblem.getOptItemSize()==0)
vcellParameterDescriptionList = vcellOptProblem.parameterDescriptionList
for vcellParam in vcellParameterDescriptionList:
assert(isinstance(vcellParam,VCELLOPT.ParameterDescription))
paramModelValue = getModelValue(model,str(vcellParam.name))
assert(isinstance(paramModelValue,COPASI.CModelValue))
paramModelValueRef = paramModelValue.getInitialValueReference()
assert(isinstance(paramModelValueRef,COPASI.CCopasiObject))
fitItem = COPASI.CFitItem(dataModel)
assert(isinstance(fitItem,COPASI.CFitItem))
fitItem.setObjectCN(paramModelValueRef.getCN())
fitItem.setStartValue(vcellParam.initialValue)
fitItem.setLowerBound(COPASI.CCopasiObjectName(str(vcellParam.minValue)))
fitItem.setUpperBound(COPASI.CCopasiObjectName(str(vcellParam.maxValue)))
# todo: what about scale?
# add the fit item to the correct parameter group
optimizationItemGroup = fitProblem.getParameter("OptimizationItemList")
assert(isinstance(optimizationItemGroup,COPASI.CCopasiParameterGroup))
optimizationItemGroup.addParameter(fitItem)
# addParameter makes a copy of the fit item, so we have to get it back
#fitItem = optimizationItemGroup.getParameter(0)
#assert(isinstance(fitItem,COPASI.CFitItem))
model.compileIfNecessary()
#print optimizationItemGroup.printToString() # debug anyway, not present in COPASI build 140
# --------------------------------------------------------------------------------------
# Run the optimization (N times)
# --------------------------------------------------------------------------------------
leastError = 1e8
paramNames = []
paramValues = []
numObjFuncEvals = 0
numParamsToFit = fitProblem.getOptItemSize()
# Create intermediate results file
with open(interresults[0], 'a') as foutput:
foutput.write(str(vcellOptProblem.numberOfOptimizationRuns)+"\n")
foutput.close()
for i in range(0, vcellOptProblem.numberOfOptimizationRuns):
result = True
try:
print ("This can take some time...")
initialize = (i==0)
result = fitTask.processWithOutputFlags(initialize, COPASI.CCopasiTask.NO_OUTPUT) # NO_OUTPUT
except:
print "Unexpected error:", sys.exc_info()[0]
return 1
if result == False:
sys.stderr.write("An error occured while running the Parameter estimation.\n")
# dataModel.saveModel('test_failed.cps', True)
sys.stderr.write("fitTask warning: '" + str(fitTask.getProcessWarning()) + "'")
sys.stderr.write("fitTask error: '" + str(fitTask.getProcessError()) + "'")
# check if there are additional error messages
if COPASI.CCopasiMessage.size() > 0:
# print the messages in chronological order
sys.stderr.write(COPASI.CCopasiMessage.getAllMessageText(True))
return 1
currentFuncValue = fitProblem.getSolutionValue()
print "currFuncValue = " + str(currentFuncValue)
if (currentFuncValue < leastError) or (i == 0): # current run has the smallest error so far
bestObjectiveFunction = currentFuncValue
numObjFuncEvals = fitProblem.getFunctionEvaluations()
paramNames = []
paramValues = []
for j in range(0, numParamsToFit):
optItem = fitProblem.getOptItemList()[j]
paramName = optItem.getObject().getCN().getRemainder().getRemainder().getElementName(0)
paramNames.append(paramName)
paramValue = fitProblem.getSolutionVariables().get(j)
paramValues.append(paramValue)
print "param " + paramName + " --> " + str(paramValue)
assert isinstance(currentFuncValue, float)
leastError = currentFuncValue
with open(interresults[0], 'a') as foutput:
foutput.write(str(i)+" "+str(leastError)+" "+str(numObjFuncEvals)+"\n")
foutput.close()
#result = dataModel.saveModel('test_succeeded.cps', True)
#assert(result==True)
optRun = VCELLOPT.OptRun()
optRun.optProblem = vcellOptProblem
optRun.statusMessage = "complete"
optRun.status = VCELLOPT.OptRunStatus.Complete
optResultSet = VCELLOPT.OptResultSet()
optResultSet.numFunctionEvaluations = numObjFuncEvals
optResultSet.objectiveFunction = leastError
optResultSet.optParameterValues = []
paramValueDict = dict(zip(paramNames,paramValues))
for paramName in paramNames:
optParameterValue = VCELLOPT.OptParameterValue(paramName,paramValueDict[paramName])
optResultSet.optParameterValues.append(optParameterValue)
optRun.optResultSet = optResultSet
protocol_factory = TBinaryProtocol.TBinaryProtocolFactory
optRunBlob = serialize(vcellOptProblem, protocol_factory = protocol_factory())
transportOut = TTransport.TMemoryBuffer()
protocolOut = TBinaryProtocol.TBinaryProtocol(transportOut)
optRun.write(protocolOut)
with open(resultFile, 'wb') as foutput:
foutput.write(transportOut.getvalue())
foutput.close()
with open(interresults[0], 'a') as foutput:
foutput.write("done")
foutput.close()
# writeOptSolverResultSet(resultFile, leastError, numObjFuncEvals, paramNames, paramValues)
except:
e_info = sys.exc_info()
traceback.print_exception(e_info[0],e_info[1],e_info[2],file=sys.stdout)
sys.stderr.write("exception: "+str(e_info[0])+": "+str(e_info[1])+"\n")
sys.stderr.flush()
if interresults[0] != None:
with open(interresults[0], 'a') as foutput:
foutput.write("exception: "+str(e_info[0])+": "+str(e_info[1]))
foutput.close()
return -1
else:
return 0
finally:
os.unlink(tmpExportCSVFile.name)
def send(self, topic, data):
serialized_data = serialize(KafkaMessage(data))
self.producer.send(topic, serialized_data)
def encoder_float(self, data):
return serialize(test_ttypes.Float(data))
def encoder_struct_map(self, data):
return serialize(struct_map_ttypes.StructMap(**data))
def _thrift_to_json(self, config):
return json.loads(
serialize(config, protocol_factory=TJSONProtocol.TSimpleJSONProtocolFactory()))
def toBytes(self):
return serialize(self.message)